From a85768444250a63d7b289b7ceb08d5e1b2b46d79 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sun, 6 Mar 2022 20:10:11 +0100 Subject: [PATCH] Apply changes in YoutubeStreamExtractor Extract post live DVR streams as post live streams instead of live streams. A new class has been in order to improve code: ItagInfo, which stores an itag, the content (URL) extracted and if its an URL or not. A functional interface has been added in order to abstract the stream building: StreamBuilderHelper. Also add the cver parameter added by the desktop web client on the corresponding streams (a new method has been added in YoutubeParsingHelper to check this and another for Android streams). Some code in these classes has been also refactored/improved/optimized. --- .../extractor/services/youtube/ItagInfo.java | 37 ++ .../youtube/YoutubeParsingHelper.java | 50 ++- .../extractors/YoutubeStreamExtractor.java | 372 +++++++++++------- 3 files changed, 311 insertions(+), 148 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagInfo.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagInfo.java new file mode 100644 index 00000000..cdb5dc2d --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/ItagInfo.java @@ -0,0 +1,37 @@ +package org.schabi.newpipe.extractor.services.youtube; + +import javax.annotation.Nonnull; +import java.io.Serializable; + +public final class ItagInfo implements Serializable { + + @Nonnull + private final String content; + @Nonnull + private final ItagItem itagItem; + private boolean isUrl; + + public ItagInfo(@Nonnull final String content, + @Nonnull final ItagItem itagItem) { + this.content = content; + this.itagItem = itagItem; + } + + public void setIsUrl(final boolean isUrl) { + this.isUrl = isUrl; + } + + @Nonnull + public String getContent() { + return content; + } + + @Nonnull + public ItagItem getItagItem() { + return itagItem; + } + + public boolean getIsUrl() { + return isUrl; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index eddb69cd..28b802b9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -71,6 +71,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Random; +import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -246,6 +247,11 @@ public final class YoutubeParsingHelper { private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; + private static final Pattern C_WEB_PATTERN = Pattern.compile("&c=WEB"); + private static final Pattern C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN = + Pattern.compile("&c=TVHTML5_SIMPLY_EMBEDDED_PLAYER"); + private static final Pattern C_ANDROID_PATTERN = Pattern.compile("&c=ANDROID"); + private static final Pattern C_IOS_PATTERN = Pattern.compile("&c=IOS"); private static boolean isGoogleURL(final String url) { final String cachedUrl = extractCachedUrlIfNeeded(url); @@ -1190,7 +1196,7 @@ public final class YoutubeParsingHelper { @Nonnull final Localization localization, @Nonnull final ContentCountry contentCountry, @Nonnull final String videoId) { - // @formatter:off + // @formatter:off return JsonObject.builder() .object("context") .object("client") @@ -1588,4 +1594,46 @@ public final class YoutubeParsingHelper { return RandomStringFromAlphabetGenerator.generate( CONTENT_PLAYBACK_NONCE_ALPHABET, 12, numberGenerator); } + + /** + * Check if the streaming URL is a URL from the YouTube {@code WEB} client. + * + * @param url the streaming URL on which check if it's a {@code WEB} streaming URL. + * @return true if it's a {@code WEB} streaming URL, false otherwise + */ + public static boolean isWebStreamingUrl(@Nonnull final String url) { + return Parser.isMatch(C_WEB_PATTERN, url); + } + + /** + * Check if the streaming URL is a URL from the YouTube {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} + * client. + * + * @param url the streaming URL on which check if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} + * streaming URL. + * @return true if it's a {@code TVHTML5_SIMPLY_EMBEDDED_PLAYER} streaming URL, false otherwise + */ + public static boolean isTvHtml5SimplyEmbeddedPlayerStreamingUrl(@Nonnull final String url) { + return Parser.isMatch(C_TVHTML5_SIMPLY_EMBEDDED_PLAYER_PATTERN, url); + } + + /** + * Check if the streaming URL is a URL from the YouTube {@code ANDROID} client. + * + * @param url the streaming URL on which check if it's a {@code ANDROID} streaming URL. + * @return true if it's a {@code ANDROID} streaming URL, false otherwise + */ + public static boolean isAndroidStreamingUrl(@Nonnull final String url) { + return Parser.isMatch(C_ANDROID_PATTERN, url); + } + + /** + * Check if the streaming URL is a URL from the YouTube {@code IOS} client. + * + * @param url the streaming URL on which check if it's a {@code IOS} streaming URL. + * @return true if it's a {@code IOS} streaming URL, false otherwise + */ + public static boolean isIosStreamingUrl(@Nonnull final String url) { + return Parser.isMatch(C_IOS_PATTERN, url); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 46676ff3..5617fd12 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -1,3 +1,23 @@ +/* + * Created by Christian Schabesberger on 06.08.15. + * + * Copyright (C) Christian Schabesberger 2019 + * YoutubeStreamExtractor.java is part of NewPipe Extractor. + * + * NewPipe Extractor is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * NewPipe Extractor is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with NewPipe Extractor. If not, see . + */ + package org.schabi.newpipe.extractor.services.youtube.extractors; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.CONTENT_CHECK_OK; @@ -8,10 +28,12 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateContentPlaybackNonce; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.generateTParameter; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonAndroidPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonIosPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isWebStreamingUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareAndroidMobileJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; @@ -44,12 +66,14 @@ import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager; +import org.schabi.newpipe.extractor.services.youtube.ItagInfo; import org.schabi.newpipe.extractor.services.youtube.ItagItem; import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.stream.AudioStream; +import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; import org.schabi.newpipe.extractor.stream.Frameset; import org.schabi.newpipe.extractor.stream.Stream; @@ -64,7 +88,6 @@ import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.OffsetDateTime; @@ -72,7 +95,6 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -82,26 +104,6 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; -/* - * Created by Christian Schabesberger on 06.08.15. - * - * Copyright (C) Christian Schabesberger 2019 - * YoutubeStreamExtractor.java is part of NewPipe. - * - * NewPipe is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * NewPipe is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with NewPipe. If not, see . - */ - public class YoutubeStreamExtractor extends StreamExtractor { /*////////////////////////////////////////////////////////////////////////// // Exceptions @@ -113,7 +115,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - /*//////////////////////////////////////////////////////////////////////////*/ + /*////////////////////////////////////////////////////////////////////////*/ @Nullable private static String cachedDeobfuscationCode = null; @@ -140,8 +142,6 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject playerMicroFormatRenderer; private int ageLimit = -1; private StreamType streamType; - @Nullable - private List subtitles = null; // We need to store the contentPlaybackNonces because we need to append them to videoplayback // URLs (with the cpn parameter). @@ -580,73 +580,25 @@ public class YoutubeStreamExtractor extends StreamExtractor { .orElse(EMPTY_STRING); } - @FunctionalInterface - interface StreamTypeStreamBuilderHelper { - T buildStream(String url, ItagItem itagItem); - } - - /** - * Abstract method for - * {@link #getAudioStreams()}, {@link #getVideoOnlyStreams()} and {@link #getVideoStreams()}. - * - * @param itags A map of Urls + ItagItems - * @param streamBuilder Builds the stream from the provided data - * @param exMsgStreamType Stream type inside the exception message e.g. "video streams" - * @param Type of the stream - * @return - * @throws ExtractionException - */ - private List getStreamsByType( - final Map itags, - final StreamTypeStreamBuilderHelper streamBuilder, - final String exMsgStreamType - ) throws ExtractionException { - final List streams = new ArrayList<>(); - - try { - for (final Map.Entry entry : itags.entrySet()) { - final String url = tryDecryptUrl(entry.getKey(), getId()); - - final T stream = streamBuilder.buildStream(url, entry.getValue()); - if (!Stream.containSimilarStream(stream, streams)) { - streams.add(stream); - } - } - } catch (final Exception e) { - throw new ParsingException("Could not get " + exMsgStreamType, e); - } - - return streams; - } - @Override public List getAudioStreams() throws ExtractionException { assertPageFetched(); - return getStreamsByType( - getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO), - AudioStream::new, - "audio streams" - ); + return getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO, + getAudioStreamBuilderHelper(), "audio"); } @Override public List getVideoStreams() throws ExtractionException { assertPageFetched(); - return getStreamsByType( - getItags(FORMATS, ItagItem.ItagType.VIDEO), - (url, itag) -> new VideoStream(url, false, itag), - "video streams" - ); + return getItags(FORMATS, ItagItem.ItagType.VIDEO, + getVideoStreamBuilderHelper(false), "video"); } @Override public List getVideoOnlyStreams() throws ExtractionException { assertPageFetched(); - return getStreamsByType( - getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY), - (url, itag) -> new VideoStream(url, true, itag), - "video only streams" - ); + return getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY, + getVideoStreamBuilderHelper(true), "video-only"); } /** @@ -672,18 +624,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull public List getSubtitles(final MediaFormat format) throws ParsingException { assertPageFetched(); - if (subtitles != null) { - // Already calculated - return subtitles; - } + // We cannot store the subtitles list because the media format may change + final List subtitlesToReturn = new ArrayList<>(); final JsonObject renderer = playerResponse.getObject("captions") .getObject("playerCaptionsTracklistRenderer"); final JsonArray captionsArray = renderer.getArray("captionTracks"); // TODO: use this to apply auto translation to different language from a source language // final JsonArray autoCaptionsArray = renderer.getArray("translationLanguages"); - subtitles = new ArrayList<>(); for (int i = 0; i < captionsArray.size(); i++) { final String languageCode = captionsArray.getObject(i).getString("languageCode"); final String baseUrl = captionsArray.getObject(i).getString("baseUrl"); @@ -692,15 +641,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (languageCode != null && baseUrl != null && vssId != null) { final boolean isAutoGenerated = vssId.startsWith("a."); final String cleanUrl = baseUrl - .replaceAll("&fmt=[^&]*", "") // Remove preexisting format if exists - .replaceAll("&tlang=[^&]*", ""); // Remove translation language + // Remove preexisting format if exists + .replaceAll("&fmt=[^&]*", "") + // Remove translation language + .replaceAll("&tlang=[^&]*", ""); - subtitles.add(new SubtitlesStream(format, languageCode, - cleanUrl + "&fmt=" + format.getSuffix(), isAutoGenerated)); + subtitlesToReturn.add(new SubtitlesStream.Builder() + .setContent(cleanUrl + "&fmt=" + format.getSuffix(), true) + .setMediaFormat(format) + .setLanguageCode(languageCode) + .setAutoGenerated(isAutoGenerated) + .build()); } } - return subtitles; + return subtitlesToReturn; } @Override @@ -788,6 +743,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { private static final String STREAMING_DATA = "streamingData"; private static final String PLAYER = "player"; private static final String NEXT = "next"; + private static final String SIGNATURE_CIPHER = "signatureCipher"; + private static final String CIPHER = "cipher"; private static final String[] REGEXES = { "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)" @@ -827,7 +784,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { final JsonObject playabilityStatus = playerResponse.getObject("playabilityStatus"); - final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING) + final boolean isAgeRestricted = playabilityStatus.getString("reason", EMPTY_STRING) .contains("age"); setStreamType(); @@ -837,12 +794,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { fetchTvHtml5EmbedJsonPlayer(contentCountry, localization, videoId); } catch (final Exception ignored) { } - - // Refresh the stream type because the stream type may be not properly known for - // age-restricted videos - setStreamType(); } + // Refresh the stream type because the stream type may be not properly known for + // age-restricted videos + setStreamType(); + if (html5StreamingData == null && playerResponse.has(STREAMING_DATA)) { html5StreamingData = playerResponse.getObject(STREAMING_DATA); } @@ -866,7 +823,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getBytes(StandardCharsets.UTF_8); nextResponse = getJsonPostResponse(NEXT, body, localization); - if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM) + if ((!isAgeRestricted && streamType == StreamType.VIDEO_STREAM) || isAndroidClientFetchForced) { try { fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId); @@ -874,7 +831,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } - if ((!ageRestricted && streamType == StreamType.LIVE_STREAM) + if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM) || isIosClientFetchForced) { try { fetchIosMobileJsonPlayer(contentCountry, localization, videoId); @@ -1183,44 +1140,133 @@ public class YoutubeStreamExtractor extends StreamExtractor { return videoSecondaryInfoRenderer; } - @Nonnull - private Map getItags(@Nonnull final String streamingDataKey, - @Nonnull final ItagItem.ItagType itagTypeWanted) { - final Map urlAndItags = new LinkedHashMap<>(); - if (html5StreamingData == null && androidStreamingData == null - && iosStreamingData == null) { - return urlAndItags; - } - - final List> streamingDataAndCpnLoopList = new ArrayList<>(); - // Use the androidStreamingData object first because there is no n param and no - // signatureCiphers in streaming URLs of the Android client - streamingDataAndCpnLoopList.add(new Pair<>(androidStreamingData, androidCpn)); - streamingDataAndCpnLoopList.add(new Pair<>(html5StreamingData, html5Cpn)); - // Use the iosStreamingData object in the last position because most of the available - // streams can be extracted with the Android and web clients and also because the iOS - // client is only enabled by default on livestreams - streamingDataAndCpnLoopList.add(new Pair<>(iosStreamingData, iosCpn)); - - for (final Pair pair : streamingDataAndCpnLoopList) { - urlAndItags.putAll(getStreamsFromStreamingDataKey(pair.getFirst(), streamingDataKey, - itagTypeWanted, pair.getSecond())); - } - - return urlAndItags; + @FunctionalInterface + private interface StreamBuilderHelper { + @Nonnull + T buildStream(ItagInfo itagInfo); } @Nonnull - private Map getStreamsFromStreamingDataKey( + private List getItags( + final String streamingDataKey, + final ItagItem.ItagType itagTypeWanted, + final StreamBuilderHelper streamBuilderHelper, + final String streamTypeExceptionMessage) throws ParsingException { + try { + final List itagInfos = new ArrayList<>(); + if (html5StreamingData == null && androidStreamingData == null + && iosStreamingData == null) { + return Collections.emptyList(); + } + + final List> streamingDataAndCpnLoopList = new ArrayList<>(); + // Use the androidStreamingData object first because there is no n param and no + // signatureCiphers in streaming URLs of the Android client + streamingDataAndCpnLoopList.add(new Pair<>(androidStreamingData, androidCpn)); + streamingDataAndCpnLoopList.add(new Pair<>(html5StreamingData, html5Cpn)); + // Use the iosStreamingData object in the last position because most of the available + // streams can be extracted with the Android and web clients and also because the iOS + // client is only enabled by default on livestreams + streamingDataAndCpnLoopList.add(new Pair<>(iosStreamingData, iosCpn)); + + for (final Pair pair : streamingDataAndCpnLoopList) { + itagInfos.addAll(getStreamsFromStreamingDataKey(pair.getFirst(), streamingDataKey, + itagTypeWanted, streamType, pair.getSecond())); + } + + final List streamList = new ArrayList<>(); + for (final ItagInfo itagInfo : itagInfos) { + final T stream = streamBuilderHelper.buildStream(itagInfo); + if (!Stream.containSimilarStream(stream, streamList)) { + streamList.add(stream); + } + } + + return streamList; + } catch (final Exception e) { + throw new ParsingException( + "Could not get " + streamTypeExceptionMessage + " streams", e); + } + } + + @Nonnull + private StreamBuilderHelper getAudioStreamBuilderHelper() { + return new StreamBuilderHelper() { + @Nonnull + @Override + public AudioStream buildStream(@Nonnull final ItagInfo itagInfo) { + final ItagItem itagItem = itagInfo.getItagItem(); + final AudioStream.Builder builder = new AudioStream.Builder() + .setId(String.valueOf(itagItem.id)) + .setContent(itagInfo.getContent(), itagInfo.getIsUrl()) + .setMediaFormat(itagItem.getMediaFormat()) + .setAverageBitrate(itagItem.getAverageBitrate()) + .setItagItem(itagItem); + + if (streamType != StreamType.VIDEO_STREAM || !itagInfo.getIsUrl()) { + // YouTube uses the DASH delivery method for videos on OTF streams and + // for all streams of post-live streams and live streams + builder.setDeliveryMethod(DeliveryMethod.DASH); + } + + return builder.build(); + } + }; + } + + @Nonnull + private StreamBuilderHelper getVideoStreamBuilderHelper( + final boolean areStreamsVideoOnly) { + return new StreamBuilderHelper() { + @Nonnull + @Override + public VideoStream buildStream(@Nonnull final ItagInfo itagInfo) { + final ItagItem itagItem = itagInfo.getItagItem(); + final VideoStream.Builder builder = new VideoStream.Builder() + .setId(String.valueOf(itagItem.id)) + .setContent(itagInfo.getContent(), itagInfo.getIsUrl()) + .setMediaFormat(itagItem.getMediaFormat()) + .setIsVideoOnly(areStreamsVideoOnly) + .setItagItem(itagItem); + + final int height = itagItem.getHeight(); + if (height > 0) { + final StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(height); + stringBuilder.append("p"); + final int fps = itagItem.getFps(); + if (fps > 30) { + stringBuilder.append(fps); + } + builder.setResolution(stringBuilder.toString()); + } else { + final String resolutionString = itagItem.getResolutionString(); + builder.setResolution(resolutionString != null ? resolutionString : ""); + } + + if (streamType != StreamType.VIDEO_STREAM || !itagInfo.getIsUrl()) { + // YouTube uses the DASH delivery method for videos on OTF streams and + // for all streams of post-live streams and live streams + builder.setDeliveryMethod(DeliveryMethod.DASH); + } + + return builder.build(); + } + }; + } + + @Nonnull + private List getStreamsFromStreamingDataKey( final JsonObject streamingData, - @Nonnull final String streamingDataKey, + final String streamingDataKey, @Nonnull final ItagItem.ItagType itagTypeWanted, + @Nonnull final StreamType contentStreamType, @Nonnull final String contentPlaybackNonce) { if (streamingData == null || !streamingData.has(streamingDataKey)) { - return Collections.emptyMap(); + return Collections.emptyList(); } - final Map urlAndItagsFromStreamingDataObject = new LinkedHashMap<>(); + final List itagInfos = new ArrayList<>(); final JsonArray formats = streamingData.getArray(streamingDataKey); for (int i = 0; i < formats.size(); i++) { final JsonObject formatData = formats.getObject(i); @@ -1232,53 +1278,85 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { final ItagItem itagItem = ItagItem.getItag(itag); + final ItagItem.ItagType itagType = itagItem.itagType; if (itagItem.itagType != itagTypeWanted) { continue; } - - // Ignore streams that are delivered using YouTube's OTF format, - // as those only work with DASH and not with progressive HTTP. - if ("FORMAT_STREAM_TYPE_OTF".equalsIgnoreCase(formatData.getString("type"))) { - continue; - } - - final String streamUrl; + String streamUrl; if (formatData.has("url")) { - streamUrl = formatData.getString("url"); + streamUrl = formatData.getString("url") + "&cpn=" + + contentPlaybackNonce; } else { // This url has an obfuscated signature - final String cipherString = formatData.has("cipher") - ? formatData.getString("cipher") - : formatData.getString("signatureCipher"); + final String cipherString = formatData.has(CIPHER) + ? formatData.getString(CIPHER) + : formatData.getString(SIGNATURE_CIPHER); final Map cipher = Parser.compatParseMap( cipherString); streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + deobfuscateSignature(cipher.get("s")); } + if (isWebStreamingUrl(streamUrl)) { + streamUrl = tryDecryptUrl(streamUrl, getId()) + "&cver=" + + getClientVersion(); + } + final JsonObject initRange = formatData.getObject("initRange"); final JsonObject indexRange = formatData.getObject("indexRange"); final String mimeType = formatData.getString("mimeType", EMPTY_STRING); final String codec = mimeType.contains("codecs") - ? mimeType.split("\"")[1] - : EMPTY_STRING; + ? mimeType.split("\"")[1] : EMPTY_STRING; itagItem.setBitrate(formatData.getInt("bitrate")); itagItem.setWidth(formatData.getInt("width")); itagItem.setHeight(formatData.getInt("height")); - itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1"))); - itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1"))); - itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1"))); - itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1"))); - itagItem.fps = formatData.getInt("fps"); + itagItem.setInitStart(Integer.parseInt(initRange.getString("start", + "-1"))); + itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", + "-1"))); + itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", + "-1"))); + itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", + "-1"))); itagItem.setQuality(formatData.getString("quality")); itagItem.setCodec(codec); + if (contentStreamType != StreamType.VIDEO_STREAM) { + itagItem.setTargetDurationSec(formatData.getInt( + "targetDurationSec")); + } + if (itagType == ItagItem.ItagType.VIDEO + || itagType == ItagItem.ItagType.VIDEO_ONLY) { + itagItem.setFps(formatData.getInt("fps")); + } + if (itagType == ItagItem.ItagType.AUDIO) { + itagItem.setSampleRate(Integer.parseInt(formatData.getString( + "audioSampleRate"))); + itagItem.setAudioChannels(formatData.getInt("audioChannels")); + } + itagItem.setContentLength(Long.parseLong(formatData.getString( + "contentLength", "-1"))); - urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem); - } catch (final UnsupportedEncodingException | ParsingException ignored) { + final ItagInfo itagInfo = new ItagInfo(streamUrl, itagItem); + + if (contentStreamType == StreamType.VIDEO_STREAM) { + itagInfo.setIsUrl(!formatData.getString("type", EMPTY_STRING) + .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")); + } else { + // We are currently not able to generate DASH manifests for running + // livestreams, so because of the requirements of StreamInfo + // objects, return these streams as DASH URL streams (even if they + // are not playable). + // Ended livestreams are returned as non URL streams + itagInfo.setIsUrl(contentStreamType != StreamType.POST_LIVE_STREAM); + } + + itagInfos.add(itagInfo); + } catch (final IOException | ExtractionException ignored) { } } - return urlAndItagsFromStreamingDataObject; + + return itagInfos; }