From fe30eb43a9b3c17226f404c8141685d8f4417861 Mon Sep 17 00:00:00 2001 From: litetex <40789489+litetex@users.noreply.github.com> Date: Sun, 1 May 2022 16:38:05 +0200 Subject: [PATCH] Cleanup ``YoutubeStreamExtractor`` and some related classes * Fixed obvious sonar(lint) warnings * Abstracted some code (get*Streams) * Used some new lines to make code better readable * Chopped down brace-jungle in some methods * Use StandardCharset (Java 8 4tw) --- .../extractors/YoutubeStreamExtractor.java | 730 +++++++++--------- .../newpipe/extractor/stream/Stream.java | 8 +- .../schabi/newpipe/extractor/utils/Utils.java | 9 +- 3 files changed, 394 insertions(+), 353 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 1785b755..bef96fde 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -16,7 +16,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; -import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -65,6 +64,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; @@ -77,6 +77,8 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; + import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -187,19 +189,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { return playerMicroFormatRenderer.getString("uploadDate"); } else if (!playerMicroFormatRenderer.getString("publishDate", EMPTY_STRING).isEmpty()) { return playerMicroFormatRenderer.getString("publishDate"); - } else { - final JsonObject liveDetails = playerMicroFormatRenderer.getObject( - "liveBroadcastDetails"); - if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) { - // an ended live stream - return liveDetails.getString("endTimestamp"); - } else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) { - // a running live stream - return liveDetails.getString("startTimestamp"); - } else if (getStreamType() == StreamType.LIVE_STREAM) { - // this should never be reached, but a live stream without upload date is valid - return null; - } + } + + final JsonObject liveDetails = playerMicroFormatRenderer.getObject( + "liveBroadcastDetails"); + if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) { + // an ended live stream + return liveDetails.getString("endTimestamp"); + } else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) { + // a running live stream + return liveDetails.getString("startTimestamp"); + } else if (getStreamType() == StreamType.LIVE_STREAM) { + // this should never be reached, but a live stream without upload date is valid + return null; } if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")) @@ -259,10 +261,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getThumbnailUrl() throws ParsingException { assertPageFetched(); try { - final JsonArray thumbnails = playerResponse.getObject("videoDetails") - .getObject("thumbnail").getArray("thumbnails"); + final JsonArray thumbnails = playerResponse + .getObject("videoDetails") + .getObject("thumbnail") + .getArray("thumbnails"); // the last thumbnail is the one with the highest resolution - final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url"); + final String url = thumbnails + .getObject(thumbnails.size() - 1) + .getString("url"); return fixThumbnailUrl(url); } catch (final Exception e) { @@ -277,8 +283,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { assertPageFetched(); // Description with more info on links try { - final String description = getTextFromObject(getVideoSecondaryInfoRenderer() - .getObject("description"), true); + final String description = getTextFromObject( + getVideoSecondaryInfoRenderer().getObject("description"), + true); if (!isNullOrEmpty(description)) { return new Description(description, Description.HTML); } @@ -299,27 +306,35 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public int getAgeLimit() throws ParsingException { - if (ageLimit == -1) { - ageLimit = NO_AGE_LIMIT; - - final JsonArray metadataRows = getVideoSecondaryInfoRenderer() - .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer") - .getArray("rows"); - for (final Object metadataRow : metadataRows) { - final JsonArray contents = ((JsonObject) metadataRow) - .getObject("metadataRowRenderer").getArray("contents"); - for (final Object content : contents) { - final JsonArray runs = ((JsonObject) content).getArray("runs"); - for (final Object run : runs) { - final String rowText = ((JsonObject) run).getString("text", EMPTY_STRING); - if (rowText.contains("Age-restricted")) { - ageLimit = 18; - return ageLimit; - } - } - } - } + if (ageLimit != -1) { + return ageLimit; } + + final boolean ageRestricted = getVideoSecondaryInfoRenderer() + .getObject("metadataRowContainer") + .getObject("metadataRowContainerRenderer") + .getArray("rows") + .stream() + // Only JsonObjects allowed + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .flatMap(metadataRow -> metadataRow + .getObject("metadataRowRenderer") + .getArray("contents") + .stream() + // Only JsonObjects allowed + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast)) + .flatMap(content -> content + .getArray("runs") + .stream() + // Only JsonObjects allowed + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast)) + .map(run -> run.getString("text", EMPTY_STRING)) + .anyMatch(rowText -> rowText.contains("Age-restricted")); + + ageLimit = ageRestricted ? 18 : NO_AGE_LIMIT; return ageLimit; } @@ -370,9 +385,8 @@ public class YoutubeStreamExtractor extends StreamExtractor { if (timestamp == -2) { // Regex for timestamp was not found return 0; - } else { - return timestamp; } + return timestamp; } @Override @@ -476,10 +490,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public boolean isUploaderVerified() throws ParsingException { - final JsonArray badges = getVideoSecondaryInfoRenderer().getObject("owner") - .getObject("videoOwnerRenderer").getArray("badges"); - - return YoutubeParsingHelper.isVerified(badges); + return YoutubeParsingHelper.isVerified( + getVideoSecondaryInfoRenderer() + .getObject("owner") + .getObject("videoOwnerRenderer") + .getArray("badges")); } @Nonnull @@ -490,9 +505,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { String url = null; try { - url = getVideoSecondaryInfoRenderer().getObject("owner") - .getObject("videoOwnerRenderer").getObject("thumbnail") - .getArray("thumbnails").getObject(0).getString("url"); + url = getVideoSecondaryInfoRenderer() + .getObject("owner") + .getObject("videoOwnerRenderer") + .getObject("thumbnail") + .getArray("thumbnails") + .getObject(0) + .getString("url"); } catch (final ParsingException ignored) { // Age-restricted videos cause a ParsingException here } @@ -530,8 +549,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { // There is no DASH manifest available in the iOS clients and the DASH manifest of the // Android client doesn't contain all available streams (mainly the WEBM ones) - return getManifestUrl("dash", Arrays.asList(html5StreamingData, - androidStreamingData)); + return getManifestUrl( + "dash", + Arrays.asList(html5StreamingData, androidStreamingData)); } @Nonnull @@ -542,93 +562,91 @@ public class YoutubeStreamExtractor extends StreamExtractor { // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest // returned has separated audio and video streams // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response - return getManifestUrl("hls", Arrays.asList(iosStreamingData, html5StreamingData, - androidStreamingData)); + return getManifestUrl( + "hls", + Arrays.asList(iosStreamingData, html5StreamingData, androidStreamingData)); } @Nonnull private static String getManifestUrl(@Nonnull final String manifestType, @Nonnull final List streamingDataObjects) { final String manifestKey = manifestType + "ManifestUrl"; - for (final JsonObject streamingDataObject : streamingDataObjects) { - if (streamingDataObject != null) { - final String manifestKeyValue = streamingDataObject.getString(manifestKey); - if (manifestKeyValue != null) { - return manifestKeyValue; + + return streamingDataObjects.stream() + .filter(Objects::nonNull) + .map(streamingDataObject -> streamingDataObject.getString(manifestKey)) + .filter(Objects::nonNull) + .findFirst() + .orElse(EMPTY_STRING); + } + + @FunctionalInterface + interface StreamTypeStreamBuilderHelper { + T buildStream(String url, ItagItem itagItem); + } + + /** + * Abstract method for + * {@link #getAudioStreams()}, {@link #getVideoOnlyStreams()} and {@link #getVideoStreams()}. + * + * @param itags A map of Urls + ItagItems + * @param streamBuilder Builds the stream from the provided data + * @param exMsgStreamType Stream type inside the exception message e.g. "video streams" + * @param Type of the stream + * @return + * @throws ExtractionException + */ + private List getStreamsByType( + final Map itags, + final StreamTypeStreamBuilderHelper streamBuilder, + final String exMsgStreamType + ) throws ExtractionException { + final List streams = new ArrayList<>(); + + try { + for (final Map.Entry entry : itags.entrySet()) { + final String url = tryDecryptUrl(entry.getKey(), getId()); + + final T stream = streamBuilder.buildStream(url, entry.getValue()); + if (!Stream.containSimilarStream(stream, streams)) { + streams.add(stream); } } + } catch (final Exception e) { + throw new ParsingException("Could not get " + exMsgStreamType, e); } - return EMPTY_STRING; + return streams; } @Override public List getAudioStreams() throws ExtractionException { assertPageFetched(); - final List audioStreams = new ArrayList<>(); - - try { - for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, - ItagItem.ItagType.AUDIO).entrySet()) { - final ItagItem itag = entry.getValue(); - final String url = tryDecryption(entry.getKey(), getId()); - - final AudioStream audioStream = new AudioStream(url, itag); - if (!Stream.containSimilarStream(audioStream, audioStreams)) { - audioStreams.add(audioStream); - } - } - } catch (final Exception e) { - throw new ParsingException("Could not get audio streams", e); - } - - return audioStreams; + return getStreamsByType( + getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO), + AudioStream::new, + "audio streams" + ); } @Override public List getVideoStreams() throws ExtractionException { assertPageFetched(); - final List videoStreams = new ArrayList<>(); - - try { - for (final Map.Entry entry : getItags(FORMATS, - ItagItem.ItagType.VIDEO).entrySet()) { - final ItagItem itag = entry.getValue(); - final String url = tryDecryption(entry.getKey(), getId()); - - final VideoStream videoStream = new VideoStream(url, false, itag); - if (!Stream.containSimilarStream(videoStream, videoStreams)) { - videoStreams.add(videoStream); - } - } - } catch (final Exception e) { - throw new ParsingException("Could not get video streams", e); - } - - return videoStreams; + return getStreamsByType( + getItags(FORMATS, ItagItem.ItagType.VIDEO), + (url, itag) -> new VideoStream(url, false, itag), + "video streams" + ); } @Override public List getVideoOnlyStreams() throws ExtractionException { assertPageFetched(); - final List videoOnlyStreams = new ArrayList<>(); - - try { - for (final Map.Entry entry : getItags(ADAPTIVE_FORMATS, - ItagItem.ItagType.VIDEO_ONLY).entrySet()) { - final ItagItem itag = entry.getValue(); - final String url = tryDecryption(entry.getKey(), getId()); - - final VideoStream videoStream = new VideoStream(url, true, itag); - if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) { - videoOnlyStreams.add(videoStream); - } - } - } catch (final Exception e) { - throw new ParsingException("Could not get video only streams", e); - } - - return videoOnlyStreams; + return getStreamsByType( + getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY), + (url, itag) -> new VideoStream(url, true, itag), + "video only streams" + ); } /** @@ -636,7 +654,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { * always needed. * This way a breaking change from YouTube does not result in a broken extractor. */ - private String tryDecryption(final String url, final String videoId) { + private String tryDecryptUrl(final String url, final String videoId) { try { return YoutubeThrottlingDecrypter.apply(url, videoId); } catch (final ParsingException e) { @@ -713,25 +731,33 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId()); - final JsonArray results = nextResponse.getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("secondaryResults") - .getObject("secondaryResults").getArray("results"); + final JsonArray results = nextResponse + .getObject("contents") + .getObject("twoColumnWatchNextResults") + .getObject("secondaryResults") + .getObject("secondaryResults") + .getArray("results"); final TimeAgoParser timeAgoParser = getTimeAgoParser(); + results.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(result -> { + if (result.has("compactVideoRenderer")) { + return new YoutubeStreamInfoItemExtractor( + result.getObject("compactVideoRenderer"), timeAgoParser); + } else if (result.has("compactRadioRenderer")) { + return new YoutubeMixOrPlaylistInfoItemExtractor( + result.getObject("compactRadioRenderer")); + } else if (result.has("compactPlaylistRenderer")) { + return new YoutubeMixOrPlaylistInfoItemExtractor( + result.getObject("compactPlaylistRenderer")); + } + return null; + }) + .filter(Objects::nonNull) + .forEach(collector::commit); - for (final Object resultObject : results) { - final JsonObject result = (JsonObject) resultObject; - if (result.has("compactVideoRenderer")) { - collector.commit(new YoutubeStreamInfoItemExtractor( - result.getObject("compactVideoRenderer"), timeAgoParser)); - } else if (result.has("compactRadioRenderer")) { - collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor( - result.getObject("compactRadioRenderer"))); - } else if (result.has("compactPlaylistRenderer")) { - collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor( - result.getObject("compactPlaylistRenderer"))); - } - } return collector; } catch (final Exception e) { throw new ParsingException("Could not get related videos", e); @@ -777,9 +803,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - if (sts == null) { - getStsFromPlayerJs(); - } + initStsFromPlayerJsIfNeeded(); final String videoId = getId(); final Localization localization = getExtractorLocalization(); @@ -833,13 +857,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { playerMicroFormatRenderer = youtubePlayerResponse.getObject("microformat") .getObject("playerMicroformatRenderer"); - final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization, - contentCountry) - .value(VIDEO_ID, videoId) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(UTF_8); + final byte[] body = JsonWriter.string( + prepareDesktopJsonBuilder(localization, contentCountry) + .value(VIDEO_ID, videoId) + .value(CONTENT_CHECK_OK, true) + .value(RACY_CHECK_OK, true) + .done()) + .getBytes(StandardCharsets.UTF_8); nextResponse = getJsonPostResponse(NEXT, body, localization); if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM) @@ -863,53 +887,56 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull final JsonObject playabilityStatus) throws ParsingException { String status = playabilityStatus.getString("status"); + if (status == null || status.equalsIgnoreCase("ok")) { + return; + } + // If status exist, and is not "OK", throw the specific exception based on error message // or a ContentNotAvailableException with the reason text if it's an unknown reason. - if (status != null && !status.equalsIgnoreCase("ok")) { - final JsonObject newPlayabilityStatus - = youtubePlayerResponse.getObject("playabilityStatus"); - status = newPlayabilityStatus.getString("status"); - final String reason = newPlayabilityStatus.getString("reason"); + final JsonObject newPlayabilityStatus = + youtubePlayerResponse.getObject("playabilityStatus"); + status = newPlayabilityStatus.getString("status"); + final String reason = newPlayabilityStatus.getString("reason"); - if (status.equalsIgnoreCase("login_required")) { - if (reason == null) { - final String message = newPlayabilityStatus.getArray("messages").getString(0); - if (message != null && message.contains("private")) { - throw new PrivateContentException("This video is private."); - } - } else if (reason.contains("age")) { - // No streams can be fetched, therefore throw an AgeRestrictedContentException - // explicitly. - throw new AgeRestrictedContentException( - "This age-restricted video cannot be watched."); + if (status.equalsIgnoreCase("login_required")) { + if (reason == null) { + final String message = newPlayabilityStatus.getArray("messages").getString(0); + if (message != null && message.contains("private")) { + throw new PrivateContentException("This video is private."); } + } else if (reason.contains("age")) { + // No streams can be fetched, therefore throw an AgeRestrictedContentException + // explicitly. + throw new AgeRestrictedContentException( + "This age-restricted video cannot be watched."); } - - if (status.equalsIgnoreCase("unplayable") && reason != null) { - if (reason.contains("Music Premium")) { - throw new YoutubeMusicPremiumContentException(); - } - if (reason.contains("payment")) { - throw new PaidContentException("This video is a paid video"); - } - if (reason.contains("members-only")) { - throw new PaidContentException("This video is only available" - + " for members of the channel of this video"); - } - - if (reason.contains("unavailable")) { - final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus - .getObject("errorScreen").getObject("playerErrorMessageRenderer") - .getObject("subreason")); - if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) { - throw new GeographicRestrictionException( - "This video is not available in client's country."); - } - } - } - - throw new ContentNotAvailableException("Got error: \"" + reason + "\""); } + + if (status.equalsIgnoreCase("unplayable") && reason != null) { + if (reason.contains("Music Premium")) { + throw new YoutubeMusicPremiumContentException(); + } + if (reason.contains("payment")) { + throw new PaidContentException("This video is a paid video"); + } + if (reason.contains("members-only")) { + throw new PaidContentException("This video is only available" + + " for members of the channel of this video"); + } + + if (reason.contains("unavailable")) { + final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus + .getObject("errorScreen") + .getObject("playerErrorMessageRenderer") + .getObject("subreason")); + if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) { + throw new GeographicRestrictionException( + "This video is not available in client's country."); + } + } + } + + throw new ContentNotAvailableException("Got error: \"" + reason + "\""); } /** @@ -921,14 +948,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull final String videoId) throws IOException, ExtractionException { androidCpn = generateContentPlaybackNonce(); - final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder( - localization, contentCountry) - .value(VIDEO_ID, videoId) - .value(CPN, androidCpn) - .value(CONTENT_CHECK_OK, true) - .value(RACY_CHECK_OK, true) - .done()) - .getBytes(UTF_8); + final byte[] mobileBody = JsonWriter.string( + prepareAndroidMobileJsonBuilder(localization, contentCountry) + .value(VIDEO_ID, videoId) + .value(CPN, androidCpn) + .value(CONTENT_CHECK_OK, true) + .value(RACY_CHECK_OK, true) + .done()) + .getBytes(StandardCharsets.UTF_8); final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER, mobileBody, localization, "&t=" + generateTParameter() @@ -952,14 +979,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull final String videoId) throws IOException, ExtractionException { iosCpn = generateContentPlaybackNonce(); - final byte[] mobileBody = JsonWriter.string(prepareIosMobileJsonBuilder( - localization, contentCountry) + final byte[] mobileBody = JsonWriter.string( + prepareIosMobileJsonBuilder(localization, contentCountry) .value(VIDEO_ID, videoId) .value(CPN, iosCpn) .value(CONTENT_CHECK_OK, true) .value(RACY_CHECK_OK, true) .done()) - .getBytes(UTF_8); + .getBytes(StandardCharsets.UTF_8); final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER, mobileBody, localization, "&t=" + generateTParameter() @@ -987,9 +1014,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull final Localization localization, @Nonnull final String videoId) throws IOException, ExtractionException { - if (sts == null) { - getStsFromPlayerJs(); - } + initStsFromPlayerJsIfNeeded(); // Because a cpn is unique to each request, we need to generate it again html5Cpn = generateContentPlaybackNonce(); @@ -1072,7 +1097,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { return cachedDeobfuscationCode; } - private static void getStsFromPlayerJs() throws ParsingException { + private static void initStsFromPlayerJsIfNeeded() throws ParsingException { if (!isNullOrEmpty(sts)) { return; } @@ -1134,31 +1159,28 @@ public class YoutubeStreamExtractor extends StreamExtractor { return theVideoPrimaryInfoRenderer; } + @Nonnull private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException { if (videoSecondaryInfoRenderer != null) { return videoSecondaryInfoRenderer; } - final JsonArray contents = nextResponse.getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("results").getObject("results") - .getArray("contents"); + videoSecondaryInfoRenderer = nextResponse + .getObject("contents") + .getObject("twoColumnWatchNextResults") + .getObject("results") + .getObject("results") + .getArray("contents") + .stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .filter(content -> content.has("videoSecondaryInfoRenderer")) + .map(content -> content.getObject("videoSecondaryInfoRenderer")) + .findFirst() + .orElseThrow( + () -> new ParsingException("Could not find videoSecondaryInfoRenderer")); - JsonObject theVideoSecondaryInfoRenderer = null; - - for (final Object content : contents) { - if (((JsonObject) content).has("videoSecondaryInfoRenderer")) { - theVideoSecondaryInfoRenderer = ((JsonObject) content) - .getObject("videoSecondaryInfoRenderer"); - break; - } - } - - if (isNullOrEmpty(theVideoSecondaryInfoRenderer)) { - throw new ParsingException("Could not find videoSecondaryInfoRenderer"); - } - - videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer; - return theVideoSecondaryInfoRenderer; + return videoSecondaryInfoRenderer; } @Nonnull @@ -1194,83 +1216,82 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull final String streamingDataKey, @Nonnull final ItagItem.ItagType itagTypeWanted, @Nonnull final String contentPlaybackNonce) { - - final Map urlAndItagsFromStreamingDataObject = new LinkedHashMap<>(); - if (streamingData != null && streamingData.has(streamingDataKey)) { - final JsonArray formats = streamingData.getArray(streamingDataKey); - for (int i = 0; i != formats.size(); ++i) { - final JsonObject formatData = formats.getObject(i); - final int itag = formatData.getInt("itag"); - - if (ItagItem.isSupported(itag)) { - try { - final ItagItem itagItem = ItagItem.getItag(itag); - if (itagItem.itagType == itagTypeWanted) { - // Ignore streams that are delivered using YouTube's OTF format, - // as those only work with DASH and not with progressive HTTP. - if (formatData.getString("type", EMPTY_STRING) - .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { - continue; - } - - final String streamUrl; - if (formatData.has("url")) { - streamUrl = formatData.getString("url") + "&cpn=" - + contentPlaybackNonce; - } else { - // This url has an obfuscated signature - final String cipherString = formatData.has("cipher") - ? formatData.getString("cipher") - : formatData.getString("signatureCipher"); - final Map cipher = Parser.compatParseMap( - cipherString); - streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" - + deobfuscateSignature(cipher.get("s")); - } - - final JsonObject initRange = formatData.getObject("initRange"); - final JsonObject indexRange = formatData.getObject("indexRange"); - final String mimeType = formatData.getString("mimeType", EMPTY_STRING); - final String codec = mimeType.contains("codecs") - ? mimeType.split("\"")[1] : EMPTY_STRING; - - itagItem.setBitrate(formatData.getInt("bitrate")); - itagItem.setWidth(formatData.getInt("width")); - itagItem.setHeight(formatData.getInt("height")); - itagItem.setInitStart(Integer.parseInt(initRange.getString("start", - "-1"))); - itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", - "-1"))); - itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", - "-1"))); - itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", - "-1"))); - itagItem.fps = formatData.getInt("fps"); - itagItem.setQuality(formatData.getString("quality")); - itagItem.setCodec(codec); - - urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem); - } - } catch (final UnsupportedEncodingException | ParsingException ignored) { - } - } - } + if (streamingData == null || !streamingData.has(streamingDataKey)) { + return Collections.emptyMap(); } + final Map urlAndItagsFromStreamingDataObject = new LinkedHashMap<>(); + final JsonArray formats = streamingData.getArray(streamingDataKey); + for (int i = 0; i < formats.size(); i++) { + final JsonObject formatData = formats.getObject(i); + final int itag = formatData.getInt("itag"); + + if (!ItagItem.isSupported(itag)) { + continue; + } + + try { + final ItagItem itagItem = ItagItem.getItag(itag); + if (itagItem.itagType != itagTypeWanted) { + continue; + } + + // Ignore streams that are delivered using YouTube's OTF format, + // as those only work with DASH and not with progressive HTTP. + if ("FORMAT_STREAM_TYPE_OTF".equalsIgnoreCase(formatData.getString("type"))) { + continue; + } + + final String streamUrl; + if (formatData.has("url")) { + streamUrl = formatData.getString("url"); + } else { + // This url has an obfuscated signature + final String cipherString = formatData.has("cipher") + ? formatData.getString("cipher") + : formatData.getString("signatureCipher"); + final Map cipher = Parser.compatParseMap( + cipherString); + streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + + deobfuscateSignature(cipher.get("s")); + } + + final JsonObject initRange = formatData.getObject("initRange"); + final JsonObject indexRange = formatData.getObject("indexRange"); + final String mimeType = formatData.getString("mimeType", EMPTY_STRING); + final String codec = mimeType.contains("codecs") + ? mimeType.split("\"")[1] + : EMPTY_STRING; + + itagItem.setBitrate(formatData.getInt("bitrate")); + itagItem.setWidth(formatData.getInt("width")); + itagItem.setHeight(formatData.getInt("height")); + itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1"))); + itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1"))); + itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1"))); + itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1"))); + itagItem.fps = formatData.getInt("fps"); + itagItem.setQuality(formatData.getString("quality")); + itagItem.setCodec(codec); + + urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem); + } catch (final UnsupportedEncodingException | ParsingException ignored) { + } + } return urlAndItagsFromStreamingDataObject; } + @Nonnull @Override public List getFrames() throws ExtractionException { try { final JsonObject storyboards = playerResponse.getObject("storyboards"); - final JsonObject storyboardsRenderer; - if (storyboards.has("playerLiveStoryboardSpecRenderer")) { - storyboardsRenderer = storyboards.getObject("playerLiveStoryboardSpecRenderer"); - } else { - storyboardsRenderer = storyboards.getObject("playerStoryboardSpecRenderer"); - } + final JsonObject storyboardsRenderer = storyboards.getObject( + storyboards.has("playerLiveStoryboardSpecRenderer") + ? "playerLiveStoryboardSpecRenderer" + : "playerStoryboardSpecRenderer" + ); if (storyboardsRenderer == null) { return Collections.emptyList(); @@ -1283,15 +1304,13 @@ public class YoutubeStreamExtractor extends StreamExtractor { final String[] spec = storyboardsRendererSpec.split("\\|"); final String url = spec[0]; - final ArrayList result = new ArrayList<>(spec.length - 1); + final List result = new ArrayList<>(spec.length - 1); for (int i = 1; i < spec.length; ++i) { final String[] parts = spec[i].split("#"); if (parts.length != 8 || Integer.parseInt(parts[5]) == 0) { continue; } - final int frameWidth = Integer.parseInt(parts[0]); - final int frameHeight = Integer.parseInt(parts[1]); final int totalCount = Integer.parseInt(parts[2]); final int framesPerPageX = Integer.parseInt(parts[3]); final int framesPerPageY = Integer.parseInt(parts[4]); @@ -1310,15 +1329,14 @@ public class YoutubeStreamExtractor extends StreamExtractor { } result.add(new Frameset( urls, - frameWidth, - frameHeight, + /*frameWidth=*/Integer.parseInt(parts[0]), + /*frameHeight=*/Integer.parseInt(parts[1]), totalCount, - Integer.parseInt(parts[5]), + /*durationPerFrame=*/Integer.parseInt(parts[5]), framesPerPageX, framesPerPageY )); } - result.trimToSize(); return result; } catch (final Exception e) { throw new ExtractionException("Could not get frames", e); @@ -1328,8 +1346,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public Privacy getPrivacy() { - final boolean isUnlisted = playerMicroFormatRenderer.getBoolean("isUnlisted"); - return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC; + return playerMicroFormatRenderer.getBoolean("isUnlisted") + ? Privacy.UNLISTED + : Privacy.PUBLIC; } @Nonnull @@ -1342,14 +1361,18 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getLicence() throws ParsingException { final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer() - .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer") + .getObject("metadataRowContainer") + .getObject("metadataRowContainerRenderer") .getArray("rows") - .getObject(0).getObject("metadataRowRenderer"); + .getObject(0) + .getObject("metadataRowRenderer"); final JsonArray contents = metadataRowRenderer.getArray("contents"); final String license = getTextFromObject(contents.getObject(0)); - return license != null && "Licence".equals(getTextFromObject(metadataRowRenderer - .getObject("title"))) ? license : "YouTube licence"; + return license != null + && "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title"))) + ? license + : "YouTube licence"; } @Override @@ -1367,63 +1390,73 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public List getStreamSegments() throws ParsingException { - final ArrayList segments = new ArrayList<>(); - if (nextResponse.has("engagementPanels")) { - final JsonArray panels = nextResponse.getArray("engagementPanels"); - JsonArray segmentsArray = null; - // Search for correct panel containing the data - for (int i = 0; i < panels.size(); i++) { - final String panelIdentifier = panels.getObject(i) + if (!nextResponse.has("engagementPanels")) { + return Collections.emptyList(); + } + + final JsonArray segmentsArray = nextResponse.getArray("engagementPanels") + .stream() + // Check if object is a JsonObject + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + // Check if the panel is the correct one + .filter(panel -> "engagement-panel-macro-markers-description-chapters".equals( + panel + .getObject("engagementPanelSectionListRenderer") + .getString("panelIdentifier"))) + // Extract the data + .map(panel -> panel .getObject("engagementPanelSectionListRenderer") - .getString("panelIdentifier"); - // panelIdentifier might be null if the panel has something to do with ads - // See https://github.com/TeamNewPipe/NewPipe/issues/7792#issuecomment-1030900188 - if ("engagement-panel-macro-markers-description-chapters".equals(panelIdentifier)) { - segmentsArray = panels.getObject(i) - .getObject("engagementPanelSectionListRenderer").getObject("content") - .getObject("macroMarkersListRenderer").getArray("contents"); - break; - } + .getObject("content") + .getObject("macroMarkersListRenderer") + .getArray("contents")) + .findFirst() + .orElse(null); + + // If no data was found exit + if (segmentsArray == null) { + return Collections.emptyList(); + } + + final long duration = getLength(); + final List segments = new ArrayList<>(); + for (final JsonObject segmentJson : segmentsArray.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast) + .map(object -> object.getObject("macroMarkersListItemRenderer")) + .collect(Collectors.toList()) + ) { + final int startTimeSeconds = segmentJson.getObject("onTap") + .getObject("watchEndpoint").getInt("startTimeSeconds", -1); + + if (startTimeSeconds == -1) { + throw new ParsingException("Could not get stream segment start time."); + } + if (startTimeSeconds > duration) { + break; } - if (segmentsArray != null) { - final long duration = getLength(); - for (final Object object : segmentsArray) { - final JsonObject segmentJson = ((JsonObject) object) - .getObject("macroMarkersListItemRenderer"); + final String title = getTextFromObject(segmentJson.getObject("title")); + if (isNullOrEmpty(title)) { + throw new ParsingException("Could not get stream segment title."); + } - final int startTimeSeconds = segmentJson.getObject("onTap") - .getObject("watchEndpoint").getInt("startTimeSeconds", -1); - - if (startTimeSeconds == -1) { - throw new ParsingException("Could not get stream segment start time."); - } - if (startTimeSeconds > duration) { - break; - } - - final String title = getTextFromObject(segmentJson.getObject("title")); - if (isNullOrEmpty(title)) { - throw new ParsingException("Could not get stream segment title."); - } - - final StreamSegment segment = new StreamSegment(title, startTimeSeconds); - segment.setUrl(getUrl() + "?t=" + startTimeSeconds); - if (segmentJson.has("thumbnail")) { - final JsonArray previewsArray = segmentJson.getObject("thumbnail") - .getArray("thumbnails"); - if (!previewsArray.isEmpty()) { - // Assume that the thumbnail with the highest resolution is at the - // last position - final String url = previewsArray - .getObject(previewsArray.size() - 1).getString("url"); - segment.setPreviewUrl(fixThumbnailUrl(url)); - } - } - segments.add(segment); + final StreamSegment segment = new StreamSegment(title, startTimeSeconds); + segment.setUrl(getUrl() + "?t=" + startTimeSeconds); + if (segmentJson.has("thumbnail")) { + final JsonArray previewsArray = segmentJson + .getObject("thumbnail") + .getArray("thumbnails"); + if (!previewsArray.isEmpty()) { + // Assume that the thumbnail with the highest resolution is at the last position + final String url = previewsArray + .getObject(previewsArray.size() - 1) + .getString("url"); + segment.setPreviewUrl(fixThumbnailUrl(url)); } } + segments.add(segment); } return segments; } @@ -1431,9 +1464,12 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull @Override public List getMetaInfo() throws ParsingException { - return YoutubeParsingHelper.getMetaInfo( - nextResponse.getObject("contents").getObject("twoColumnWatchNextResults") - .getObject("results").getObject("results").getArray("contents")); + return YoutubeParsingHelper.getMetaInfo(nextResponse + .getObject("contents") + .getObject("twoColumnWatchNextResults") + .getObject("results") + .getObject("results") + .getArray("contents")); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java index d87a2214..5b827c15 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java @@ -1,19 +1,19 @@ package org.schabi.newpipe.extractor.stream; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import org.schabi.newpipe.extractor.MediaFormat; import java.io.Serializable; import java.util.List; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - /** * Creates a stream object from url, format and optional torrent url */ public abstract class Stream implements Serializable { private final MediaFormat mediaFormat; - public final String url; - public final String torrentUrl; + private final String url; + private final String torrentUrl; /** * @deprecated Use {@link #getFormat()} or {@link #getFormatId()} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java index 96e53158..014dffd4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java @@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.utils; import org.schabi.newpipe.extractor.exceptions.ParsingException; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URL; @@ -18,10 +16,17 @@ import java.util.Map; import java.util.Objects; import java.util.regex.Pattern; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + public final class Utils { public static final String HTTP = "http://"; public static final String HTTPS = "https://"; + /** + * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} + */ + @Deprecated public static final String UTF_8 = "UTF-8"; public static final String EMPTY_STRING = ""; private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");