Merge pull request #782 from litetex/cleanup-yt-stream-extractor

Cleanup of ``YoutubeStreamExtractor`` and some related classes

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -16,7 +16,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareIosMobileJsonBuilder;
 import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
-import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 import com.grack.nanojson.JsonArray;
@@ -66,6 +65,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeFormatter;
@@ -77,6 +77,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
+
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -187,19 +189,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             return playerMicroFormatRenderer.getString("uploadDate");
         } else if (!playerMicroFormatRenderer.getString("publishDate", EMPTY_STRING).isEmpty()) {
             return playerMicroFormatRenderer.getString("publishDate");
-        } else {
+        }
-            final JsonObject liveDetails = playerMicroFormatRenderer.getObject(
+
-                    "liveBroadcastDetails");
+        final JsonObject liveDetails = playerMicroFormatRenderer.getObject(
-            if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) {
+                "liveBroadcastDetails");
-                // an ended live stream
+        if (!liveDetails.getString("endTimestamp", EMPTY_STRING).isEmpty()) {
-                return liveDetails.getString("endTimestamp");
+            // an ended live stream
-            } else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) {
+            return liveDetails.getString("endTimestamp");
-                // a running live stream
+        } else if (!liveDetails.getString("startTimestamp", EMPTY_STRING).isEmpty()) {
-                return liveDetails.getString("startTimestamp");
+            // a running live stream
-            } else if (getStreamType() == StreamType.LIVE_STREAM) {
+            return liveDetails.getString("startTimestamp");
-                // this should never be reached, but a live stream without upload date is valid
+        } else if (getStreamType() == StreamType.LIVE_STREAM) {
-                return null;
+            // this should never be reached, but a live stream without upload date is valid
-            }
+            return null;
         }
 
         if (getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))
@@ -259,10 +261,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getThumbnailUrl() throws ParsingException {
         assertPageFetched();
         try {
-            final JsonArray thumbnails = playerResponse.getObject("videoDetails")
+            final JsonArray thumbnails = playerResponse
-                    .getObject("thumbnail").getArray("thumbnails");
+                    .getObject("videoDetails")
+                    .getObject("thumbnail")
+                    .getArray("thumbnails");
             // the last thumbnail is the one with the highest resolution
-            final String url = thumbnails.getObject(thumbnails.size() - 1).getString("url");
+            final String url = thumbnails
+                    .getObject(thumbnails.size() - 1)
+                    .getString("url");
 
             return fixThumbnailUrl(url);
         } catch (final Exception e) {
@@ -277,8 +283,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         assertPageFetched();
         // Description with more info on links
         try {
-            final String description = getTextFromObject(getVideoSecondaryInfoRenderer()
+            final String description = getTextFromObject(
-                    .getObject("description"), true);
+                    getVideoSecondaryInfoRenderer().getObject("description"),
+                    true);
             if (!isNullOrEmpty(description)) {
                 return new Description(description, Description.HTML);
             }
@@ -299,27 +306,35 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Override
     public int getAgeLimit() throws ParsingException {
-        if (ageLimit == -1) {
+        if (ageLimit != -1) {
-            ageLimit = NO_AGE_LIMIT;
+            return ageLimit;
-
-            final JsonArray metadataRows = getVideoSecondaryInfoRenderer()
-                    .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
-                    .getArray("rows");
-            for (final Object metadataRow : metadataRows) {
-                final JsonArray contents = ((JsonObject) metadataRow)
-                        .getObject("metadataRowRenderer").getArray("contents");
-                for (final Object content : contents) {
-                    final JsonArray runs = ((JsonObject) content).getArray("runs");
-                    for (final Object run : runs) {
-                        final String rowText = ((JsonObject) run).getString("text", EMPTY_STRING);
-                        if (rowText.contains("Age-restricted")) {
-                            ageLimit = 18;
-                            return ageLimit;
-                        }
-                    }
-                }
-            }
         }
+
+        final boolean ageRestricted = getVideoSecondaryInfoRenderer()
+                .getObject("metadataRowContainer")
+                .getObject("metadataRowContainerRenderer")
+                .getArray("rows")
+                .stream()
+                // Only JsonObjects allowed
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .flatMap(metadataRow -> metadataRow
+                        .getObject("metadataRowRenderer")
+                        .getArray("contents")
+                        .stream()
+                        // Only JsonObjects allowed
+                        .filter(JsonObject.class::isInstance)
+                        .map(JsonObject.class::cast))
+                .flatMap(content -> content
+                        .getArray("runs")
+                        .stream()
+                        // Only JsonObjects allowed
+                        .filter(JsonObject.class::isInstance)
+                        .map(JsonObject.class::cast))
+                .map(run -> run.getString("text", EMPTY_STRING))
+                .anyMatch(rowText -> rowText.contains("Age-restricted"));
+
+        ageLimit = ageRestricted ? 18 : NO_AGE_LIMIT;
         return ageLimit;
     }
 
@@ -370,9 +385,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         if (timestamp == -2) {
             // Regex for timestamp was not found
             return 0;
-        } else {
-            return timestamp;
         }
+        return timestamp;
     }
 
     @Override
@@ -476,10 +490,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
     @Override
     public boolean isUploaderVerified() throws ParsingException {
-        final JsonArray badges = getVideoSecondaryInfoRenderer().getObject("owner")
+        return YoutubeParsingHelper.isVerified(
-                .getObject("videoOwnerRenderer").getArray("badges");
+                getVideoSecondaryInfoRenderer()
-
+                        .getObject("owner")
-        return YoutubeParsingHelper.isVerified(badges);
+                        .getObject("videoOwnerRenderer")
+                        .getArray("badges"));
     }
 
     @Nonnull
@@ -490,9 +505,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         String url = null;
 
         try {
-            url = getVideoSecondaryInfoRenderer().getObject("owner")
+            url = getVideoSecondaryInfoRenderer()
-                    .getObject("videoOwnerRenderer").getObject("thumbnail")
+                    .getObject("owner")
-                    .getArray("thumbnails").getObject(0).getString("url");
+                    .getObject("videoOwnerRenderer")
+                    .getObject("thumbnail")
+                    .getArray("thumbnails")
+                    .getObject(0)
+                    .getString("url");
         } catch (final ParsingException ignored) {
             // Age-restricted videos cause a ParsingException here
         }
@@ -530,8 +549,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
         // There is no DASH manifest available in the iOS clients and the DASH manifest of the
         // Android client doesn't contain all available streams (mainly the WEBM ones)
-        return getManifestUrl("dash", Arrays.asList(html5StreamingData,
+        return getManifestUrl(
-                androidStreamingData));
+                "dash",
+                Arrays.asList(html5StreamingData, androidStreamingData));
     }
 
     @Nonnull
@@ -542,93 +562,91 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest
         // returned has separated audio and video streams
         // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response
-        return getManifestUrl("hls", Arrays.asList(iosStreamingData, html5StreamingData,
+        return getManifestUrl(
-                androidStreamingData));
+                "hls",
+                Arrays.asList(iosStreamingData, html5StreamingData, androidStreamingData));
     }
 
     @Nonnull
     private static String getManifestUrl(@Nonnull final String manifestType,
                                          @Nonnull final List<JsonObject> streamingDataObjects) {
         final String manifestKey = manifestType + "ManifestUrl";
-        for (final JsonObject streamingDataObject : streamingDataObjects) {
+
-            if (streamingDataObject != null) {
+        return streamingDataObjects.stream()
-                final String manifestKeyValue = streamingDataObject.getString(manifestKey);
+                .filter(Objects::nonNull)
-                if (manifestKeyValue != null) {
+                .map(streamingDataObject -> streamingDataObject.getString(manifestKey))
-                    return manifestKeyValue;
+                .filter(Objects::nonNull)
+                .findFirst()
+                .orElse(EMPTY_STRING);
+    }
+
+    @FunctionalInterface
+    interface StreamTypeStreamBuilderHelper<T extends Stream> {
+        T buildStream(String url, ItagItem itagItem);
+    }
+
+    /**
+     * Abstract method for
+     * {@link #getAudioStreams()}, {@link #getVideoOnlyStreams()} and {@link #getVideoStreams()}.
+     *
+     * @param itags           A map of Urls + ItagItems
+     * @param streamBuilder   Builds the stream from the provided data
+     * @param exMsgStreamType Stream type inside the exception message e.g. "video streams"
+     * @param <T>             Type of the stream
+     * @return
+     * @throws ExtractionException
+     */
+    private <T extends Stream> List<T> getStreamsByType(
+            final Map<String, ItagItem> itags,
+            final StreamTypeStreamBuilderHelper<T> streamBuilder,
+            final String exMsgStreamType
+    ) throws ExtractionException {
+        final List<T> streams = new ArrayList<>();
+
+        try {
+            for (final Map.Entry<String, ItagItem> entry : itags.entrySet()) {
+                final String url = tryDecryptUrl(entry.getKey(), getId());
+
+                final T stream = streamBuilder.buildStream(url, entry.getValue());
+                if (!Stream.containSimilarStream(stream, streams)) {
+                    streams.add(stream);
                 }
             }
+        } catch (final Exception e) {
+            throw new ParsingException("Could not get " + exMsgStreamType, e);
         }
 
-        return EMPTY_STRING;
+        return streams;
     }
 
     @Override
     public List<AudioStream> getAudioStreams() throws ExtractionException {
         assertPageFetched();
-        final List<AudioStream> audioStreams = new ArrayList<>();
+        return getStreamsByType(
-
+                getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO),
-        try {
+                AudioStream::new,
-            for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
+                "audio streams"
-                    ItagItem.ItagType.AUDIO).entrySet()) {
+        );
-                final ItagItem itag = entry.getValue();
-                final String url = tryDecryption(entry.getKey(), getId());
-
-                final AudioStream audioStream = new AudioStream(url, itag);
-                if (!Stream.containSimilarStream(audioStream, audioStreams)) {
-                    audioStreams.add(audioStream);
-                }
-            }
-        } catch (final Exception e) {
-            throw new ParsingException("Could not get audio streams", e);
-        }
-
-        return audioStreams;
     }
 
     @Override
     public List<VideoStream> getVideoStreams() throws ExtractionException {
         assertPageFetched();
-        final List<VideoStream> videoStreams = new ArrayList<>();
+        return getStreamsByType(
-
+                getItags(FORMATS, ItagItem.ItagType.VIDEO),
-        try {
+                (url, itag) -> new VideoStream(url, false, itag),
-            for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS,
+                "video streams"
-                    ItagItem.ItagType.VIDEO).entrySet()) {
+        );
-                final ItagItem itag = entry.getValue();
-                final String url = tryDecryption(entry.getKey(), getId());
-
-                final VideoStream videoStream = new VideoStream(url, false, itag);
-                if (!Stream.containSimilarStream(videoStream, videoStreams)) {
-                    videoStreams.add(videoStream);
-                }
-            }
-        } catch (final Exception e) {
-            throw new ParsingException("Could not get video streams", e);
-        }
-
-        return videoStreams;
     }
 
     @Override
     public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
         assertPageFetched();
-        final List<VideoStream> videoOnlyStreams = new ArrayList<>();
+        return getStreamsByType(
-
+                getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY),
-        try {
+                (url, itag) -> new VideoStream(url, true, itag),
-            for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
+                "video only streams"
-                    ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
+        );
-                final ItagItem itag = entry.getValue();
-                final String url = tryDecryption(entry.getKey(), getId());
-
-                final VideoStream videoStream = new VideoStream(url, true, itag);
-                if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
-                    videoOnlyStreams.add(videoStream);
-                }
-            }
-        } catch (final Exception e) {
-            throw new ParsingException("Could not get video only streams", e);
-        }
-
-        return videoOnlyStreams;
     }
 
     /**
@@ -636,7 +654,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
      * always needed.
      * This way a breaking change from YouTube does not result in a broken extractor.
      */
-    private String tryDecryption(final String url, final String videoId) {
+    private String tryDecryptUrl(final String url, final String videoId) {
         try {
             return YoutubeThrottlingDecrypter.apply(url, videoId);
         } catch (final ParsingException e) {
@@ -713,25 +731,33 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         try {
             final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
 
-            final JsonArray results = nextResponse.getObject("contents")
+            final JsonArray results = nextResponse
-                    .getObject("twoColumnWatchNextResults").getObject("secondaryResults")
+                    .getObject("contents")
-                    .getObject("secondaryResults").getArray("results");
+                    .getObject("twoColumnWatchNextResults")
+                    .getObject("secondaryResults")
+                    .getObject("secondaryResults")
+                    .getArray("results");
 
             final TimeAgoParser timeAgoParser = getTimeAgoParser();
+            results.stream()
+                    .filter(JsonObject.class::isInstance)
+                    .map(JsonObject.class::cast)
+                    .map(result -> {
+                        if (result.has("compactVideoRenderer")) {
+                            return new YoutubeStreamInfoItemExtractor(
+                                    result.getObject("compactVideoRenderer"), timeAgoParser);
+                        } else if (result.has("compactRadioRenderer")) {
+                            return new YoutubeMixOrPlaylistInfoItemExtractor(
+                                    result.getObject("compactRadioRenderer"));
+                        } else if (result.has("compactPlaylistRenderer")) {
+                            return new YoutubeMixOrPlaylistInfoItemExtractor(
+                                    result.getObject("compactPlaylistRenderer"));
+                        }
+                        return null;
+                    })
+                    .filter(Objects::nonNull)
+                    .forEach(collector::commit);
 
-            for (final Object resultObject : results) {
-                final JsonObject result = (JsonObject) resultObject;
-                if (result.has("compactVideoRenderer")) {
-                    collector.commit(new YoutubeStreamInfoItemExtractor(
-                            result.getObject("compactVideoRenderer"), timeAgoParser));
-                } else if (result.has("compactRadioRenderer")) {
-                    collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor(
-                            result.getObject("compactRadioRenderer")));
-                } else if (result.has("compactPlaylistRenderer")) {
-                    collector.commit(new YoutubeMixOrPlaylistInfoItemExtractor(
-                            result.getObject("compactPlaylistRenderer")));
-                }
-            }
             return collector;
         } catch (final Exception e) {
             throw new ParsingException("Could not get related videos", e);
@@ -777,9 +803,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
-        if (sts == null) {
+        initStsFromPlayerJsIfNeeded();
-            getStsFromPlayerJs();
-        }
 
         final String videoId = getId();
         final Localization localization = getExtractorLocalization();
@@ -833,13 +857,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         playerMicroFormatRenderer = youtubePlayerResponse.getObject("microformat")
                 .getObject("playerMicroformatRenderer");
 
-        final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(localization,
+        final byte[] body = JsonWriter.string(
-                contentCountry)
+                prepareDesktopJsonBuilder(localization, contentCountry)
-                .value(VIDEO_ID, videoId)
+                        .value(VIDEO_ID, videoId)
-                .value(CONTENT_CHECK_OK, true)
+                        .value(CONTENT_CHECK_OK, true)
-                .value(RACY_CHECK_OK, true)
+                        .value(RACY_CHECK_OK, true)
-                .done())
+                        .done())
-                .getBytes(UTF_8);
+                .getBytes(StandardCharsets.UTF_8);
         nextResponse = getJsonPostResponse(NEXT, body, localization);
 
         if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM)
@@ -863,53 +887,56 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                                         @Nonnull final JsonObject playabilityStatus)
             throws ParsingException {
         String status = playabilityStatus.getString("status");
+        if (status == null || status.equalsIgnoreCase("ok")) {
+            return;
+        }
+
         // If status exist, and is not "OK", throw the specific exception based on error message
         // or a ContentNotAvailableException with the reason text if it's an unknown reason.
-        if (status != null && !status.equalsIgnoreCase("ok")) {
+        final JsonObject newPlayabilityStatus =
-            final JsonObject newPlayabilityStatus
+                youtubePlayerResponse.getObject("playabilityStatus");
-                    = youtubePlayerResponse.getObject("playabilityStatus");
+        status = newPlayabilityStatus.getString("status");
-            status = newPlayabilityStatus.getString("status");
+        final String reason = newPlayabilityStatus.getString("reason");
-            final String reason = newPlayabilityStatus.getString("reason");
 
-            if (status.equalsIgnoreCase("login_required")) {
+        if (status.equalsIgnoreCase("login_required")) {
-                if (reason == null) {
+            if (reason == null) {
-                    final String message = newPlayabilityStatus.getArray("messages").getString(0);
+                final String message = newPlayabilityStatus.getArray("messages").getString(0);
-                    if (message != null && message.contains("private")) {
+                if (message != null && message.contains("private")) {
-                        throw new PrivateContentException("This video is private.");
+                    throw new PrivateContentException("This video is private.");
-                    }
-                } else if (reason.contains("age")) {
-                    // No streams can be fetched, therefore throw an AgeRestrictedContentException
-                    // explicitly.
-                    throw new AgeRestrictedContentException(
-                            "This age-restricted video cannot be watched.");
                 }
+            } else if (reason.contains("age")) {
+                // No streams can be fetched, therefore throw an AgeRestrictedContentException
+                // explicitly.
+                throw new AgeRestrictedContentException(
+                        "This age-restricted video cannot be watched.");
             }
-
-            if (status.equalsIgnoreCase("unplayable") && reason != null) {
-                if (reason.contains("Music Premium")) {
-                    throw new YoutubeMusicPremiumContentException();
-                }
-                if (reason.contains("payment")) {
-                    throw new PaidContentException("This video is a paid video");
-                }
-                if (reason.contains("members-only")) {
-                    throw new PaidContentException("This video is only available"
-                            + " for members of the channel of this video");
-                }
-
-                if (reason.contains("unavailable")) {
-                    final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
-                            .getObject("errorScreen").getObject("playerErrorMessageRenderer")
-                            .getObject("subreason"));
-                    if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) {
-                        throw new GeographicRestrictionException(
-                                "This video is not available in client's country.");
-                    }
-                }
-            }
-
-            throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
         }
+
+        if (status.equalsIgnoreCase("unplayable") && reason != null) {
+            if (reason.contains("Music Premium")) {
+                throw new YoutubeMusicPremiumContentException();
+            }
+            if (reason.contains("payment")) {
+                throw new PaidContentException("This video is a paid video");
+            }
+            if (reason.contains("members-only")) {
+                throw new PaidContentException("This video is only available"
+                        + " for members of the channel of this video");
+            }
+
+            if (reason.contains("unavailable")) {
+                final String detailedErrorMessage = getTextFromObject(newPlayabilityStatus
+                        .getObject("errorScreen")
+                        .getObject("playerErrorMessageRenderer")
+                        .getObject("subreason"));
+                if (detailedErrorMessage != null && detailedErrorMessage.contains("country")) {
+                    throw new GeographicRestrictionException(
+                            "This video is not available in client's country.");
+                }
+            }
+        }
+
+        throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
     }
 
     /**
@@ -921,14 +948,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                                               @Nonnull final String videoId)
             throws IOException, ExtractionException {
         androidCpn = generateContentPlaybackNonce();
-        final byte[] mobileBody = JsonWriter.string(prepareAndroidMobileJsonBuilder(
+        final byte[] mobileBody = JsonWriter.string(
-                localization, contentCountry)
+                prepareAndroidMobileJsonBuilder(localization, contentCountry)
-                .value(VIDEO_ID, videoId)
+                        .value(VIDEO_ID, videoId)
-                .value(CPN, androidCpn)
+                        .value(CPN, androidCpn)
-                .value(CONTENT_CHECK_OK, true)
+                        .value(CONTENT_CHECK_OK, true)
-                .value(RACY_CHECK_OK, true)
+                        .value(RACY_CHECK_OK, true)
-                .done())
+                        .done())
-                .getBytes(UTF_8);
+                .getBytes(StandardCharsets.UTF_8);
 
         final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER,
                 mobileBody, localization, "&t=" + generateTParameter()
@@ -952,14 +979,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                                           @Nonnull final String videoId)
             throws IOException, ExtractionException {
         iosCpn = generateContentPlaybackNonce();
-        final byte[] mobileBody = JsonWriter.string(prepareIosMobileJsonBuilder(
+        final byte[] mobileBody = JsonWriter.string(
-                        localization, contentCountry)
+                prepareIosMobileJsonBuilder(localization, contentCountry)
                         .value(VIDEO_ID, videoId)
                         .value(CPN, iosCpn)
                         .value(CONTENT_CHECK_OK, true)
                         .value(RACY_CHECK_OK, true)
                         .done())
-                .getBytes(UTF_8);
+                .getBytes(StandardCharsets.UTF_8);
 
         final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER,
                 mobileBody, localization, "&t=" + generateTParameter()
@@ -987,9 +1014,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                                              @Nonnull final Localization localization,
                                              @Nonnull final String videoId)
             throws IOException, ExtractionException {
-        if (sts == null) {
+        initStsFromPlayerJsIfNeeded();
-            getStsFromPlayerJs();
-        }
 
         // Because a cpn is unique to each request, we need to generate it again
         html5Cpn = generateContentPlaybackNonce();
@@ -1072,7 +1097,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return cachedDeobfuscationCode;
     }
 
-    private static void getStsFromPlayerJs() throws ParsingException {
+    private static void initStsFromPlayerJsIfNeeded() throws ParsingException {
         if (!isNullOrEmpty(sts)) {
             return;
         }
@@ -1134,31 +1159,28 @@ public class YoutubeStreamExtractor extends StreamExtractor {
         return theVideoPrimaryInfoRenderer;
     }
 
+    @Nonnull
     private JsonObject getVideoSecondaryInfoRenderer() throws ParsingException {
         if (videoSecondaryInfoRenderer != null) {
             return videoSecondaryInfoRenderer;
         }
 
-        final JsonArray contents = nextResponse.getObject("contents")
+        videoSecondaryInfoRenderer = nextResponse
-                .getObject("twoColumnWatchNextResults").getObject("results").getObject("results")
+                .getObject("contents")
-                .getArray("contents");
+                .getObject("twoColumnWatchNextResults")
+                .getObject("results")
+                .getObject("results")
+                .getArray("contents")
+                .stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .filter(content -> content.has("videoSecondaryInfoRenderer"))
+                .map(content -> content.getObject("videoSecondaryInfoRenderer"))
+                .findFirst()
+                .orElseThrow(
+                        () -> new ParsingException("Could not find videoSecondaryInfoRenderer"));
 
-        JsonObject theVideoSecondaryInfoRenderer = null;
+        return videoSecondaryInfoRenderer;
-
-        for (final Object content : contents) {
-            if (((JsonObject) content).has("videoSecondaryInfoRenderer")) {
-                theVideoSecondaryInfoRenderer = ((JsonObject) content)
-                        .getObject("videoSecondaryInfoRenderer");
-                break;
-            }
-        }
-
-        if (isNullOrEmpty(theVideoSecondaryInfoRenderer)) {
-            throw new ParsingException("Could not find videoSecondaryInfoRenderer");
-        }
-
-        videoSecondaryInfoRenderer = theVideoSecondaryInfoRenderer;
-        return theVideoSecondaryInfoRenderer;
     }
 
     @Nonnull
@@ -1194,83 +1216,82 @@ public class YoutubeStreamExtractor extends StreamExtractor {
             @Nonnull final String streamingDataKey,
             @Nonnull final ItagItem.ItagType itagTypeWanted,
             @Nonnull final String contentPlaybackNonce) {
-
+        if (streamingData == null || !streamingData.has(streamingDataKey)) {
-        final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
+            return Collections.emptyMap();
-        if (streamingData != null && streamingData.has(streamingDataKey)) {
-            final JsonArray formats = streamingData.getArray(streamingDataKey);
-            for (int i = 0; i != formats.size(); ++i) {
-                final JsonObject formatData = formats.getObject(i);
-                final int itag = formatData.getInt("itag");
-
-                if (ItagItem.isSupported(itag)) {
-                    try {
-                        final ItagItem itagItem = ItagItem.getItag(itag);
-                        if (itagItem.itagType == itagTypeWanted) {
-                            // Ignore streams that are delivered using YouTube's OTF format,
-                            // as those only work with DASH and not with progressive HTTP.
-                            if (formatData.getString("type", EMPTY_STRING)
-                                    .equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) {
-                                continue;
-                            }
-
-                            final String streamUrl;
-                            if (formatData.has("url")) {
-                                streamUrl = formatData.getString("url") + "&cpn="
-                                        + contentPlaybackNonce;
-                            } else {
-                                // This url has an obfuscated signature
-                                final String cipherString = formatData.has("cipher")
-                                        ? formatData.getString("cipher")
-                                        : formatData.getString("signatureCipher");
-                                final Map<String, String> cipher = Parser.compatParseMap(
-                                        cipherString);
-                                streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
-                                        + deobfuscateSignature(cipher.get("s"));
-                            }
-
-                            final JsonObject initRange = formatData.getObject("initRange");
-                            final JsonObject indexRange = formatData.getObject("indexRange");
-                            final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
-                            final String codec = mimeType.contains("codecs")
-                                    ? mimeType.split("\"")[1] : EMPTY_STRING;
-
-                            itagItem.setBitrate(formatData.getInt("bitrate"));
-                            itagItem.setWidth(formatData.getInt("width"));
-                            itagItem.setHeight(formatData.getInt("height"));
-                            itagItem.setInitStart(Integer.parseInt(initRange.getString("start",
-                                    "-1")));
-                            itagItem.setInitEnd(Integer.parseInt(initRange.getString("end",
-                                    "-1")));
-                            itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start",
-                                    "-1")));
-                            itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end",
-                                    "-1")));
-                            itagItem.fps = formatData.getInt("fps");
-                            itagItem.setQuality(formatData.getString("quality"));
-                            itagItem.setCodec(codec);
-
-                            urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
-                        }
-                    } catch (final UnsupportedEncodingException | ParsingException ignored) {
-                    }
-                }
-            }
         }
 
+        final Map<String, ItagItem> urlAndItagsFromStreamingDataObject = new LinkedHashMap<>();
+        final JsonArray formats = streamingData.getArray(streamingDataKey);
+        for (int i = 0; i < formats.size(); i++) {
+            final JsonObject formatData = formats.getObject(i);
+            final int itag = formatData.getInt("itag");
+
+            if (!ItagItem.isSupported(itag)) {
+                continue;
+            }
+
+            try {
+                final ItagItem itagItem = ItagItem.getItag(itag);
+                if (itagItem.itagType != itagTypeWanted) {
+                    continue;
+                }
+
+                // Ignore streams that are delivered using YouTube's OTF format,
+                // as those only work with DASH and not with progressive HTTP.
+                if ("FORMAT_STREAM_TYPE_OTF".equalsIgnoreCase(formatData.getString("type"))) {
+                    continue;
+                }
+
+                final String streamUrl;
+                if (formatData.has("url")) {
+                    streamUrl = formatData.getString("url");
+                } else {
+                    // This url has an obfuscated signature
+                    final String cipherString = formatData.has("cipher")
+                            ? formatData.getString("cipher")
+                            : formatData.getString("signatureCipher");
+                    final Map<String, String> cipher = Parser.compatParseMap(
+                            cipherString);
+                    streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+                            + deobfuscateSignature(cipher.get("s"));
+                }
+
+                final JsonObject initRange = formatData.getObject("initRange");
+                final JsonObject indexRange = formatData.getObject("indexRange");
+                final String mimeType = formatData.getString("mimeType", EMPTY_STRING);
+                final String codec = mimeType.contains("codecs")
+                        ? mimeType.split("\"")[1]
+                        : EMPTY_STRING;
+
+                itagItem.setBitrate(formatData.getInt("bitrate"));
+                itagItem.setWidth(formatData.getInt("width"));
+                itagItem.setHeight(formatData.getInt("height"));
+                itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
+                itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
+                itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
+                itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
+                itagItem.fps = formatData.getInt("fps");
+                itagItem.setQuality(formatData.getString("quality"));
+                itagItem.setCodec(codec);
+
+                urlAndItagsFromStreamingDataObject.put(streamUrl, itagItem);
+            } catch (final UnsupportedEncodingException | ParsingException ignored) {
+            }
+        }
         return urlAndItagsFromStreamingDataObject;
     }
 
+
     @Nonnull
     @Override
     public List<Frameset> getFrames() throws ExtractionException {
         try {
             final JsonObject storyboards = playerResponse.getObject("storyboards");
-            final JsonObject storyboardsRenderer;
+            final JsonObject storyboardsRenderer = storyboards.getObject(
-            if (storyboards.has("playerLiveStoryboardSpecRenderer")) {
+                    storyboards.has("playerLiveStoryboardSpecRenderer")
-                storyboardsRenderer = storyboards.getObject("playerLiveStoryboardSpecRenderer");
+                            ? "playerLiveStoryboardSpecRenderer"
-            } else {
+                            : "playerStoryboardSpecRenderer"
-                storyboardsRenderer = storyboards.getObject("playerStoryboardSpecRenderer");
+            );
-            }
 
             if (storyboardsRenderer == null) {
                 return Collections.emptyList();
@@ -1283,15 +1304,13 @@ public class YoutubeStreamExtractor extends StreamExtractor {
 
             final String[] spec = storyboardsRendererSpec.split("\\|");
             final String url = spec[0];
-            final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
+            final List<Frameset> result = new ArrayList<>(spec.length - 1);
 
             for (int i = 1; i < spec.length; ++i) {
                 final String[] parts = spec[i].split("#");
                 if (parts.length != 8 || Integer.parseInt(parts[5]) == 0) {
                     continue;
                 }
-                final int frameWidth = Integer.parseInt(parts[0]);
-                final int frameHeight = Integer.parseInt(parts[1]);
                 final int totalCount = Integer.parseInt(parts[2]);
                 final int framesPerPageX = Integer.parseInt(parts[3]);
                 final int framesPerPageY = Integer.parseInt(parts[4]);
@@ -1310,15 +1329,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
                 }
                 result.add(new Frameset(
                         urls,
-                        frameWidth,
+                        /*frameWidth=*/Integer.parseInt(parts[0]),
-                        frameHeight,
+                        /*frameHeight=*/Integer.parseInt(parts[1]),
                         totalCount,
-                        Integer.parseInt(parts[5]),
+                        /*durationPerFrame=*/Integer.parseInt(parts[5]),
                         framesPerPageX,
                         framesPerPageY
                 ));
             }
-            result.trimToSize();
             return result;
         } catch (final Exception e) {
             throw new ExtractionException("Could not get frames", e);
@@ -1328,8 +1346,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public Privacy getPrivacy() {
-        final boolean isUnlisted = playerMicroFormatRenderer.getBoolean("isUnlisted");
+        return playerMicroFormatRenderer.getBoolean("isUnlisted")
-        return isUnlisted ? Privacy.UNLISTED : Privacy.PUBLIC;
+                ? Privacy.UNLISTED
+                : Privacy.PUBLIC;
     }
 
     @Nonnull
@@ -1342,14 +1361,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Override
     public String getLicence() throws ParsingException {
         final JsonObject metadataRowRenderer = getVideoSecondaryInfoRenderer()
-                .getObject("metadataRowContainer").getObject("metadataRowContainerRenderer")
+                .getObject("metadataRowContainer")
+                .getObject("metadataRowContainerRenderer")
                 .getArray("rows")
-                .getObject(0).getObject("metadataRowRenderer");
+                .getObject(0)
+                .getObject("metadataRowRenderer");
 
         final JsonArray contents = metadataRowRenderer.getArray("contents");
         final String license = getTextFromObject(contents.getObject(0));
-        return license != null && "Licence".equals(getTextFromObject(metadataRowRenderer
+        return license != null
-                .getObject("title"))) ? license : "YouTube licence";
+                && "Licence".equals(getTextFromObject(metadataRowRenderer.getObject("title")))
+                ? license
+                : "YouTube licence";
     }
 
     @Override
@@ -1367,63 +1390,73 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<StreamSegment> getStreamSegments() throws ParsingException {
-        final ArrayList<StreamSegment> segments = new ArrayList<>();
-        if (nextResponse.has("engagementPanels")) {
-            final JsonArray panels = nextResponse.getArray("engagementPanels");
-            JsonArray segmentsArray = null;
 
-            // Search for correct panel containing the data
+        if (!nextResponse.has("engagementPanels")) {
-            for (int i = 0; i < panels.size(); i++) {
+            return Collections.emptyList();
-                final String panelIdentifier = panels.getObject(i)
+        }
+
+        final JsonArray segmentsArray = nextResponse.getArray("engagementPanels")
+                .stream()
+                // Check if object is a JsonObject
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                // Check if the panel is the correct one
+                .filter(panel -> "engagement-panel-macro-markers-description-chapters".equals(
+                        panel
+                                .getObject("engagementPanelSectionListRenderer")
+                                .getString("panelIdentifier")))
+                // Extract the data
+                .map(panel -> panel
                         .getObject("engagementPanelSectionListRenderer")
-                        .getString("panelIdentifier");
+                        .getObject("content")
-                // panelIdentifier might be null if the panel has something to do with ads
+                        .getObject("macroMarkersListRenderer")
-                // See https://github.com/TeamNewPipe/NewPipe/issues/7792#issuecomment-1030900188
+                        .getArray("contents"))
-                if ("engagement-panel-macro-markers-description-chapters".equals(panelIdentifier)) {
+                .findFirst()
-                    segmentsArray = panels.getObject(i)
+                .orElse(null);
-                            .getObject("engagementPanelSectionListRenderer").getObject("content")
+
-                            .getObject("macroMarkersListRenderer").getArray("contents");
+        // If no data was found exit
-                    break;
+        if (segmentsArray == null) {
-                }
+            return Collections.emptyList();
+        }
+
+        final long duration = getLength();
+        final List<StreamSegment> segments = new ArrayList<>();
+        for (final JsonObject segmentJson : segmentsArray.stream()
+                .filter(JsonObject.class::isInstance)
+                .map(JsonObject.class::cast)
+                .map(object -> object.getObject("macroMarkersListItemRenderer"))
+                .collect(Collectors.toList())
+        ) {
+            final int startTimeSeconds = segmentJson.getObject("onTap")
+                    .getObject("watchEndpoint").getInt("startTimeSeconds", -1);
+
+            if (startTimeSeconds == -1) {
+                throw new ParsingException("Could not get stream segment start time.");
+            }
+            if (startTimeSeconds > duration) {
+                break;
             }
 
-            if (segmentsArray != null) {
+            final String title = getTextFromObject(segmentJson.getObject("title"));
-                final long duration = getLength();
+            if (isNullOrEmpty(title)) {
-                for (final Object object : segmentsArray) {
+                throw new ParsingException("Could not get stream segment title.");
-                    final JsonObject segmentJson = ((JsonObject) object)
+            }
-                            .getObject("macroMarkersListItemRenderer");
 
-                    final int startTimeSeconds = segmentJson.getObject("onTap")
+            final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
-                            .getObject("watchEndpoint").getInt("startTimeSeconds", -1);
+            segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
-
+            if (segmentJson.has("thumbnail")) {
-                    if (startTimeSeconds == -1) {
+                final JsonArray previewsArray = segmentJson
-                        throw new ParsingException("Could not get stream segment start time.");
+                        .getObject("thumbnail")
-                    }
+                        .getArray("thumbnails");
-                    if (startTimeSeconds > duration) {
+                if (!previewsArray.isEmpty()) {
-                        break;
+                    // Assume that the thumbnail with the highest resolution is at the last position
-                    }
+                    final String url = previewsArray
-
+                            .getObject(previewsArray.size() - 1)
-                    final String title = getTextFromObject(segmentJson.getObject("title"));
+                            .getString("url");
-                    if (isNullOrEmpty(title)) {
+                    segment.setPreviewUrl(fixThumbnailUrl(url));
-                        throw new ParsingException("Could not get stream segment title.");
-                    }
-
-                    final StreamSegment segment = new StreamSegment(title, startTimeSeconds);
-                    segment.setUrl(getUrl() + "?t=" + startTimeSeconds);
-                    if (segmentJson.has("thumbnail")) {
-                        final JsonArray previewsArray = segmentJson.getObject("thumbnail")
-                                .getArray("thumbnails");
-                        if (!previewsArray.isEmpty()) {
-                            // Assume that the thumbnail with the highest resolution is at the
-                            // last position
-                            final String url = previewsArray
-                                    .getObject(previewsArray.size() - 1).getString("url");
-                            segment.setPreviewUrl(fixThumbnailUrl(url));
-                        }
-                    }
-                    segments.add(segment);
                 }
             }
+            segments.add(segment);
         }
         return segments;
     }
@@ -1431,9 +1464,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     @Nonnull
     @Override
     public List<MetaInfo> getMetaInfo() throws ParsingException {
-        return YoutubeParsingHelper.getMetaInfo(
+        return YoutubeParsingHelper.getMetaInfo(nextResponse
-                nextResponse.getObject("contents").getObject("twoColumnWatchNextResults")
+                .getObject("contents")
-                        .getObject("results").getObject("results").getArray("contents"));
+                .getObject("twoColumnWatchNextResults")
+                .getObject("results")
+                .getObject("results")
+                .getArray("contents"));
     }
 
     /**

extractor/src/main/java/org/schabi/newpipe/extractor/stream/Stream.java

@@ -1,19 +1,19 @@
 package org.schabi.newpipe.extractor.stream;
 
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
+
 import org.schabi.newpipe.extractor.MediaFormat;
 
 import java.io.Serializable;
 import java.util.List;
 
-import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
-
 /**
  * Creates a stream object from url, format and optional torrent url
  */
 public abstract class Stream implements Serializable {
     private final MediaFormat mediaFormat;
-    public final String url;
+    private final String url;
-    public final String torrentUrl;
+    private final String torrentUrl;
 
     /**
      * @deprecated Use {@link #getFormat()}  or {@link #getFormatId()}

extractor/src/main/java/org/schabi/newpipe/extractor/utils/Utils.java

@@ -2,8 +2,6 @@ package org.schabi.newpipe.extractor.utils;
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -18,10 +16,17 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Pattern;
 
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
 public final class Utils {
 
     public static final String HTTP = "http://";
     public static final String HTTPS = "https://";
+    /**
+     * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8}
+     */
+    @Deprecated
     public static final String UTF_8 = "UTF-8";
     public static final String EMPTY_STRING = "";
     private static final Pattern M_PATTERN = Pattern.compile("(https?)?:\\/\\/m\\.");