diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 12a992e1..55fbb8e7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -87,6 +87,7 @@ public final class YoutubeParsingHelper { private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; private static final String ANDROID_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; + private static final String IOS_YOUTUBE_KEY = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc"; private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.49.37"; private static String clientVersion; @@ -717,9 +718,9 @@ public final class YoutubeParsingHelper { return youtubeMusicKey; } - String musicClientVersion = null; - String musicKey = null; - String musicClientName = null; + String musicClientVersion; + String musicKey; + String musicClientName; try { final String url = "https://music.youtube.com/sw.js"; @@ -950,16 +951,15 @@ public final class YoutubeParsingHelper { public static JsonObject getJsonAndroidPostResponse( final String endpoint, final byte[] body, - @Nonnull final ContentCountry contentCountry, - final Localization localization, + @Nonnull final Localization localization, @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { final Map> headers = new HashMap<>(); headers.put("Content-Type", Collections.singletonList("application/json")); // Spoofing an Android 11 device with the hardcoded version of the Android app headers.put("User-Agent", Collections.singletonList("com.google.android.youtube/" + MOBILE_YOUTUBE_CLIENT_VERSION + " (Linux; U; Android 11; " - + contentCountry.getCountryCode() + ") gzip")); - headers.put("x-goog-api-format-version", Collections.singletonList("2")); + + localization.getCountryCode() + ") gzip")); + headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2")); final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint + "?key=" + ANDROID_YOUTUBE_KEY; @@ -971,6 +971,29 @@ public final class YoutubeParsingHelper { return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); } + public static JsonObject getJsonIosPostResponse( + final String endpoint, + final byte[] body, + @Nonnull final Localization localization, + @Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException { + final Map> headers = new HashMap<>(); + headers.put("Content-Type", Collections.singletonList("application/json")); + // Spoofing an iPhone 13 running iOS 15.2 with the hardcoded mobile client version + headers.put("User-Agent", Collections.singletonList("com.google.ios.youtube/" + + MOBILE_YOUTUBE_CLIENT_VERSION + "(iPhone14,5; U; CPU iOS 15_2 like Mac OS X; " + + localization.getCountryCode() + ")")); + headers.put("X-Goog-Api-Format-Version", Collections.singletonList("2")); + + final String baseEndpointUrl = "https://youtubei.googleapis.com/youtubei/v1/" + endpoint + + "?key=" + IOS_YOUTUBE_KEY; + + final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest) + ? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest, + headers, body, localization); + + return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + } + @Nonnull public static JsonBuilder prepareDesktopJsonBuilder( @Nonnull final Localization localization, @@ -1011,6 +1034,32 @@ public final class YoutubeParsingHelper { .object("client") .value("clientName", "ANDROID") .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION) + .value("platform", "MOBILE") + .value("hl", localization.getLocalizationCode()) + .value("gl", contentCountry.getCountryCode()) + .end() + .object("user") + // TO DO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .end(); + // @formatter:on + } + + @Nonnull + public static JsonBuilder prepareIosMobileJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry) { + // @formatter:off + return JsonObject.builder() + .object("context") + .object("client") + .value("clientName", "IOS") + .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION) + // Device model is required to get 60fps streams + .value("deviceModel", "iPhone14,5") + .value("platform", "MOBILE") .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .end() @@ -1070,6 +1119,45 @@ public final class YoutubeParsingHelper { .value("clientName", "ANDROID") .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION) .value("clientScreen", "EMBED") + .value("platform", "MOBILE") + .value("hl", localization.getLocalizationCode()) + .value("gl", contentCountry.getCountryCode()) + .end() + .object("thirdParty") + .value("embedUrl", "https://www.youtube.com/watch?v=" + videoId) + .end() + .object("request") + .array("internalExperimentFlags") + .end() + .value("useSsl", true) + .end() + .object("user") + // TO DO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .end() + .value(CPN, contentPlaybackNonce) + .value(VIDEO_ID, videoId); + // @formatter:on + } + + @Nonnull + public static JsonBuilder prepareIosMobileEmbedVideoJsonBuilder( + @Nonnull final Localization localization, + @Nonnull final ContentCountry contentCountry, + @Nonnull final String videoId, + @Nonnull final String contentPlaybackNonce) { + // @formatter:off + return JsonObject.builder() + .object("context") + .object("client") + .value("clientName", "IOS") + .value("clientVersion", MOBILE_YOUTUBE_CLIENT_VERSION) + .value("clientScreen", "EMBED") + // Device model is required to get 60fps streams + .value("deviceModel", "iPhone14,5") + .value("platform", "MOBILE") .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .end() diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index c07d9e33..151677d8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -116,6 +116,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nullable private static String playerCode = null; + private static boolean isAndroidClientFetchForced = false; + private static boolean isIosClientFetchForced = false; + private JsonObject playerResponse; private JsonObject nextResponse; @@ -123,14 +126,19 @@ public class YoutubeStreamExtractor extends StreamExtractor { private JsonObject desktopStreamingData; @Nullable private JsonObject androidStreamingData; + @Nullable + private JsonObject iosStreamingData; + private JsonObject videoPrimaryInfoRenderer; private JsonObject videoSecondaryInfoRenderer; private int ageLimit = -1; + private StreamType streamType; @Nullable private List subtitles = null; private String desktopCpn; private String androidCpn; + private String iosCpn; public YoutubeStreamExtractor(final StreamingService service, final LinkHandler linkHandler) { super(service, linkHandler); @@ -511,10 +519,15 @@ public class YoutubeStreamExtractor extends StreamExtractor { public String getHlsUrl() throws ParsingException { assertPageFetched(); - if (desktopStreamingData != null) { - return desktopStreamingData.getString("hlsManifestUrl"); + // Return HLS manifest of the iOS client first because on livestreams, the HLS manifest + // returned has separated audio and video streams + // Also, on videos, non-iOS clients don't have an HLS manifest URL in their player response + if (iosStreamingData != null) { + return iosStreamingData.getString("hlsManifestUrl", EMPTY_STRING); + } else if (desktopStreamingData != null) { + return desktopStreamingData.getString("hlsManifestUrl", EMPTY_STRING); } else if (androidStreamingData != null) { - return androidStreamingData.getString("hlsManifestUrl"); + return androidStreamingData.getString("hlsManifestUrl", EMPTY_STRING); } else { return EMPTY_STRING; } @@ -651,11 +664,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { public StreamType getStreamType() { assertPageFetched(); + return streamType; + } + + private void setStreamType() { if (playerResponse.getObject("playabilityStatus").has("liveStreamability") || playerResponse.getObject("videoDetails").getBoolean("isPostLiveDvr", false)) { - return StreamType.LIVE_STREAM; + streamType = StreamType.LIVE_STREAM; + } else { + streamType = StreamType.VIDEO_STREAM; } - return StreamType.VIDEO_STREAM; } @Nullable @@ -763,14 +781,30 @@ public class YoutubeStreamExtractor extends StreamExtractor { final boolean ageRestricted = playabilityStatus.getString("reason", EMPTY_STRING) .contains("age"); + setStreamType(); + if (!playerResponse.has(STREAMING_DATA)) { try { fetchDesktopEmbedJsonPlayer(contentCountry, localization, videoId); } catch (final Exception ignored) { } - try { - fetchAndroidEmbedJsonPlayer(contentCountry, localization, videoId); - } catch (final Exception ignored) { + + // Refresh the stream type because the stream type maybe not properly known for + // age-restricted videos + setStreamType(); + + if (streamType == StreamType.VIDEO_STREAM || isAndroidClientFetchForced) { + try { + fetchAndroidEmbedJsonPlayer(contentCountry, localization, videoId); + } catch (final Exception ignored) { + } + } + + if (streamType == StreamType.LIVE_STREAM || isIosClientFetchForced) { + try { + fetchIosEmbedJsonPlayer(contentCountry, localization, videoId); + } catch (final Exception ignored) { + } } } @@ -798,12 +832,21 @@ public class YoutubeStreamExtractor extends StreamExtractor { nextResponse = getJsonPostResponse(NEXT, body, localization); } - if (!ageRestricted) { + if ((!ageRestricted && streamType == StreamType.VIDEO_STREAM) + || isAndroidClientFetchForced) { try { fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId); } catch (final Exception ignored) { } } + + if ((!ageRestricted && streamType == StreamType.LIVE_STREAM) + || isIosClientFetchForced) { + try { + fetchIosMobileJsonPlayer(contentCountry, localization, videoId); + } catch (final Exception ignored) { + } + } } private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse, @@ -860,7 +903,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { } /** - * Fetch the Android Mobile API and assign the streaming data to the mobileStreamingData JSON + * Fetch the Android Mobile API and assign the streaming data to the androidStreamingData JSON * object. */ private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, @@ -875,7 +918,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { .getBytes(UTF_8); final JsonObject androidPlayerResponse = getJsonAndroidPostResponse(PLAYER, - mobileBody, contentCountry, localization, "&t=" + generateTParameter() + mobileBody, localization, "&t=" + generateTParameter() + "&id=" + videoId); final JsonObject streamingData = androidPlayerResponse.getObject(STREAMING_DATA); @@ -888,7 +931,45 @@ public class YoutubeStreamExtractor extends StreamExtractor { } /** - * Download again the desktop JSON player as an embed client to bypass some age-restrictions. + * Fetch the iOS Mobile API and assign the streaming data to the iosStreamingData JSON + * object. + */ + private void fetchIosMobileJsonPlayer(@Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId) + throws IOException, ExtractionException { + iosCpn = generateContentPlaybackNonce(); + final byte[] mobileBody = JsonWriter.string(prepareIosMobileJsonBuilder( + localization, contentCountry) + .value(VIDEO_ID, videoId) + .value(CPN, iosCpn) + .done()) + .getBytes(UTF_8); + + final JsonObject iosPlayerResponse = getJsonIosPostResponse(PLAYER, + mobileBody, localization, "&t=" + generateTParameter() + + "&id=" + videoId); + + final JsonObject streamingData = iosPlayerResponse.getObject(STREAMING_DATA); + if (!isNullOrEmpty(streamingData)) { + iosStreamingData = streamingData; + if (desktopStreamingData == null) { + playerResponse = iosPlayerResponse; + } + } + } + + /** + * Download the web desktop JSON player as an embed client to bypass some age-restrictions and + * assign the streaming data to the desktopStreamingData JSON object. + * + * @param contentCountry the content country to use + * @param localization the localization to use + * @param videoId the video id + * @throws IOException if something goes wrong when fetching the web desktop embed + * player endpoint + * @throws ExtractionException if something goes wrong when fetching the web desktop embed + * player endpoint */ private void fetchDesktopEmbedJsonPlayer(@Nonnull final ContentCountry contentCountry, @Nonnull final Localization localization, @@ -914,7 +995,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { } /** - * Download the Android mobile JSON player as an embed client to bypass some age-restrictions. + * Download the Android mobile JSON player as an embed client to bypass some age-restrictions + * and assign the streaming data to the androidStreamingData JSON object. + * + * @param contentCountry the content country to use + * @param localization the localization to use + * @param videoId the video id + * @throws IOException if something goes wrong when fetching the Android embed player + * endpoint + * @throws ExtractionException if something goes wrong when fetching the Android embed player + * endpoint */ private void fetchAndroidEmbedJsonPlayer(@Nonnull final ContentCountry contentCountry, @Nonnull final Localization localization, @@ -929,7 +1019,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { .done()) .getBytes(UTF_8); final JsonObject androidMobileEmbedPlayerResponse = getJsonAndroidPostResponse(PLAYER, - androidMobileEmbedBody, contentCountry, localization, "&t=" + generateTParameter() + androidMobileEmbedBody, localization, "&t=" + generateTParameter() + "&id=" + videoId); final JsonObject streamingData = androidMobileEmbedPlayerResponse.getObject( STREAMING_DATA); @@ -941,6 +1031,43 @@ public class YoutubeStreamExtractor extends StreamExtractor { } } + /** + * Download the iOS mobile JSON player as an embed client to bypass some age-restrictions and + * assign the streaming data to the iosStreamingData JSON object. + * + * @param contentCountry the content country to use + * @param localization the localization to use + * @param videoId the video id + * @throws IOException if something goes wrong when fetching the iOS embed player + * endpoint + * @throws ExtractionException if something goes wrong when fetching the iOS embed player + * endpoint + */ + private void fetchIosEmbedJsonPlayer(@Nonnull final ContentCountry contentCountry, + @Nonnull final Localization localization, + @Nonnull final String videoId) + throws IOException, ExtractionException { + // Because a cpn is unique to each request, we need to generate it again + iosCpn = generateContentPlaybackNonce(); + + final byte[] androidMobileEmbedBody = JsonWriter.string( + prepareIosMobileEmbedVideoJsonBuilder(localization, contentCountry, videoId, + iosCpn) + .done()) + .getBytes(UTF_8); + final JsonObject iosMobileEmbedPlayerResponse = getJsonIosPostResponse(PLAYER, + androidMobileEmbedBody, localization, "&t=" + generateTParameter() + + "&id=" + videoId); + final JsonObject streamingData = iosMobileEmbedPlayerResponse.getObject( + STREAMING_DATA); + if (!isNullOrEmpty(streamingData)) { + if (desktopStreamingData == null) { + playerResponse = iosMobileEmbedPlayerResponse; + } + iosStreamingData = iosMobileEmbedPlayerResponse.getObject(STREAMING_DATA); + } + } + private static void storePlayerJs() throws ParsingException { try { playerCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(); @@ -1104,12 +1231,17 @@ public class YoutubeStreamExtractor extends StreamExtractor { return urlAndItags; } - // Use the mobileStreamingData object first because there is no n param and no + // Use the androidStreamingData object first because there is no n param and no // signatureCiphers in streaming URLs of the Android client urlAndItags.putAll(getStreamsFromStreamingDataKey( androidStreamingData, streamingDataKey, itagTypeWanted, androidCpn)); urlAndItags.putAll(getStreamsFromStreamingDataKey( desktopStreamingData, streamingDataKey, itagTypeWanted, desktopCpn)); + // Use the iosStreamingData object in the last position because most of the available + // streams can be extracted with the Android and web clients and also because the iOS + // client is only enabled by default on livestreams + urlAndItags.putAll(getStreamsFromStreamingDataKey( + iosStreamingData, streamingDataKey, itagTypeWanted, androidCpn)); return urlAndItags; } @@ -1380,4 +1512,43 @@ public class YoutubeStreamExtractor extends StreamExtractor { sts = null; YoutubeJavaScriptExtractor.resetJavaScriptCode(); } + + /** + * Enable or disable the fetch of the Android client for all stream types. + * + *

+ * By default, the fetch of the Android client will be made only on videos, in order to reduce + * data usage, because available streams of the Android client will be almost equal to the ones + * available on the web client. + *

+ * + *

+ * Enabling this option will allow you to get a 48kbps audio + * stream on livestreams without fetching the DASH manifest returned in YouTube's player + * response. + *

+ * @param forceFetchOfAndroidClientValue whether to always fetch the Android client and not + * only for videos + */ + public static void forceFetchOfAndroidClient(final boolean forceFetchOfAndroidClientValue) { + isAndroidClientFetchForced = forceFetchOfAndroidClientValue; + } + + /** + * Enable or disable the fetch of the iOS client for all stream types. + * + *

+ * By default, the fetch of the iOS client will be made only on livestreams, in order to get an + * HLS manifest with separated audio and video. + *

+ *

+ * Enabling this option will allow you to get an + * HLS manifest also for videos. + *

+ * @param forceFetchOfIosClientValue whether to always fetch the iOS client and not only for + * livestreams + */ + public static void forceFetchOfIosClient(final boolean forceFetchOfIosClientValue) { + isIosClientFetchForced = forceFetchOfIosClientValue; + } }