From 4299d806a2b9353ae9cd90a30a12dd877a18c2ab Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Thu, 24 Jun 2021 18:39:16 +0200 Subject: [PATCH] Adress changes --- .../youtube/YoutubeParsingHelper.java | 63 +++++++++++-------- .../extractors/YoutubeChannelExtractor.java | 42 +++++++------ .../YoutubeMixPlaylistExtractor.java | 32 +++------- .../YoutubeMusicSearchExtractor.java | 6 +- .../extractors/YoutubeSearchExtractor.java | 19 ++---- .../extractors/YoutubeStreamExtractor.java | 38 ++++++----- ...utubeStreamExtractorAgeRestrictedTest.java | 5 +- 7 files changed, 95 insertions(+), 110 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index a11aabe6..687bec82 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -66,15 +66,15 @@ public class YoutubeParsingHelper { public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/"; - private static final String HARDCODED_CLIENT_VERSION = "2.20210603.07.00"; + private static final String HARDCODED_CLIENT_VERSION = "2.20210622.10.00"; private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; private static final String MOBILE_YOUTUBE_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"; - private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.20.36"; + private static final String MOBILE_YOUTUBE_CLIENT_VERSION = "16.23.36"; private static String clientVersion; private static String key; private static final String[] HARDCODED_YOUTUBE_MUSIC_KEY = - {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"}; + {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "1.20210621.00.00"}; private static String[] youtubeMusicKey; private static boolean keyAndVersionExtracted = false; @@ -102,7 +102,8 @@ public class YoutubeParsingHelper { try { final URL u = new URL(url); final String host = u.getHost(); - return host.startsWith("google.") || host.startsWith("m.google.") + return host.startsWith("google.") + || host.startsWith("m.google.") || host.startsWith("www.google."); } catch (final MalformedURLException e) { return false; @@ -111,7 +112,8 @@ public class YoutubeParsingHelper { public static boolean isYoutubeURL(@Nonnull final URL url) { final String host = url.getHost(); - return host.equalsIgnoreCase("youtube.com") || host.equalsIgnoreCase("www.youtube.com") + return host.equalsIgnoreCase("youtube.com") + || host.equalsIgnoreCase("www.youtube.com") || host.equalsIgnoreCase("m.youtube.com") || host.equalsIgnoreCase("music.youtube.com"); } @@ -234,7 +236,7 @@ public class YoutubeParsingHelper { * Checks if the given playlist id is a YouTube Mix (auto-generated playlist) * Ids from a YouTube Mix start with "RD" * - * @param playlistId the id of the playlist + * @param playlistId the playlist id * @return Whether given id belongs to a YouTube Mix */ public static boolean isYoutubeMixId(@Nonnull final String playlistId) { @@ -306,8 +308,8 @@ public class YoutubeParsingHelper { } } - public static boolean areHardcodedClientVersionAndKeyValid() throws IOException, - ExtractionException { + public static boolean areHardcodedClientVersionAndKeyValid() + throws IOException, ExtractionException { if (areHardcodedClientVersionAndKeyValidValue != null) { return areHardcodedClientVersionAndKeyValidValue; } @@ -316,11 +318,17 @@ public class YoutubeParsingHelper { .object() .object("context") .object("client") - .value("hl", "en") + .value("hl", "en-GB") .value("gl", "GB") - .value("clientName", "1") + .value("clientName", "WEB") .value("clientVersion", HARDCODED_CLIENT_VERSION) .end() + .object("user") + // TO DO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() + .value("fetchLiveState", true) .end() .end().done().getBytes(UTF_8); // @formatter:on @@ -337,9 +345,8 @@ public class YoutubeParsingHelper { final String responseBody = response.responseBody(); final int responseCode = response.responseCode(); - areHardcodedClientVersionAndKeyValidValue = responseBody.length() > 5000 + return areHardcodedClientVersionAndKeyValidValue = responseBody.length() > 5000 && responseCode == 200; // Ensure to have a valid response - return areHardcodedClientVersionAndKeyValidValue; } private static void extractClientVersionAndKey() throws IOException, ExtractionException { @@ -485,8 +492,7 @@ public class YoutubeParsingHelper { .value("hl", "en-GB") .value("gl", "GB") .array("experimentIds").end() - .value("experimentsToken", "") - .value("utcOffsetMinutes", 0) + .value("experimentsToken", EMPTY_STRING) .object("locationInfo").end() .object("musicAppInfo").end() .end() @@ -802,10 +808,13 @@ public class YoutubeParsingHelper { return JsonObject.builder() .object("context") .object("client") - .value("clientName", "WEB") - .value("clientVersion", getClientVersion()) .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) + .value("clientName", "WEB") + .value("clientVersion", getClientVersion()) + .end() + .object("user") + .value("lockedSafetyMode", false) .end() .end(); // @formatter:on @@ -826,6 +835,11 @@ public class YoutubeParsingHelper { .value("hl", localization.getLocalizationCode()) .value("gl", contentCountry.getCountryCode()) .end() + .object("user") + // TO DO: provide a way to enable restricted mode with: + // .value("enableSafetyMode", boolean) + .value("lockedSafetyMode", false) + .end() .end(); // @formatter:on } @@ -848,15 +862,11 @@ public class YoutubeParsingHelper { */ public static void addClientInfoHeaders(@Nonnull final Map> headers) throws IOException, ExtractionException { - if (headers.get("Origin") == null) { - headers.put("Origin", Collections.singletonList("https://www.youtube.com")); - } - if (headers.get("Referer") == null) { - headers.put("Referer", Collections.singletonList("https://www.youtube.com")); - } - if (headers.get("X-YouTube-Client-Name") == null) { - headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); - } + headers.computeIfAbsent("Origin", k -> Collections.singletonList( + "https://www.youtube.com")); + headers.computeIfAbsent("Referer", k -> Collections.singletonList( + "https://www.youtube.com")); + headers.computeIfAbsent("X-YouTube-Client-Name", k -> Collections.singletonList("1")); if (headers.get("X-YouTube-Client-Version") == null) { headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion())); } @@ -867,7 +877,7 @@ public class YoutubeParsingHelper { * @see #CONSENT_COOKIE * @param headers the headers which should be completed */ - public static void addCookieHeader(@Nonnull final Map> headers) { + public static void addCookieHeader(final Map> headers) { if (headers.get("Cookie") == null) { headers.put("Cookie", Arrays.asList(generateConsentCookie())); } else { @@ -1092,5 +1102,4 @@ public class YoutubeParsingHelper { .replaceAll("\\\\x5b", "[") .replaceAll("\\\\x5d", "]"); } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 96f7cac7..81e12c3f 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; @@ -84,7 +85,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final String[] channelInfo = channel_path.split("/"); String id = ""; // If the url is an URL which is not a /channel URL, we need to use the - // navigation/resolve_url endpoint of the youtubei API to get the channel id. Otherwise, we + // navigation/resolve_url endpoint of the internal API to get the channel id. Otherwise, we // couldn't get information about the channel associated with this URL, if there is one. if (!channelInfo[0].equals("channel")) { final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(), @@ -293,17 +294,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } @Override - public String getParentChannelName() throws ParsingException { + public String getParentChannelName() { return ""; } @Override - public String getParentChannelUrl() throws ParsingException { + public String getParentChannelUrl() { return ""; } @Override - public String getParentChannelAvatarUrl() throws ParsingException { + public String getParentChannelAvatarUrl() { return ""; } @@ -329,13 +330,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .getArray("contents").getObject(0).getObject("itemSectionRenderer") .getArray("contents").getObject(0).getObject("gridRenderer"); - final List channelInformations = new ArrayList<>(); - channelInformations.add(getName()); - channelInformations.add(getUrl()); + final List channelInfo = new ArrayList<>(); + channelInfo.add(getName()); + channelInfo.add(getUrl()); final JsonObject continuation = collectStreamsFrom(collector, gridRenderer - .getArray("items"), channelInformations); + .getArray("items"), channelInfo); - nextPage = getNextPageFrom(continuation, channelInformations); + nextPage = getNextPageFrom(continuation, channelInfo); } return new InfoItemsPage<>(collector, nextPage); @@ -348,7 +349,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { throw new IllegalArgumentException("Page doesn't contain an URL"); } - final List channelInformations = page.getIds(); + final List channelInfos = page.getIds(); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final Map> headers = new HashMap<>(); @@ -364,13 +365,14 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .getObject("appendContinuationItemsAction"); final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation - .getArray("continuationItems"), channelInformations); + .getArray("continuationItems"), channelInfos); - return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelInformations)); + return new InfoItemsPage<>(collector, getNextPageFrom(continuation, channelInfos)); } + @Nullable private Page getNextPageFrom(final JsonObject continuations, - final List channelInformations) throws IOException, + final List channelInfo) throws IOException, ExtractionException { if (isNullOrEmpty(continuations)) { return null; @@ -386,8 +388,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .done()) .getBytes(UTF_8); - return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelInformations, - null, body); + return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), null, channelInfo, null, body); } /** @@ -397,13 +398,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor { * @param videos the array to get videos from * @return the continuation object */ - private JsonObject collectStreamsFrom(final StreamInfoItemsCollector collector, - final JsonArray videos, - final List channelInformations) { + private JsonObject collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector, + @Nonnull final JsonArray videos, + @Nonnull final List channelInfo) { collector.reset(); - final String uploaderName = channelInformations.get(0); - final String uploaderUrl = channelInformations.get(1); + final String uploaderName = channelInfo.get(0); + final String uploaderUrl = channelInfo.get(1); final TimeAgoParser timeAgoParser = getTimeAgoParser(); JsonObject continuation = null; @@ -431,6 +432,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return continuation; } + @Nullable private JsonObject getVideoTab() throws ParsingException { if (this.videoTab != null) return this.videoTab; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 099c9fdd..4c16f8d0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -1,6 +1,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; +import com.grack.nanojson.JsonBuilder; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonWriter; @@ -61,31 +62,16 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final String videoId = getQueryValue(url, "v"); final String playlistIndexString = getQueryValue(url, "index"); - final byte[] body; + final JsonBuilder jsonBody = prepareJsonBuilder(localization, + getExtractorContentCountry()).value("playlistId", mixPlaylistId); if (videoId != null) { - if (playlistIndexString != null) { - body = JsonWriter.string(prepareJsonBuilder(localization, - getExtractorContentCountry()) - .value("videoId", videoId) - .value("playlistId", mixPlaylistId) - .value("playlistIndex", Integer.parseInt(playlistIndexString)) - .done()) - .getBytes(UTF_8); - } else { - body = JsonWriter.string(prepareJsonBuilder(localization, - getExtractorContentCountry()) - .value("videoId", videoId) - .value("playlistId", mixPlaylistId) - .done()) - .getBytes(UTF_8); - } - } else { - body = JsonWriter.string(prepareJsonBuilder(localization, - getExtractorContentCountry()) - .value("playlistId", mixPlaylistId) - .done()) - .getBytes(UTF_8); + jsonBody.value("videoId", videoId); } + if (playlistIndexString != null) { + jsonBody.value("playlistIndex", Integer.parseInt(playlistIndexString)); + } + + final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8); final Map> headers = new HashMap<>(); addClientInfoHeaders(headers); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java index da862bad..7aab9040 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java @@ -74,11 +74,10 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .object("client") .value("clientName", "WEB_REMIX") .value("clientVersion", youtubeMusicKeys[2]) - .value("hl", "en") + .value("hl", "en-GB") .value("gl", getExtractorContentCountry().getCountryCode()) .array("experimentIds").end() - .value("experimentsToken", "") - .value("utcOffsetMinutes", 0) + .value("experimentsToken", EMPTY_STRING) .object("locationInfo").end() .object("musicAppInfo").end() .end() @@ -89,6 +88,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor { .end() .object("activePlayers").end() .object("user") + // TO DO: provide a way to enable restricted mode with: .value("enableSafetyMode", false) .end() .end() diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 1248b6f3..be8a9f0b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -9,7 +9,6 @@ import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; -import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector; @@ -60,7 +59,6 @@ public class YoutubeSearchExtractor extends SearchExtractor { ExtractionException { final String query = super.getSearchString(); final Localization localization = getExtractorLocalization(); - final ContentCountry contentCountry = getExtractorContentCountry(); // Get the search parameter of the request final List contentFilters = super.getLinkHandler().getContentFilters(); @@ -72,20 +70,15 @@ public class YoutubeSearchExtractor extends SearchExtractor { params = ""; } - final byte[] body; + final JsonBuilder jsonBody = prepareJsonBuilder(localization, + getExtractorContentCountry()) + .value("query", query); if (!isNullOrEmpty(params)) { - body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry) - .value("query", query) - .value("params", params) - .done()) - .getBytes(UTF_8); - } else { - body = JsonWriter.string(prepareJsonBuilder(localization, contentCountry) - .value("query", query) - .done()) - .getBytes(UTF_8); + jsonBody.value("params", params); } + final byte[] body = JsonWriter.string(jsonBody.done()).getBytes(UTF_8); + initialData = getJsonPostResponse("search", body, localization); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 89bb1795..40db96ba 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -482,6 +482,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getDashMpdUrl() throws ParsingException { assertPageFetched(); + try { String dashManifestUrl; if (streamingData.isString("dashManifestUrl")) { @@ -643,7 +644,11 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public StreamType getStreamType() { assertPageFetched(); - return streamingData.has(FORMATS) ? StreamType.VIDEO_STREAM : StreamType.LIVE_STREAM; + + if (playerResponse.getObject("videoDetails").getBoolean("isLiveContent", false)) { + return StreamType.LIVE_STREAM; + } + return StreamType.VIDEO_STREAM; } @Nullable @@ -895,7 +900,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { try { // The JavaScript player was not found in any page fetched so far and there is // nothing cached, so try fetching embedded info. - // Don't provide a video id to get a smaller response (around 9kb instead of 21 kb + // Don't provide a video id to get a smaller response (around 9Kb instead of 21 Kb // with a video) final String embedUrl = "https://www.youtube.com/embed/"; final String embedPageContent = NewPipe.getDownloader() @@ -936,23 +941,16 @@ public class YoutubeStreamExtractor extends StreamExtractor { private boolean hasOtfStreams() { if (streamingData != null) { - boolean hasOtfStreamsValue = false; - if (streamingData.has("adaptiveFormats")) { - final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats"); - for (final Object adaptiveFormat : adaptiveFormats) { - final JsonObject jsonAdaptiveFormat = (JsonObject) adaptiveFormat; - if (jsonAdaptiveFormat.has("type")) { - final String streamTypeFormat = jsonAdaptiveFormat.getString("type", - EMPTY_STRING); - if (streamTypeFormat.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { - hasOtfStreamsValue = true; - break; - } - } + final JsonArray adaptiveFormats = streamingData.getArray("adaptiveFormats"); + for (final Object adaptiveFormat : adaptiveFormats) { + final JsonObject jsonAdaptiveFormat = (JsonObject) adaptiveFormat; + final String streamTypeFormat = jsonAdaptiveFormat.getString("type", EMPTY_STRING); + if (streamTypeFormat.equalsIgnoreCase("FORMAT_STREAM_TYPE_OTF")) { + return true; } } - return hasOtfStreamsValue; } + return false; } @@ -1123,9 +1121,9 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Nonnull private static String getVideoInfoUrl(final String id, final String sts) { // TODO: Try parsing embedded_player_response first - return "https://www.youtube.com/get_video_info?" + "video_id=" + id + - "&html5=1&eurl=https://youtube.googleapis.com/v/" + id + - "&sts=" + sts + "&ps=default&gl=US&hl=en"; + return "https://www.youtube.com/get_video_info?" + "video_id=" + id + + "&eurl=https://youtube.googleapis.com/v/" + id + "&sts=" + sts + + "&html5=1&c=TVHTML5&cver=6.20180913&hl=en&gl=US"; } @Nonnull @@ -1280,7 +1278,7 @@ public class YoutubeStreamExtractor extends StreamExtractor { @Override public String getCategory() { return playerResponse.getObject("microformat").getObject("playerMicroformatRenderer") - .getString("category"); + .getString("category", EMPTY_STRING); } @Nonnull diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java index 8a7b521f..e3145ec5 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java @@ -58,11 +58,8 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac @Override public int expectedAgeLimit() { return 18; } @Override public boolean expectedHasSubtitles() { return false; } -<<<<<<< HEAD @Override public String expectedCategory() { return ""; } // Unavailable on age restricted videos -======= - @Override public String expectedCategory() { return "Entertainment"; } ->>>>>>> d0dc7d69 (Update mocks, reenable a test and fix a test) + @Override public String expectedLicence() { return "YouTube licence"; } @Override public List expectedTags() {