From 5794eb2350aba4c01e1c3962eb5e1a92d0f12a72 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Thu, 8 Apr 2021 16:17:59 +0200 Subject: [PATCH] Use the youtubei API for YouTube playlists --- .../youtube/YoutubeParsingHelper.java | 53 ++++++++++++++----- .../extractors/YoutubeChannelExtractor.java | 16 +++--- .../extractors/YoutubePlaylistExtractor.java | 17 +++--- 3 files changed, 56 insertions(+), 30 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 1029ddef..ee59e70d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -95,7 +95,7 @@ public class YoutubeParsingHelper { final String host = u.getHost(); return host.startsWith("google.") || host.startsWith("m.google.") || host.startsWith("www.google."); - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { return false; } } @@ -207,10 +207,10 @@ public class YoutubeParsingHelper { public static OffsetDateTime parseDateFrom(final String textualUploadDate) throws ParsingException { try { return OffsetDateTime.parse(textualUploadDate); - } catch (DateTimeParseException e) { + } catch (final DateTimeParseException e) { try { return LocalDate.parse(textualUploadDate).atStartOfDay().atOffset(ZoneOffset.UTC); - } catch (DateTimeParseException e1) { + } catch (final DateTimeParseException e1) { throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e1); } } @@ -277,11 +277,11 @@ public class YoutubeParsingHelper { try { final String initialData = Parser.matchGroup1("window\\[\"ytInitialData\"\\]\\s*=\\s*(\\{.*?\\});", html); return JsonParser.object().from(initialData); - } catch (Parser.RegexException e) { + } catch (final Parser.RegexException e) { final String initialData = Parser.matchGroup1("var\\s*ytInitialData\\s*=\\s*(\\{.*?\\});", html); return JsonParser.object().from(initialData); } - } catch (JsonParserException | Parser.RegexException e) { + } catch (final JsonParserException | Parser.RegexException e) { throw new ParsingException("Could not get ytInitialData", e); } } @@ -342,7 +342,7 @@ public class YoutubeParsingHelper { clientVersion = contextClientVersion; break; } - } catch (Parser.RegexException ignored) { + } catch (final Parser.RegexException ignored) { } } @@ -352,10 +352,10 @@ public class YoutubeParsingHelper { try { key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); - } catch (Parser.RegexException e) { + } catch (final Parser.RegexException e) { try { key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html); - } catch (Parser.RegexException ignored) { + } catch (final Parser.RegexException ignored) { } } } @@ -468,7 +468,7 @@ public class YoutubeParsingHelper { String key; try { key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html); - } catch (Parser.RegexException e) { + } catch (final Parser.RegexException e) { key = Parser.matchGroup1("innertube_api_key\":\"([0-9a-zA-Z_-]+?)\"", html); } @@ -477,10 +477,10 @@ public class YoutubeParsingHelper { String clientVersion; try { clientVersion = Parser.matchGroup1("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); - } catch (Parser.RegexException e) { + } catch (final Parser.RegexException e) { try { clientVersion = Parser.matchGroup1("INNERTUBE_CLIENT_VERSION\":\"([0-9\\.]+?)\"", html); - } catch (Parser.RegexException ee) { + } catch (final Parser.RegexException ee) { clientVersion = Parser.matchGroup1("innertube_context_client_version\":\"([0-9\\.]+?)\"", html); } } @@ -491,7 +491,7 @@ public class YoutubeParsingHelper { @Nullable - public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint) throws ParsingException { + public static String getUrlFromNavigationEndpoint(final JsonObject navigationEndpoint) throws ParsingException { if (navigationEndpoint.has("urlEndpoint")) { String internUrl = navigationEndpoint.getObject("urlEndpoint").getString("url"); if (internUrl.startsWith("https://www.youtube.com/redirect?")) { @@ -508,7 +508,7 @@ public class YoutubeParsingHelper { String url; try { url = URLDecoder.decode(param.split("=")[1], UTF_8); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { return null; } return url; @@ -662,6 +662,31 @@ public class YoutubeParsingHelper { return response; } + public static String extractCookieValue(final String cookieName, final Response response) { + final List cookies = response.responseHeaders().get("set-cookie"); + int startIndex; + String result = ""; + for (final String cookie : cookies) { + startIndex = cookie.indexOf(cookieName); + if (startIndex != -1) { + result = cookie.substring(startIndex + cookieName.length() + "=".length(), + cookie.indexOf(";", startIndex)); + } + } + return result; + } + + public static JsonObject getJsonPostResponse(final String endpoint, + final byte[] body, + final Localization localization) + throws IOException, ExtractionException { + + final Response response = getDownloader().post("https://youtubei.googleapis.com/youtubei/v1/" + + endpoint + "?key=" + getKey(), new HashMap<>(), body, localization); + + return JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + } + public static JsonArray getJsonResponse(final String url, final Localization localization) throws IOException, ExtractionException { Map> headers = new HashMap<>(); @@ -885,7 +910,7 @@ public class YoutubeParsingHelper { metaInfo.addUrl(new URL(url)); final String description = getTextFromObject(clarificationRenderer.getObject("secondarySource")); metaInfo.addUrlText(description == null ? url : description); - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { throw new ParsingException("Could not get metadata info secondary URL", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index dd41d2ff..c331b5d5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -147,7 +147,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public String getName() throws ParsingException { try { return initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getString("title"); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get channel name", e); } } @@ -159,7 +159,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .getArray("thumbnails").getObject(0).getString("url"); return fixThumbnailUrl(url); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get avatar", e); } } @@ -175,7 +175,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { } return fixThumbnailUrl(url); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get banner", e); } } @@ -184,7 +184,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public String getFeedUrl() throws ParsingException { try { return YoutubeParsingHelper.getFeedUrlFrom(getId()); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get feed url", e); } } @@ -207,7 +207,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { public String getDescription() throws ParsingException { try { return initialData.getObject("metadata").getObject("channelMetadataRenderer").getString("description"); - } catch (Exception e) { + } catch (final Exception e) { throw new ParsingException("Could not get channel description", e); } } @@ -293,7 +293,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .done()) .getBytes(UTF_8); - return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), + return new Page("https://youtubei.googleapis.com/youtubei/v1/browse?key=" + getKey(), body); } @@ -314,7 +314,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { JsonObject continuation = null; - for (Object object : videos) { + for (final Object object : videos) { final JsonObject video = (JsonObject) object; if (video.has("gridVideoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor( @@ -344,7 +344,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .getArray("tabs"); JsonObject videoTab = null; - for (Object tab : tabs) { + for (final Object tab : tabs) { if (((JsonObject) tab).has("tabRenderer")) { if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) { videoTab = ((JsonObject) tab).getObject("tabRenderer"); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index d6ccc13d..9dfe92c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -29,7 +29,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; @@ -50,11 +50,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + "&pbj=1"; + final byte[] body = JsonWriter.string(prepareJsonBuilder() + .value("browseId", "VL" + getId()) + .value("params", "wgYCCAA%3D") // show unavailable videos + .done()) + .getBytes(UTF_8); - initialAjaxJson = getJsonResponse(url, getExtractorLocalization()); - - initialData = initialAjaxJson.getObject(1).getObject("response"); + initialData = getJsonPostResponse("browse", body, getExtractorLocalization()); YoutubeParsingHelper.defaultAlertsCheck(initialData); playlistInfo = getPlaylistInfo(); @@ -251,9 +253,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .done()) .getBytes(UTF_8); - return new Page( - "https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), - body); + return new Page("https://youtubei.googleapis.com/youtubei/v1/browse?key=" + + getKey(), body); } else { return null; }