From 0f9e9b8b4b99b9a1f7b7d17742a2bab8c466c5d4 Mon Sep 17 00:00:00 2001 From: TiA4f8R <74829229+TiA4f8R@users.noreply.github.com> Date: Sun, 30 May 2021 17:23:51 +0200 Subject: [PATCH] Use the youtubei API for YouTube mixes + update the corresponding test + do some improvements Use the youtubei API for YouTube mixes. The corresponding has been updated because the new API breaks the tests of YoutubeMixPlaylistExtractorTest. Remove some deprecated code (the old search code with the pbj JSON) and do some other improvements. --- .../youtube/YoutubeParsingHelper.java | 3 +- .../extractors/YoutubeChannelExtractor.java | 20 +-- .../YoutubeMixPlaylistExtractor.java | 142 ++++++++++++------ .../extractors/YoutubePlaylistExtractor.java | 20 ++- .../extractors/YoutubeSearchExtractor.java | 85 ++++------- .../YoutubeMixPlaylistExtractorTest.java | 87 +++++++---- 6 files changed, 204 insertions(+), 153 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index 13843d86..978efd02 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -64,11 +64,12 @@ public class YoutubeParsingHelper { private YoutubeParsingHelper() { } + public static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/"; + private static final String HARDCODED_CLIENT_VERSION = "2.20210526.07.00"; private static final String HARDCODED_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"; private static final String[] MOBILE_YOUTUBE_KEYS = {"AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", "16.20.35"}; - private static final String YOUTUBEI_V1_URL = "https://www.youtube.com/youtubei/v1/"; private static String clientVersion; private static String key; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 626972b4..bfddb7f0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -24,15 +24,13 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -349,9 +347,13 @@ public class YoutubeChannelExtractor extends ChannelExtractor { // Unfortunately, we have to fetch the page even if we are only getting next streams, // as they don't deliver enough information on their own (the channel name, for example). - fetchPage(); - StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + if (!isPageFetched()) fetchPage(); + + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final Map> headers = new HashMap<>(); + addClientInfoHeaders(headers); + final Response response = getDownloader().post(page.getUrl(), null, page.getBody(), getExtractorLocalization()); @@ -383,7 +385,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .done()) .getBytes(UTF_8); - return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), body); + return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body); } /** diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java index 9fdc0ab4..099c9fdd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java @@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonWriter; import org.schabi.newpipe.extractor.ListExtractor; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.StreamingService; @@ -11,6 +12,7 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; import org.schabi.newpipe.extractor.playlist.PlaylistExtractor; import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper; @@ -19,19 +21,14 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.utils.JsonUtils; import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.net.URL; +import java.util.*; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.utils.Utils.*; /** * A {@link YoutubePlaylistExtractor} for a mix (auto-generated playlist). @@ -58,13 +55,51 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - final String url = getUrl() + "&pbj=1"; - final Response response = getResponse(url, getExtractorLocalization()); - final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody()); - initialData = ajaxJson.getObject(3).getObject("response"); + final Localization localization = getExtractorLocalization(); + final URL url = stringToURL(getUrl()); + final String mixPlaylistId = getId(); + final String videoId = getQueryValue(url, "v"); + final String playlistIndexString = getQueryValue(url, "index"); + + final byte[] body; + if (videoId != null) { + if (playlistIndexString != null) { + body = JsonWriter.string(prepareJsonBuilder(localization, + getExtractorContentCountry()) + .value("videoId", videoId) + .value("playlistId", mixPlaylistId) + .value("playlistIndex", Integer.parseInt(playlistIndexString)) + .done()) + .getBytes(UTF_8); + } else { + body = JsonWriter.string(prepareJsonBuilder(localization, + getExtractorContentCountry()) + .value("videoId", videoId) + .value("playlistId", mixPlaylistId) + .done()) + .getBytes(UTF_8); + } + } else { + body = JsonWriter.string(prepareJsonBuilder(localization, + getExtractorContentCountry()) + .value("playlistId", mixPlaylistId) + .done()) + .getBytes(UTF_8); + } + + final Map> headers = new HashMap<>(); + addClientInfoHeaders(headers); + + final Response response = getDownloader().post(YOUTUBEI_V1_URL + "next?key=" + getKey(), + headers, body, localization); + + initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults") .getObject("playlist").getObject("playlist"); + if (isNullOrEmpty(playlistData)) throw new ExtractionException( + "Could not get playlistData"); cookieValue = extractCookieValue(COOKIE_NAME, response); + } @Nonnull @@ -83,10 +118,9 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { return getThumbnailUrlFromPlaylistId(playlistData.getString("playlistId")); } catch (final Exception e) { try { - //fallback to thumbnail of current video. Always the case for channel mix - return getThumbnailUrlFromVideoId( - initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint") - .getString("videoId")); + // Fallback to thumbnail of current video. Always the case for channel mix + return getThumbnailUrlFromVideoId(initialData.getObject("currentVideoEndpoint") + .getObject("watchEndpoint").getString("videoId")); } catch (final Exception ignored) { } throw new ParsingException("Could not get playlist thumbnail", e); @@ -100,19 +134,19 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public String getUploaderUrl() { - //Youtube mix are auto-generated + // YouTube mixes are auto-generated by YouTube return ""; } @Override public String getUploaderName() { - //Youtube mix are auto-generated by YouTube + // YouTube mixes are auto-generated by YouTube return "YouTube"; } @Override public String getUploaderAvatarUrl() { - //Youtube mix are auto-generated by YouTube + // YouTube mixes are auto-generated by YouTube return ""; } @@ -123,64 +157,81 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { @Override public long getStreamCount() { - // Auto-generated playlist always start with 25 videos and are endless + // Auto-generated playlists always start with 25 videos and are endless return ListExtractor.ITEM_COUNT_INFINITE; } @Nonnull @Override - public InfoItemsPage getInitialPage() throws ExtractionException { + public InfoItemsPage getInitialPage() throws IOException, + ExtractionException { final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, playlistData.getArray("contents")); final Map cookies = new HashMap<>(); cookies.put(COOKIE_NAME, cookieValue); - return new InfoItemsPage<>(collector, new Page(getNextPageUrlFrom(playlistData), cookies)); + return new InfoItemsPage<>(collector, getNextPageFrom(playlistData, cookies)); } - private String getNextPageUrlFrom(final JsonObject playlistJson) throws ExtractionException { + private Page getNextPageFrom(final JsonObject playlistJson, + final Map cookies) throws IOException, + ExtractionException { final JsonObject lastStream = ((JsonObject) playlistJson.getArray("contents") .get(playlistJson.getArray("contents").size() - 1)); if (lastStream == null || lastStream.getObject("playlistPanelVideoRenderer") == null) { throw new ExtractionException("Could not extract next page url"); } - return getUrlFromNavigationEndpoint( - lastStream.getObject("playlistPanelVideoRenderer").getObject("navigationEndpoint")) - + "&pbj=1"; + final JsonObject watchEndpoint = lastStream.getObject("playlistPanelVideoRenderer") + .getObject("navigationEndpoint").getObject("watchEndpoint"); + final String playlistId = watchEndpoint.getString("playlistId"); + final String videoId = watchEndpoint.getString("videoId"); + final int index = watchEndpoint.getInt("index"); + final String params = watchEndpoint.getString("params"); + final byte[] body = JsonWriter.string(prepareJsonBuilder(getExtractorLocalization(), + getExtractorContentCountry()) + .value("videoId", videoId) + .value("playlistId", playlistId) + .value("playlistIndex", index) + .value("params", params) + .done()) + .getBytes(UTF_8); + + return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body); } @Override - public InfoItemsPage getPage(final Page page) - throws ExtractionException, IOException { + public InfoItemsPage getPage(final Page page) throws IOException, + ExtractionException { if (page == null || isNullOrEmpty(page.getUrl())) { - throw new IllegalArgumentException("Page url is empty or null"); + throw new IllegalArgumentException("Page doesn't contain an URL"); } if (!page.getCookies().containsKey(COOKIE_NAME)) { - throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing"); + throw new IllegalArgumentException("Cookie '" + COOKIE_NAME + "' is missing"); } - final JsonArray ajaxJson = getJsonResponse(page, getExtractorLocalization()); - final JsonObject playlistJson = - ajaxJson.getObject(3).getObject("response").getObject("contents") - .getObject("twoColumnWatchNextResults").getObject("playlist") - .getObject("playlist"); + final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final Map> headers = new HashMap<>(); + addClientInfoHeaders(headers); + + final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), + getExtractorLocalization()); + final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); + final JsonObject playlistJson = ajaxJson.getObject("contents") + .getObject("twoColumnWatchNextResults").getObject("playlist").getObject("playlist"); final JsonArray allStreams = playlistJson.getArray("contents"); - // Sublist because youtube returns up to 24 previous streams in the mix + // Sublist because YouTube returns up to 24 previous streams in the mix // +1 because the stream of "currentIndex" was already extracted in previous request final List newStreams = allStreams.subList(playlistJson.getInt("currentIndex") + 1, allStreams.size()); - final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); collectStreamsFrom(collector, newStreams); - return new InfoItemsPage<>(collector, - new Page(getNextPageUrlFrom(playlistJson), page.getCookies())); + return new InfoItemsPage<>(collector, getNextPageFrom(playlistJson, page.getCookies())); } - private void collectStreamsFrom( - @Nonnull final StreamInfoItemsCollector collector, - @Nullable final List streams) { + private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector, + @Nullable final List streams) { if (streams == null) { return; @@ -193,7 +244,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { final JsonObject streamInfo = ((JsonObject) stream) .getObject("playlistPanelVideoRenderer"); if (streamInfo != null) { - collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser)); + collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, + timeAgoParser)); } } } @@ -204,7 +256,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor { if (playlistId.startsWith("RDMM")) { videoId = playlistId.substring(4); } else if (playlistId.startsWith("RDCMUC")) { - throw new ParsingException("is channel mix"); + throw new ParsingException("This playlist is a channel mix"); } else { videoId = playlistId.substring(2); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java index 9d65e1b2..be58b734 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java @@ -21,16 +21,13 @@ import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareJsonBuilder; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -224,14 +221,15 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { } final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); + final Map> headers = new HashMap<>(); + addClientInfoHeaders(headers); - final Response response = getDownloader().post(page.getUrl(), null, page.getBody(), + final Response response = getDownloader().post(page.getUrl(), headers, page.getBody(), getExtractorLocalization()); final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response)); final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions") - .getObject(0) - .getObject("appendContinuationItemsAction") + .getObject(0).getObject("appendContinuationItemsAction") .getArray("continuationItems"); collectStreamsFrom(collector, continuation); @@ -259,7 +257,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor { .done()) .getBytes(UTF_8); - return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(), body); + return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey(), body); } else { return null; } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java index 278a2ef9..1248b6f3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java @@ -153,10 +153,8 @@ public class YoutubeSearchExtractor extends SearchExtractor { .getObject("itemSectionRenderer"); collectStreamsFrom(collector, itemSectionRenderer.getArray("contents")); - - nextPage = getNextPageFrom(itemSectionRenderer.getArray("continuations")); } else if (((JsonObject) section).has("continuationItemRenderer")) { - nextPage = getNewNextPageFrom(((JsonObject) section) + nextPage = getNextPageFrom(((JsonObject) section) .getObject("continuationItemRenderer")); } } @@ -174,46 +172,34 @@ public class YoutubeSearchExtractor extends SearchExtractor { final Localization localization = getExtractorLocalization(); final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId()); - if (page.getId() == null) { - final JsonArray ajaxJson = getJsonResponse(page.getUrl(), localization); + // @formatter:off + final byte[] json = JsonWriter.string(prepareJsonBuilder(localization, + getExtractorContentCountry()) + .value("continuation", page.getId()) + .done()) + .getBytes(UTF_8); + // @formatter:on - final JsonObject itemSectionContinuation = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("itemSectionContinuation"); + final String responseBody = getValidJsonResponseBody(getDownloader().post( + page.getUrl(), new HashMap<>(), json)); - collectStreamsFrom(collector, itemSectionContinuation.getArray("contents")); - final JsonArray continuations = itemSectionContinuation.getArray("continuations"); - - return new InfoItemsPage<>(collector, getNextPageFrom(continuations)); - } else { - // @formatter:off - final byte[] json = JsonWriter.string(prepareJsonBuilder(localization, - getExtractorContentCountry()) - .value("continuation", page.getId()) - .done()) - .getBytes(UTF_8); - // @formatter:on - - final String responseBody = getValidJsonResponseBody(getDownloader().post( - page.getUrl(), new HashMap<>(), json)); - - final JsonObject ajaxJson; - try { - ajaxJson = JsonParser.object().from(responseBody); - } catch (JsonParserException e) { - throw new ParsingException("Could not parse JSON", e); - } - - final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands") - .getObject(0).getObject("appendContinuationItemsAction") - .getArray("continuationItems"); - - final JsonArray contents = continuationItems.getObject(0) - .getObject("itemSectionRenderer").getArray("contents"); - collectStreamsFrom(collector, contents); - - return new InfoItemsPage<>(collector, getNewNextPageFrom(continuationItems.getObject(1) - .getObject("continuationItemRenderer"))); + final JsonObject ajaxJson; + try { + ajaxJson = JsonParser.object().from(responseBody); + } catch (JsonParserException e) { + throw new ParsingException("Could not parse JSON", e); } + + final JsonArray continuationItems = ajaxJson.getArray("onResponseReceivedCommands") + .getObject(0).getObject("appendContinuationItemsAction") + .getArray("continuationItems"); + + final JsonArray contents = continuationItems.getObject(0) + .getObject("itemSectionRenderer").getArray("contents"); + collectStreamsFrom(collector, contents); + + return new InfoItemsPage<>(collector, getNextPageFrom(continuationItems.getObject(1) + .getObject("continuationItemRenderer"))); } private void collectStreamsFrom(final InfoItemsSearchCollector collector, @@ -239,22 +225,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { } } - private Page getNextPageFrom(final JsonArray continuations) throws ParsingException { - if (isNullOrEmpty(continuations)) { - return null; - } - - final JsonObject nextContinuationData = continuations.getObject(0) - .getObject("nextContinuationData"); - final String continuation = nextContinuationData.getString("continuation"); - final String clickTrackingParams = nextContinuationData - .getString("clickTrackingParams"); - - return new Page(getUrl() + "&pbj=1&ctoken=" + continuation + "&continuation=" - + continuation + "&itct=" + clickTrackingParams); - } - - private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, + private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException { if (isNullOrEmpty(continuationItemRenderer)) { return null; @@ -263,7 +234,7 @@ public class YoutubeSearchExtractor extends SearchExtractor { final String token = continuationItemRenderer.getObject("continuationEndpoint") .getObject("continuationCommand").getString("token"); - final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey(); + final String url = YOUTUBEI_V1_URL + "search?key=" + getKey(); return new Page(url, token); } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java index 7658d6bb..5c2aaabc 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMixPlaylistExtractorTest.java @@ -1,8 +1,8 @@ package org.schabi.newpipe.extractor.services.youtube; +import com.grack.nanojson.JsonWriter; import org.hamcrest.MatcherAssert; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -32,12 +32,13 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ServiceList.YouTube; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; @RunWith(Suite.class) @SuiteClasses({Mix.class, MixWithIndex.class, MyMix.class, Invalid.class, ChannelMix.class}) public class YoutubeMixPlaylistExtractorTest { - public static final String PBJ = "&pbj=1"; private static final String VIDEO_ID = "_AzeUSL9lZc"; private static final String VIDEO_TITLE = "Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO"; @@ -55,8 +56,8 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mix")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); + .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID + + "&list=RD" + VIDEO_ID); extractor.fetchPage(); } @@ -89,9 +90,16 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { - final InfoItemsPage streams = extractor.getPage( - new Page("https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID - + PBJ, dummyCookie)); + final byte[] body = JsonWriter.string(prepareJsonBuilder( + NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry()) + .value("videoId", VIDEO_ID) + .value("playlistId", "RD" + VIDEO_ID) + .value("params", "OAE%3D") + .done()) + .getBytes(UTF_8); + + final InfoItemsPage streams = extractor.getPage(new Page( + YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -127,7 +135,7 @@ public class YoutubeMixPlaylistExtractorTest { @Ignore public static class MixWithIndex { - private static final String INDEX = "&index=13"; + private static final int INDEX = 13; private static final String VIDEO_ID_NUMBER_13 = "qHtzO49SDmk"; @BeforeClass @@ -137,9 +145,8 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" - + VIDEO_ID + INDEX); + .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + + "&list=RD" + VIDEO_ID + "&index=" + INDEX); extractor.fetchPage(); } @@ -167,9 +174,17 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { - final InfoItemsPage streams = extractor.getPage( - new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD" - + VIDEO_ID + INDEX + PBJ, dummyCookie)); + final byte[] body = JsonWriter.string(prepareJsonBuilder( + NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry()) + .value("videoId", VIDEO_ID) + .value("playlistId", "RD" + VIDEO_ID) + .value("playlistIndex", INDEX) + .value("params", "OAE%3D") + .done()) + .getBytes(UTF_8); + + final InfoItemsPage streams = extractor.getPage(new Page( + YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -210,9 +225,8 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM" - + VIDEO_ID); + .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID + + "&list=RDMM" + VIDEO_ID); extractor.fetchPage(); } @@ -243,9 +257,16 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { - final InfoItemsPage streams = - extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID - + "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie)); + final byte[] body = JsonWriter.string(prepareJsonBuilder( + NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry()) + .value("videoId", VIDEO_ID) + .value("playlistId", "RDMM" + VIDEO_ID) + .value("params", "OAE%3D") + .done()) + .getBytes(UTF_8); + + final InfoItemsPage streams = extractor.getPage(new Page( + YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); } @@ -291,8 +312,8 @@ public class YoutubeMixPlaylistExtractorTest { @Test(expected = IllegalArgumentException.class) public void getPageEmptyUrl() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RD" + VIDEO_ID); + .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID + + "&list=RD" + VIDEO_ID); extractor.fetchPage(); extractor.getPage(new Page("")); } @@ -300,8 +321,8 @@ public class YoutubeMixPlaylistExtractorTest { @Test(expected = ExtractionException.class) public void invalidVideoId() throws Exception { extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde"); + .getPlaylistExtractor("https://www.youtube.com/watch?v=" + "abcde" + + "&list=RD" + "abcde"); extractor.fetchPage(); extractor.getName(); } @@ -321,9 +342,8 @@ public class YoutubeMixPlaylistExtractorTest { NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix")); dummyCookie.put(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever"); extractor = (YoutubeMixPlaylistExtractor) YouTube - .getPlaylistExtractor( - "https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL - + "&list=RDCM" + CHANNEL_ID); + .getPlaylistExtractor("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL + + "&list=RDCM" + CHANNEL_ID); extractor.fetchPage(); } @@ -350,9 +370,16 @@ public class YoutubeMixPlaylistExtractorTest { @Test public void getPage() throws Exception { - final InfoItemsPage streams = extractor.getPage( - new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL - + "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie)); + final byte[] body = JsonWriter.string(prepareJsonBuilder( + NewPipe.getPreferredLocalization(), NewPipe.getPreferredContentCountry()) + .value("videoId", VIDEO_ID_OF_CHANNEL) + .value("playlistId", "RDCM" + CHANNEL_ID) + .value("params", "OAE%3D") + .done()) + .getBytes(UTF_8); + + final InfoItemsPage streams = extractor.getPage(new Page( + YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, dummyCookie, body)); assertFalse(streams.getItems().isEmpty()); assertTrue(streams.hasNextPage()); }