From b0f356dd76e89346e968a043aefcafc614fbd297 Mon Sep 17 00:00:00 2001 From: bopol Date: Fri, 12 Feb 2021 14:39:09 +0100 Subject: [PATCH 1/2] YT channels: fix continuation --- .../extractors/YoutubeChannelExtractor.java | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java index 338f2294..1bc4f809 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java @@ -20,9 +20,7 @@ import org.schabi.newpipe.extractor.utils.Utils; import javax.annotation.Nonnull; import java.io.IOException; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; -import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; +import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; @@ -230,9 +228,9 @@ public class YoutubeChannelExtractor extends ChannelExtractor { .getArray("contents").getObject(0).getObject("itemSectionRenderer") .getArray("contents").getObject(0).getObject("gridRenderer"); - collectStreamsFrom(collector, gridRenderer.getArray("items")); + final JsonObject continuation = collectStreamsFrom(collector, gridRenderer.getArray("items")); - nextPage = getNextPageFrom(gridRenderer.getArray("continuations")); + nextPage = getNextPageFrom(continuation); } return new InfoItemsPage<>(collector, nextPage); @@ -252,36 +250,47 @@ public class YoutubeChannelExtractor extends ChannelExtractor { final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization()); JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") - .getObject("continuationContents").getObject("gridContinuation"); + .getArray("onResponseReceivedActions").getObject(0).getObject("appendContinuationItemsAction"); - collectStreamsFrom(collector, sectionListContinuation.getArray("items")); + final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems")); - return new InfoItemsPage<>(collector, getNextPageFrom(sectionListContinuation.getArray("continuations"))); + return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); } - private Page getNextPageFrom(final JsonArray continuations) { + private Page getNextPageFrom(final JsonObject continuations) { if (isNullOrEmpty(continuations)) { return null; } - final JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData"); - final String continuation = nextContinuationData.getString("continuation"); - final String clickTrackingParams = nextContinuationData.getString("clickTrackingParams"); + final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint"); + final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token"); + final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams"); return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation + "&continuation=" + continuation + "&itct=" + clickTrackingParams); } - private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException { + /** + * Collect streams from an array of items + * + * @param collector the collector where videos will be commited + * @param videos the array to get videos from + * @return the continuation object + * @throws ParsingException if an error happened while extracting + */ + private JsonObject collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray videos) throws ParsingException { collector.reset(); final String uploaderName = getName(); final String uploaderUrl = getUrl(); final TimeAgoParser timeAgoParser = getTimeAgoParser(); - for (Object video : videos) { - if (((JsonObject) video).has("gridVideoRenderer")) { + JsonObject continuation = null; + + for (Object object : videos) { + final JsonObject video = (JsonObject) object; + if (video.has("gridVideoRenderer")) { collector.commit(new YoutubeStreamInfoItemExtractor( - ((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) { + video.getObject("gridVideoRenderer"), timeAgoParser) { @Override public String getUploaderName() { return uploaderName; @@ -292,8 +301,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor { return uploaderUrl; } }); + } else if (video.has("continuationItemRenderer")) { + continuation = video.getObject("continuationItemRenderer"); } } + + return continuation; } private JsonObject getVideoTab() throws ParsingException { From f8d769617ada7a48f12a34280c8171dd5cfce70b Mon Sep 17 00:00:00 2001 From: TobiGr Date: Fri, 12 Feb 2021 22:14:23 +0100 Subject: [PATCH 2/2] Release 0.20.9 --- README.md | 2 +- build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 36e33583..4aea451c 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo. If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps: 1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`. -2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.8'`the `dependencies` in your `build.gradle`. Replace `v0.20.8` with the latest release. +2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.9'`the `dependencies` in your `build.gradle`. Replace `v0.20.9` with the latest release. **Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. diff --git a/build.gradle b/build.gradle index a62ca0d0..7e92443d 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ allprojects { sourceCompatibility = 1.8 targetCompatibility = 1.8 - version 'v0.20.8' + version 'v0.20.9' group 'com.github.TeamNewPipe' repositories {