[YouTube] Use new continuation API for channels

Co-authored-by: TiA4f8R <74829229+tia4f8r@users.noreply.github.com>
This commit is contained in:
TobiGr 2021-03-04 21:38:23 +01:00
parent 9256b3b848
commit 2b671b15ce
4 changed files with 65 additions and 28 deletions

View File

@ -2,10 +2,13 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException; import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
@ -15,13 +18,20 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem; import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import java.io.IOException; import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING; import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/* /*
@ -226,7 +236,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
@Nonnull @Nonnull
@Override @Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException { public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null; Page nextPage = null;
@ -254,18 +264,36 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
// as they don't deliver enough information on their own (the channel name, for example). // as they don't deliver enough information on their own (the channel name, for example).
fetchPage(); fetchPage();
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId()); // @formatter:off
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization()); byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
.value("continuation", page.getId())
.end()
.done()
.getBytes(UTF_8);
// @formatter:on
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response") StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
.getArray("onResponseReceivedActions").getObject(0).getObject("appendContinuationItemsAction"); final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems")); final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
return new InfoItemsPage<>(collector, getNextPageFrom(continuation)); return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
} }
private Page getNextPageFrom(final JsonObject continuations) { private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) { if (isNullOrEmpty(continuations)) {
return null; return null;
} }
@ -273,8 +301,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint"); final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token"); final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams"); final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams");
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams); continuation);
} }
/** /**

View File

@ -20,7 +20,7 @@
}, },
"response": { "response": {
"responseCode": 404, "responseCode": 404,
"responseMessage": "", "responseMessage": "Not Found",
"responseHeaders": { "responseHeaders": {
"alt-svc": [ "alt-svc": [
"h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\"" "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
@ -32,7 +32,7 @@
"text/html; charset\u003dutf-8" "text/html; charset\u003dutf-8"
], ],
"date": [ "date": [
"Sat, 13 Feb 2021 19:13:39 GMT" "Thu, 04 Mar 2021 14:15:28 GMT"
], ],
"expires": [ "expires": [
"Mon, 01 Jan 1990 00:00:00 GMT" "Mon, 01 Jan 1990 00:00:00 GMT"
@ -47,12 +47,15 @@
"ESF" "ESF"
], ],
"set-cookie": [ "set-cookie": [
"YSC\u003dbFLZyqvSsDQ; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone", "YSC\u003dOHweFRBSiuo; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003dYgNWpn3F6Ww; Domain\u003d.youtube.com; Expires\u003dThu, 12-Aug-2021 19:13:39 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone" "VISITOR_INFO1_LIVE\u003dDjGaxgk-6jk; Domain\u003d.youtube.com; Expires\u003dTue, 31-Aug-2021 14:15:28 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone"
], ],
"strict-transport-security": [ "strict-transport-security": [
"max-age\u003d31536000" "max-age\u003d31536000"
], ],
"transfer-encoding": [
"chunked"
],
"x-content-type-options": [ "x-content-type-options": [
"nosniff" "nosniff"
], ],
@ -63,7 +66,7 @@
"0" "0"
] ]
}, },
"responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"AkiRKVBSwx7zy61zR/BBpg\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e", "responseBody": "\u003chtml lang\u003d\"en-GB\" dir\u003d\"ltr\"\u003e\u003chead\u003e\u003ctitle\u003e404 Not Found\u003c/title\u003e\u003cstyle nonce\u003d\"H7HK+DtuIlOkI+rlUnUlnQ\"\u003e*{margin:0;padding:0;border:0}html,body{height:100%;}\u003c/style\u003e\u003clink rel\u003d\"shortcut icon\" href\u003d\"https://www.youtube.com/img/favicon.ico\" type\u003d\"image/x-icon\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_32.png\" sizes\u003d\"32x32\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_48.png\" sizes\u003d\"48x48\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_96.png\" sizes\u003d\"96x96\"\u003e\u003clink rel\u003d\"icon\" href\u003d\"https://www.youtube.com/img/favicon_144.png\" sizes\u003d\"144x144\"\u003e\u003c/head\u003e\u003cbody\u003e\u003ciframe style\u003d\"display:block;border:0;\" src\u003d\"/error?src\u003d404\u0026amp;ifr\u003d1\u0026amp;error\u003d\" width\u003d\"100%\" height\u003d\"100%\" frameborder\u003d\"\\\" scrolling\u003d\"no\"\u003e\u003c/iframe\u003e\u003c/body\u003e\u003c/html\u003e",
"latestUrl": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid" "latestUrl": "https://www.youtube.com/channel/DOESNT-EXIST/videos?pbj\u003d1\u0026view\u003d0\u0026flow\u003dgrid"
} }
} }