Merge pull request #665 from B0pol/agerestricted

[youtube] Fix extraction of age restricted videos
This commit is contained in:
bopol 2021-06-23 21:07:14 +02:00 committed by GitHub
commit 3966178979
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 283 additions and 740 deletions

View File

@ -16,15 +16,8 @@ import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
import org.schabi.newpipe.extractor.exceptions.PaidContentException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.*;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.localization.Localization;
@ -792,11 +785,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl();
final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
final String infoPageResponse = NewPipe.getDownloader()
.get(videoInfoUrl, getExtractorLocalization()).responseBody();
videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
final String videoInfoUrl = getVideoInfoUrl(getId());
final Response videoInfoResponse = NewPipe.getDownloader().get(videoInfoUrl, getExtractorLocalization());
videoInfoPage.putAll(Parser.compatParseMap(videoInfoResponse.responseBody()));
try {
playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
@ -975,11 +966,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
@Nonnull
private static String getVideoInfoUrl(final String id, final String sts) {
private static String getVideoInfoUrl(final String id) {
// TODO: Try parsing embedded_player_response first
return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
"&html5=1&eurl=https://youtube.googleapis.com/v/" + id +
"&sts=" + sts + "&ps=default&gl=US&hl=en";
return "https://www.youtube.com/get_video_info?video_id=" + id +
"&eurl=https://youtube.googleapis.com/v/" + id +
"&html5=1&c=TVHTML5&cver=6.20180913&gl=US&hl=en";
}
private Map<String, ItagItem> getItags(final String streamingDataKey,
@ -1121,7 +1112,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public String getCategory() {
return playerResponse.getObject("microformat")
.getObject("playerMicroformatRenderer")
.getString("category");
.getString("category", EMPTY_STRING);
}
@Nonnull

View File

@ -57,7 +57,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
@Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
@Override public boolean expectedHasSubtitles() { return false; }
@Override public String expectedCategory() {return "Entertainment"; }
@Override public String expectedCategory() { return ""; } // Unavailable on age restricted videos
@Override public String expectedLicence() { return "YouTube licence"; }
@Override
public List<String> expectedTags() {

View File

@ -26,7 +26,7 @@
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"no-cache, no-store, max-age\u003d0, must-revalidate"
@ -38,7 +38,7 @@
"application/json; charset\u003dutf-8"
],
"date": [
"Sat, 22 May 2021 18:30:38 GMT"
"Wed, 23 Jun 2021 10:12:06 GMT"
],
"expires": [
"Mon, 01 Jan 1990 00:00:00 GMT"
@ -56,8 +56,8 @@
"ESF"
],
"set-cookie": [
"YSC\u003dbWHlSChZfJE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+805; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
"YSC\u003d-LT8a9maoCU; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+142; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
@ -75,7 +75,7 @@
"0"
]
},
"responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r5---sn-4g5ednly.googlevideo.com\\/generate_204\",\"https:\\/\\/r5---sn-4g5ednly.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24037794,24040786,24044575,23884386,23944779,23983296,23882685,24014268,24027649,23995927,23735347,24043867,24032296,24011649,23975059,23891344,23968099,23970529,23966208,24005207,24042868,24005646,24019954,23986021,23940237,23891346,24037806,24003105,23996830,1714251,24005802,23804281,24030040,23946420,23744176,24035514,23890959,23918597,24007246,23974595,24045350,23934970,24001373,24036086,23968386,23997375,24042653,24027133,23857949,24004644,24035275,24024949,24036899,24042802\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0xd8a06d19cc578a00\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210324\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwi_h9z79d3wAhXw1xEIHUqjCfI\u003d\"}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgswTjM5MHFXN0hlayjOm6WFBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbmQyWVlOb0RMYzZ2YXVzdmhjWDZpRjdLQUJMQXxBQ3Jtc0trS2FPZGRQUzJZNFhoTU5EcnZjQjZFV0dMYnE0ZkNxYUNkeVlLd3N4MU9fRGpLYjRzdkliTVE4N3prNVByTVl2Mi04aFpZcHdvckhGSDI2U1Ewa24tbWFHNjNMa1pBb2lET0JSQW0xTUY4WklUOEtTcw\\u003d\\u003d\",\"url\": \"/watch?v\\u003dINVALID_ID_\",\"endpoint\": {\"clickTrackingParams\":\"IhMIobfb-_Xd8AIVzWjgCh0aDgYWMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dINVALID_ID_\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"INVALID_ID_\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\": 0.0 }}}]\r\n",
"responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r5---sn-hgn7yn7z.googlevideo.com\\/generate_204\",\"https:\\/\\/r5---sn-hgn7yn7z.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24001373,23946420,23970529,24049577,24042868,23966208,23804281,24027649,23998056,24049575,24036947,24004644,24017848,23891344,24007246,24037794,24052246,23963745,23934970,23996830,24030040,23983296,23891346,23884386,24037806,24042870,24012513,24058813,23857948,24053867,24049569,23968386,24049820,1714249,23973490,24049567,23918597,24003105,24049573,23744176,23974595,24058380,24003103,23882685,23944779,24038425,23890959,23999405\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0x58da0be5a82f9e65\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210526\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwiG75Ocwq3xAhU_NPEFHUM9A70\u003d\",\"frameworkUpdates\":{\"entityBatchUpdate\":{\"mutations\":[{\"entityKey\":\"Eg0KC0lOVkFMSURfSURfIPYBKAE%3D\",\"type\":\"ENTITY_MUTATION_TYPE_REPLACE\",\"payload\":{\"offlineabilityEntity\":{\"key\":\"Eg0KC0lOVkFMSURfSURfIPYBKAE%3D\",\"accessState\":\"OFFLINEABILITY_FEATURE_ACCESS_STATE_UNKNOWN\"}}}],\"timestamp\":{\"seconds\":\"1624443126\",\"nanos\":616071922}}}}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"Cgs1SUlhQUVDenVSUSj2kcyGBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbkdDQldjVWFhaHN4ZUQzTjMtY01fU1FpVElFd3xBQ3Jtc0tseTZPT0J2ZzA2eDNuYXRMREtXaGN0RWJsUWJ5eW1vWDQ4aGg5TDJHTjZvX3lSM2tRamh2M3lDdDNXc3haZE00TUJveWNZa2JBbWdYX21URUI2dTM2MTlfX0pIcWU1RVNXRHpreExHeHNfaW5LQjdvQQ\\u003d\\u003d\",\"url\": \"/watch?v\\u003dINVALID_ID_\",\"endpoint\": {\"clickTrackingParams\":\"IhMIpImTnMKt8QIV9TvxBR0TrgLLMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dINVALID_ID_\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"INVALID_ID_\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\": 0.0 }}}]\r\n",
"latestUrl": "https://www.youtube.com/watch?v\u003dINVALID_ID_\u0026pbj\u003d1"
}
}

View File

@ -1,81 +0,0 @@
{
"request": {
"httpMethod": "GET",
"url": "https://www.youtube.com/watch?v\u003ddon-t-exist\u0026pbj\u003d1",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
],
"Cookie": [
"CONSENT\u003dPENDING+100854"
],
"X-YouTube-Client-Name": [
"1"
],
"X-YouTube-Client-Version": [
"2.20200214.04.00"
]
},
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 200,
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"no-cache, no-store, max-age\u003d0, must-revalidate"
],
"content-disposition": [
"attachment"
],
"content-type": [
"application/json; charset\u003dutf-8"
],
"date": [
"Sat, 22 May 2021 18:30:40 GMT"
],
"expires": [
"Mon, 01 Jan 1990 00:00:00 GMT"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
],
"permissions-policy": [
"ch-ua-full-version\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*, ch-ua-arch\u003d*, ch-ua-model\u003d*"
],
"pragma": [
"no-cache"
],
"server": [
"ESF"
],
"set-cookie": [
"YSC\u003dlWKLRR9UNg0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+495; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-spf-response-type": [
"multipart"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r3---sn-4g5e6nlk.googlevideo.com\\/generate_204\",\"https:\\/\\/r3---sn-4g5e6nlk.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24036085,23891344,23804281,23997375,23946420,24001373,24039105,24004644,24042868,23891346,23966208,24030040,9479112,23970529,23974595,24043867,23968099,24037806,23884386,23940238,23882502,24027649,23934970,24037794,23857950,24007246,24022729,23918597,23986023,24035164,24044568,24040786,1714242,24005802,24046663,23944779,24042802,23983296,9406121,24045792,24014268,24005646,24027133,24042860,23968386,23939457,24035275,23996830,24043529,23890959,23984883,24010465,24019954,23744176,24045810,24026152,24036004,24044576,23995927,23990877,23868992\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0x899d9973680ec201\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210324\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwj_zob99d3wAhXBrnsKHS0SBPI\u003d\"}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgtuN1hhZ1RER2c2WSjQm6WFBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbHE4OV9DcUszaUJyQU5QNnNOSWowczFaMVJad3xBQ3Jtc0ttb1NRdjVEQXdGQ1BhaE5CLUxpNEZMZkRuWnBrU2tHWk92elZ3VVFsTlM4YWYweEFRSDB5SVllSVBfTVZrRzQwOFlDclF1b1hXOVA2TF9UV3A2eDVLRTBtY0VyRktQM0hCdldqT1hvOTJJcnNJZkFIZw\\u003d\\u003d\",\"url\": \"/watch?v\\u003ddon-t-exist\",\"endpoint\": {\"clickTrackingParams\":\"IhMIvfaF_fXd8AIVbcURCB0PPgimMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003ddon-t-exist\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"don-t-exist\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\": 0.0 }}}]\r\n",
"latestUrl": "https://www.youtube.com/watch?v\u003ddon-t-exist\u0026pbj\u003d1"
}
}