Merge pull request #665 from B0pol/agerestricted

[youtube] Fix extraction of age restricted videos

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -16,15 +16,8 @@ import org.schabi.newpipe.extractor.MetaInfo;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.downloader.Downloader;
-import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
+import org.schabi.newpipe.extractor.downloader.Response;
-import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+import org.schabi.newpipe.extractor.exceptions.*;
-import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
-import org.schabi.newpipe.extractor.exceptions.PaidContentException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
-import org.schabi.newpipe.extractor.exceptions.PrivateContentException;
-import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
-import org.schabi.newpipe.extractor.exceptions.YoutubeMusicPremiumContentException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
 import org.schabi.newpipe.extractor.localization.DateWrapper;
 import org.schabi.newpipe.extractor.localization.Localization;
@@ -792,11 +785,9 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, IOException {
-        final String sts = getEmbeddedInfoStsAndStorePlayerJsUrl();
+        final String videoInfoUrl = getVideoInfoUrl(getId());
-        final String videoInfoUrl = getVideoInfoUrl(getId(), sts);
+        final Response videoInfoResponse = NewPipe.getDownloader().get(videoInfoUrl, getExtractorLocalization());
-        final String infoPageResponse = NewPipe.getDownloader()
+        videoInfoPage.putAll(Parser.compatParseMap(videoInfoResponse.responseBody()));
-                .get(videoInfoUrl, getExtractorLocalization()).responseBody();
-        videoInfoPage.putAll(Parser.compatParseMap(infoPageResponse));
 
         try {
             playerResponse = JsonParser.object().from(videoInfoPage.get("player_response"));
@@ -975,11 +966,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     }
 
     @Nonnull
-    private static String getVideoInfoUrl(final String id, final String sts) {
+    private static String getVideoInfoUrl(final String id) {
         // TODO: Try parsing embedded_player_response first
-        return "https://www.youtube.com/get_video_info?" + "video_id=" + id +
+        return "https://www.youtube.com/get_video_info?video_id=" + id +
-                "&html5=1&eurl=https://youtube.googleapis.com/v/" + id +
+                "&eurl=https://youtube.googleapis.com/v/" + id +
-                "&sts=" + sts + "&ps=default&gl=US&hl=en";
+                "&html5=1&c=TVHTML5&cver=6.20180913&gl=US&hl=en";
     }
 
     private Map<String, ItagItem> getItags(final String streamingDataKey,
@@ -1121,7 +1112,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
     public String getCategory() {
         return playerResponse.getObject("microformat")
                 .getObject("playerMicroformatRenderer")
-                .getString("category");
+                .getString("category", EMPTY_STRING);
     }
 
     @Nonnull

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java

@@ -57,7 +57,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
     @Nullable @Override public String expectedErrorMessage() { return "Sign in to confirm your age"; }
     @Override public boolean expectedHasSubtitles() { return false; }
 
-    @Override public String expectedCategory() {return "Entertainment"; }
+    @Override public String expectedCategory() { return ""; } // Unavailable on age restricted videos
     @Override public String expectedLicence() { return "YouTube licence"; }
     @Override
     public List<String> expectedTags() {

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/notAvailable/generated_mock_1.json

@@ -26,7 +26,7 @@
     "responseMessage": "",
     "responseHeaders": {
       "alt-svc": [
-        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+        "h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
       ],
       "cache-control": [
         "no-cache, no-store, max-age\u003d0, must-revalidate"
@@ -38,7 +38,7 @@
         "application/json; charset\u003dutf-8"
       ],
       "date": [
-        "Sat, 22 May 2021 18:30:38 GMT"
+        "Wed, 23 Jun 2021 10:12:06 GMT"
       ],
       "expires": [
         "Mon, 01 Jan 1990 00:00:00 GMT"
@@ -56,8 +56,8 @@
         "ESF"
       ],
       "set-cookie": [
-        "YSC\u003dbWHlSChZfJE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
+        "YSC\u003d-LT8a9maoCU; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+805; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
+        "CONSENT\u003dPENDING+142; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
       ],
       "strict-transport-security": [
         "max-age\u003d31536000"
@@ -75,7 +75,7 @@
         "0"
       ]
     },
-    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r5---sn-4g5ednly.googlevideo.com\\/generate_204\",\"https:\\/\\/r5---sn-4g5ednly.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24037794,24040786,24044575,23884386,23944779,23983296,23882685,24014268,24027649,23995927,23735347,24043867,24032296,24011649,23975059,23891344,23968099,23970529,23966208,24005207,24042868,24005646,24019954,23986021,23940237,23891346,24037806,24003105,23996830,1714251,24005802,23804281,24030040,23946420,23744176,24035514,23890959,23918597,24007246,23974595,24045350,23934970,24001373,24036086,23968386,23997375,24042653,24027133,23857949,24004644,24035275,24024949,24036899,24042802\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0xd8a06d19cc578a00\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210324\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwi_h9z79d3wAhXw1xEIHUqjCfI\u003d\"}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgswTjM5MHFXN0hlayjOm6WFBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbmQyWVlOb0RMYzZ2YXVzdmhjWDZpRjdLQUJMQXxBQ3Jtc0trS2FPZGRQUzJZNFhoTU5EcnZjQjZFV0dMYnE0ZkNxYUNkeVlLd3N4MU9fRGpLYjRzdkliTVE4N3prNVByTVl2Mi04aFpZcHdvckhGSDI2U1Ewa24tbWFHNjNMa1pBb2lET0JSQW0xTUY4WklUOEtTcw\\u003d\\u003d\",\"url\": \"/watch?v\\u003dINVALID_ID_\",\"endpoint\": {\"clickTrackingParams\":\"IhMIobfb-_Xd8AIVzWjgCh0aDgYWMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dINVALID_ID_\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"INVALID_ID_\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
+    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r5---sn-hgn7yn7z.googlevideo.com\\/generate_204\",\"https:\\/\\/r5---sn-hgn7yn7z.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24001373,23946420,23970529,24049577,24042868,23966208,23804281,24027649,23998056,24049575,24036947,24004644,24017848,23891344,24007246,24037794,24052246,23963745,23934970,23996830,24030040,23983296,23891346,23884386,24037806,24042870,24012513,24058813,23857948,24053867,24049569,23968386,24049820,1714249,23973490,24049567,23918597,24003105,24049573,23744176,23974595,24058380,24003103,23882685,23944779,24038425,23890959,23999405\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0x58da0be5a82f9e65\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210526\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwiG75Ocwq3xAhU_NPEFHUM9A70\u003d\",\"frameworkUpdates\":{\"entityBatchUpdate\":{\"mutations\":[{\"entityKey\":\"Eg0KC0lOVkFMSURfSURfIPYBKAE%3D\",\"type\":\"ENTITY_MUTATION_TYPE_REPLACE\",\"payload\":{\"offlineabilityEntity\":{\"key\":\"Eg0KC0lOVkFMSURfSURfIPYBKAE%3D\",\"accessState\":\"OFFLINEABILITY_FEATURE_ACCESS_STATE_UNKNOWN\"}}}],\"timestamp\":{\"seconds\":\"1624443126\",\"nanos\":616071922}}}}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"Cgs1SUlhQUVDenVSUSj2kcyGBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbkdDQldjVWFhaHN4ZUQzTjMtY01fU1FpVElFd3xBQ3Jtc0tseTZPT0J2ZzA2eDNuYXRMREtXaGN0RWJsUWJ5eW1vWDQ4aGg5TDJHTjZvX3lSM2tRamh2M3lDdDNXc3haZE00TUJveWNZa2JBbWdYX21URUI2dTM2MTlfX0pIcWU1RVNXRHpreExHeHNfaW5LQjdvQQ\\u003d\\u003d\",\"url\": \"/watch?v\\u003dINVALID_ID_\",\"endpoint\": {\"clickTrackingParams\":\"IhMIpImTnMKt8QIV9TvxBR0TrgLLMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003dINVALID_ID_\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"INVALID_ID_\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
     "latestUrl": "https://www.youtube.com/watch?v\u003dINVALID_ID_\u0026pbj\u003d1"
   }
 }

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/stream/notAvailable/generated_mock_13.json

@@ -1,81 +0,0 @@
-{
-  "request": {
-    "httpMethod": "GET",
-    "url": "https://www.youtube.com/watch?v\u003ddon-t-exist\u0026pbj\u003d1",
-    "headers": {
-      "Accept-Language": [
-        "en-GB, en;q\u003d0.9"
-      ],
-      "Cookie": [
-        "CONSENT\u003dPENDING+100854"
-      ],
-      "X-YouTube-Client-Name": [
-        "1"
-      ],
-      "X-YouTube-Client-Version": [
-        "2.20200214.04.00"
-      ]
-    },
-    "localization": {
-      "languageCode": "en",
-      "countryCode": "GB"
-    }
-  },
-  "response": {
-    "responseCode": 200,
-    "responseMessage": "",
-    "responseHeaders": {
-      "alt-svc": [
-        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
-      ],
-      "cache-control": [
-        "no-cache, no-store, max-age\u003d0, must-revalidate"
-      ],
-      "content-disposition": [
-        "attachment"
-      ],
-      "content-type": [
-        "application/json; charset\u003dutf-8"
-      ],
-      "date": [
-        "Sat, 22 May 2021 18:30:40 GMT"
-      ],
-      "expires": [
-        "Mon, 01 Jan 1990 00:00:00 GMT"
-      ],
-      "p3p": [
-        "CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
-      ],
-      "permissions-policy": [
-        "ch-ua-full-version\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*, ch-ua-arch\u003d*, ch-ua-model\u003d*"
-      ],
-      "pragma": [
-        "no-cache"
-      ],
-      "server": [
-        "ESF"
-      ],
-      "set-cookie": [
-        "YSC\u003dlWKLRR9UNg0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
-        "CONSENT\u003dPENDING+495; expires\u003dFri, 01-Jan-2038 00:00:00 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
-      ],
-      "strict-transport-security": [
-        "max-age\u003d31536000"
-      ],
-      "x-content-type-options": [
-        "nosniff"
-      ],
-      "x-frame-options": [
-        "SAMEORIGIN"
-      ],
-      "x-spf-response-type": [
-        "multipart"
-      ],
-      "x-xss-protection": [
-        "0"
-      ]
-    },
-    "responseBody": "[\r\n{\"page\": \"watch\",\"rootVe\": \"3832\"},\r\n{\"page\": \"watch\",\"preconnect\": [\"https:\\/\\/r3---sn-4g5e6nlk.googlevideo.com\\/generate_204\",\"https:\\/\\/r3---sn-4g5e6nlk.googlevideo.com\\/generate_204?conn2\"]},\r\n{\"page\": \"watch\",\"playerResponse\": {\"responseContext\":{\"serviceTrackingParams\":[{\"service\":\"GFEEDBACK\",\"params\":[{\"key\":\"is_viewed_live\",\"value\":\"False\"},{\"key\":\"logged_in\",\"value\":\"0\"},{\"key\":\"e\",\"value\":\"24036085,23891344,23804281,23997375,23946420,24001373,24039105,24004644,24042868,23891346,23966208,24030040,9479112,23970529,23974595,24043867,23968099,24037806,23884386,23940238,23882502,24027649,23934970,24037794,23857950,24007246,24022729,23918597,23986023,24035164,24044568,24040786,1714242,24005802,24046663,23944779,24042802,23983296,9406121,24045792,24014268,24005646,24027133,24042860,23968386,23939457,24035275,23996830,24043529,23890959,23984883,24010465,24019954,23744176,24045810,24026152,24036004,24044576,23995927,23990877,23868992\"}]},{\"service\":\"CSI\",\"params\":[{\"key\":\"c\",\"value\":\"WEB\"},{\"key\":\"cver\",\"value\":\"2.20200214.04.00\"},{\"key\":\"yt_li\",\"value\":\"0\"},{\"key\":\"GetPlayer_rid\",\"value\":\"0x899d9973680ec201\"}]},{\"service\":\"GUIDED_HELP\",\"params\":[{\"key\":\"logged_in\",\"value\":\"0\"}]},{\"service\":\"ECATCHER\",\"params\":[{\"key\":\"client.version\",\"value\":\"2.20210324\"},{\"key\":\"client.name\",\"value\":\"WEB\"}]}],\"mainAppWebResponseContext\":{\"loggedOut\":true},\"webResponseContextExtensionData\":{\"hasDecorated\":true}},\"playabilityStatus\":{\"status\":\"ERROR\",\"reason\":\"Video unavailable\",\"errorScreen\":{\"playerErrorMessageRenderer\":{\"reason\":{\"simpleText\":\"Video unavailable\"},\"thumbnail\":{\"thumbnails\":[{\"url\":\"//s.ytimg.com/yts/img/meh7-vflGevej7.png\",\"width\":140,\"height\":100}]},\"icon\":{\"iconType\":\"ERROR_OUTLINE\"}}},\"contextParams\":\"Q0FBU0FnZ0E\u003d\"},\"trackingParams\":\"CAAQu2kiEwj_zob99d3wAhXBrnsKHS0SBPI\u003d\"}},\r\n{\"page\": \"watch\",\"response\": {\"responseContext\":{\"webResponseContextExtensionData\":{\"ytConfigData\":{\"visitorData\":\"CgtuN1hhZ1RER2c2WSjQm6WFBg%3D%3D\",\"rootVisualElementType\":3832}}}},\"xsrf_token\": \"QUFFLUhqbHE4OV9DcUszaUJyQU5QNnNOSWowczFaMVJad3xBQ3Jtc0ttb1NRdjVEQXdGQ1BhaE5CLUxpNEZMZkRuWnBrU2tHWk92elZ3VVFsTlM4YWYweEFRSDB5SVllSVBfTVZrRzQwOFlDclF1b1hXOVA2TF9UV3A2eDVLRTBtY0VyRktQM0hCdldqT1hvOTJJcnNJZkFIZw\\u003d\\u003d\",\"url\": \"/watch?v\\u003ddon-t-exist\",\"endpoint\": {\"clickTrackingParams\":\"IhMIvfaF_fXd8AIVbcURCB0PPgimMghleHRlcm5hbA\u003d\u003d\",\"commandMetadata\":{\"webCommandMetadata\":{\"url\":\"/watch?v\u003ddon-t-exist\",\"webPageType\":\"WEB_PAGE_TYPE_WATCH\",\"rootVe\":3832}},\"watchEndpoint\":{\"videoId\":\"don-t-exist\"}}},\r\n{\"page\": \"watch\",\"timing\": {\"info\": {\"st\":  0.0 }}}]\r\n",
-    "latestUrl": "https://www.youtube.com/watch?v\u003ddon-t-exist\u0026pbj\u003d1"
-  }
-}