Use the youtubei API for YouTube searches + update mocks

Add getSearchParameter, a new method in YoutubeSearchQueryHandlerFactory class which returns the params field for a search, or an empty string if there is no one.
Update mocks of YoutubeSearchExtractorTest.

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java

@@ -662,20 +662,6 @@ public class YoutubeParsingHelper {
         return response;
     }
 
-    public static String extractCookieValue(final String cookieName, final Response response) {
-        final List<String> cookies = response.responseHeaders().get("set-cookie");
-        int startIndex;
-        String result = "";
-        for (final String cookie : cookies) {
-            startIndex = cookie.indexOf(cookieName);
-            if (startIndex != -1) {
-                result = cookie.substring(startIndex + cookieName.length() + "=".length(),
-                        cookie.indexOf(";", startIndex));
-            }
-        }
-        return result;
-    }
-
     public static JsonObject getJsonPostResponse(final String endpoint,
                                                 final byte[] body,
                                                 final Localization localization)

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -94,8 +94,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
             final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
                     body, getExtractorLocalization());
 
-            System.out.println(jsonResponse.toString());
-
             if (jsonResponse.has("error")) {
                 if (jsonResponse.getInt("code") == 404) {
                     throw new ContentNotAvailableException("No channel associated with this user"

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java

@@ -22,6 +22,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
 import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -55,11 +56,33 @@ public class YoutubeSearchExtractor extends SearchExtractor {
 
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
-        final String url = getUrl() + "&pbj=1";
+        final String query = super.getSearchString();
 
-        final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
+        // Get the search parameter of the request
+        final List<String> contentFilters = super.getLinkHandler().getContentFilters();
+        final String params;
+        if (!isNullOrEmpty(contentFilters)) {
+            final String searchType = contentFilters.get(0);
+            params = getSearchParameter(searchType);
+        } else {
+            params = "";
+        }
 
-        initialData = ajaxJson.getObject(1).getObject("response");
+        final byte[] body;
+        if (!isNullOrEmpty(params)) {
+            body = JsonWriter.string(prepareJsonBuilder()
+                    .value("query", query)
+                    .value("params", params)
+                    .done())
+                    .getBytes(UTF_8);
+        } else {
+            body = JsonWriter.string(prepareJsonBuilder()
+                    .value("query", query)
+                    .done())
+                    .getBytes(UTF_8);
+        }
+
+        initialData = getJsonPostResponse("search", body, getExtractorLocalization());
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeSearchQueryHandlerFactory.java

@@ -8,6 +8,7 @@ import java.net.URLEncoder;
 import java.util.List;
 
 import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
+import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 
 public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
 
@@ -72,4 +73,27 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
 //                MUSIC_ARTISTS
         };
     }
+
+    public static String getSearchParameter(final String contentFilter) {
+        if (!isNullOrEmpty(contentFilter)) {
+            switch (contentFilter) {
+                case VIDEOS:
+                    return "EgIQAQ%3D%3D";
+                case CHANNELS:
+                    return "EgIQAg%3D%3D";
+                case PLAYLISTS:
+                    return "EgIQAw%3D%3D";
+                case ALL:
+                case MUSIC_SONGS:
+                case MUSIC_VIDEOS:
+                case MUSIC_ALBUMS:
+                case MUSIC_PLAYLISTS:
+                case MUSIC_ARTISTS:
+                default:
+                    return "";
+            }
+        } else {
+            return "";
+        }
+    }
 }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelLocalizationTest.java

@@ -40,10 +40,10 @@ public class YoutubeChannelLocalizationTest {
         testLocalizationsFor("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg");
     }
 
-    private void testLocalizationsFor(String channelUrl) throws Exception {
+    private void testLocalizationsFor(final String channelUrl) throws Exception {
 
         final List<Localization> supportedLocalizations = YouTube.getSupportedLocalizations();
-//        final List<Localization> supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr"));
+        // final List<Localization> supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr"));
         final Map<Localization, List<StreamInfoItem>> results = new LinkedHashMap<>();
 
         for (Localization currentLocalization : supportedLocalizations) {
@@ -55,7 +55,7 @@ public class YoutubeChannelLocalizationTest {
                 extractor.forceLocalization(currentLocalization);
                 extractor.fetchPage();
                 itemsPage = defaultTestRelatedItems(extractor);
-            } catch (Throwable e) {
+            } catch (final Throwable e) {
                 System.out.println("[!] " + currentLocalization + " → failed");
                 throw e;
             }

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeCommentsLinkHandlerFactoryTest.java

@@ -11,7 +11,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
-public class YouTubeCommentsLinkHandlerFactoryTest {
+public class YoutubeCommentsLinkHandlerFactoryTest {
 
     private static YoutubeCommentsLinkHandlerFactory linkHandler;
 

extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java

@@ -61,7 +61,7 @@ public class YoutubePlaylistExtractorTest {
         }
 
         @Test(expected = ContentNotAvailableException.class)
-        @Ignore("Broken, now invalid playlists redirect to youtube homepage")
+        @Ignore("Broken, now invalid playlists redirect to YouTube homepage")
         public void invalidId() throws Exception {
             final PlaylistExtractor extractor =
                     YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID");

extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/channel/notAvailable/generated_mock_9.json

@@ -0,0 +1,176 @@
+{
+  "request": {
+    "httpMethod": "POST",
+    "url": "https://youtubei.googleapis.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
+    "headers": {
+      "Accept-Language": [
+        "en-GB, en;q\u003d0.9"
+      ]
+    },
+    "dataToSend": [
+      123,
+      34,
+      98,
+      114,
+      111,
+      119,
+      115,
+      101,
+      73,
+      100,
+      34,
+      58,
+      34,
+      68,
+      79,
+      69,
+      83,
+      78,
+      84,
+      45,
+      69,
+      88,
+      73,
+      83,
+      84,
+      34,
+      44,
+      34,
+      99,
+      111,
+      110,
+      116,
+      101,
+      120,
+      116,
+      34,
+      58,
+      123,
+      34,
+      99,
+      108,
+      105,
+      101,
+      110,
+      116,
+      34,
+      58,
+      123,
+      34,
+      99,
+      108,
+      105,
+      101,
+      110,
+      116,
+      78,
+      97,
+      109,
+      101,
+      34,
+      58,
+      34,
+      49,
+      34,
+      44,
+      34,
+      99,
+      108,
+      105,
+      101,
+      110,
+      116,
+      86,
+      101,
+      114,
+      115,
+      105,
+      111,
+      110,
+      34,
+      58,
+      34,
+      50,
+      46,
+      50,
+      48,
+      50,
+      49,
+      48,
+      54,
+      48,
+      54,
+      34,
+      125,
+      125,
+      44,
+      34,
+      112,
+      97,
+      114,
+      97,
+      109,
+      115,
+      34,
+      58,
+      34,
+      69,
+      103,
+      90,
+      50,
+      97,
+      87,
+      82,
+      108,
+      98,
+      51,
+      77,
+      37,
+      51,
+      68,
+      34,
+      125
+    ],
+    "localization": {
+      "languageCode": "en",
+      "countryCode": "GB"
+    }
+  },
+  "response": {
+    "responseCode": 400,
+    "responseMessage": "",
+    "responseHeaders": {
+      "alt-svc": [
+        "h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
+      ],
+      "cache-control": [
+        "private"
+      ],
+      "content-type": [
+        "application/json; charset\u003dUTF-8"
+      ],
+      "date": [
+        "Tue, 08 Jun 2021 13:14:16 GMT"
+      ],
+      "server": [
+        "ESF"
+      ],
+      "vary": [
+        "Origin",
+        "X-Origin",
+        "Referer"
+      ],
+      "x-content-type-options": [
+        "nosniff"
+      ],
+      "x-frame-options": [
+        "SAMEORIGIN"
+      ],
+      "x-xss-protection": [
+        "0"
+      ]
+    },
+    "responseBody": "{\n  \"error\": {\n    \"code\": 400,\n    \"message\": \"Request contains an invalid argument.\",\n    \"errors\": [\n      {\n        \"message\": \"Request contains an invalid argument.\",\n        \"domain\": \"global\",\n        \"reason\": \"badRequest\"\n      }\n    ],\n    \"status\": \"INVALID_ARGUMENT\"\n  }\n}\n",
+    "latestUrl": "https://youtubei.googleapis.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
+  }
+}