Use the youtubei API for YouTube searches + update mocks

Add getSearchParameter, a new method in YoutubeSearchQueryHandlerFactory class which returns the params field for a search, or an empty string if there is no one.
Update mocks of YoutubeSearchExtractorTest.
This commit is contained in:
TiA4f8R 2021-04-09 17:00:38 +02:00
parent a12c69da7d
commit f461224b2b
No known key found for this signature in database
GPG key ID: E6D3E7F5949450DD
111 changed files with 18512 additions and 6359 deletions

View file

@ -662,20 +662,6 @@ public class YoutubeParsingHelper {
return response;
}
public static String extractCookieValue(final String cookieName, final Response response) {
final List<String> cookies = response.responseHeaders().get("set-cookie");
int startIndex;
String result = "";
for (final String cookie : cookies) {
startIndex = cookie.indexOf(cookieName);
if (startIndex != -1) {
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
cookie.indexOf(";", startIndex));
}
}
return result;
}
public static JsonObject getJsonPostResponse(final String endpoint,
final byte[] body,
final Localization localization)

View file

@ -94,8 +94,6 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
final JsonObject jsonResponse = getJsonPostResponse("navigation/resolve_url",
body, getExtractorLocalization());
System.out.println(jsonResponse.toString());
if (jsonResponse.has("error")) {
if (jsonResponse.getInt("code") == 404) {
throw new ContentNotAvailableException("No channel associated with this user"

View file

@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.getSearchParameter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@ -55,11 +56,33 @@ public class YoutubeSearchExtractor extends SearchExtractor {
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final String query = super.getSearchString();
final JsonArray ajaxJson = getJsonResponse(url, getExtractorLocalization());
// Get the search parameter of the request
final List<String> contentFilters = super.getLinkHandler().getContentFilters();
final String params;
if (!isNullOrEmpty(contentFilters)) {
final String searchType = contentFilters.get(0);
params = getSearchParameter(searchType);
} else {
params = "";
}
initialData = ajaxJson.getObject(1).getObject("response");
final byte[] body;
if (!isNullOrEmpty(params)) {
body = JsonWriter.string(prepareJsonBuilder()
.value("query", query)
.value("params", params)
.done())
.getBytes(UTF_8);
} else {
body = JsonWriter.string(prepareJsonBuilder()
.value("query", query)
.done())
.getBytes(UTF_8);
}
initialData = getJsonPostResponse("search", body, getExtractorLocalization());
}
@Nonnull

View file

@ -8,6 +8,7 @@ import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@ -72,4 +73,27 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
// MUSIC_ARTISTS
};
}
public static String getSearchParameter(final String contentFilter) {
if (!isNullOrEmpty(contentFilter)) {
switch (contentFilter) {
case VIDEOS:
return "EgIQAQ%3D%3D";
case CHANNELS:
return "EgIQAg%3D%3D";
case PLAYLISTS:
return "EgIQAw%3D%3D";
case ALL:
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS:
default:
return "";
}
} else {
return "";
}
}
}

View file

@ -40,10 +40,10 @@ public class YoutubeChannelLocalizationTest {
testLocalizationsFor("https://www.youtube.com/channel/UCEOXxzW2vU0P-0THehuIIeg");
}
private void testLocalizationsFor(String channelUrl) throws Exception {
private void testLocalizationsFor(final String channelUrl) throws Exception {
final List<Localization> supportedLocalizations = YouTube.getSupportedLocalizations();
// final List<Localization> supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr"));
// final List<Localization> supportedLocalizations = Arrays.asList(Localization.DEFAULT, new Localization("sr"));
final Map<Localization, List<StreamInfoItem>> results = new LinkedHashMap<>();
for (Localization currentLocalization : supportedLocalizations) {
@ -55,7 +55,7 @@ public class YoutubeChannelLocalizationTest {
extractor.forceLocalization(currentLocalization);
extractor.fetchPage();
itemsPage = defaultTestRelatedItems(extractor);
} catch (Throwable e) {
} catch (final Throwable e) {
System.out.println("[!] " + currentLocalization + " → failed");
throw e;
}

View file

@ -11,7 +11,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class YouTubeCommentsLinkHandlerFactoryTest {
public class YoutubeCommentsLinkHandlerFactoryTest {
private static YoutubeCommentsLinkHandlerFactory linkHandler;

View file

@ -61,7 +61,7 @@ public class YoutubePlaylistExtractorTest {
}
@Test(expected = ContentNotAvailableException.class)
@Ignore("Broken, now invalid playlists redirect to youtube homepage")
@Ignore("Broken, now invalid playlists redirect to YouTube homepage")
public void invalidId() throws Exception {
final PlaylistExtractor extractor =
YouTube.getPlaylistExtractor("https://www.youtube.com/playlist?list=INVALID_ID");

View file

@ -0,0 +1,176 @@
{
"request": {
"httpMethod": "POST",
"url": "https://youtubei.googleapis.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
"headers": {
"Accept-Language": [
"en-GB, en;q\u003d0.9"
]
},
"dataToSend": [
123,
34,
98,
114,
111,
119,
115,
101,
73,
100,
34,
58,
34,
68,
79,
69,
83,
78,
84,
45,
69,
88,
73,
83,
84,
34,
44,
34,
99,
111,
110,
116,
101,
120,
116,
34,
58,
123,
34,
99,
108,
105,
101,
110,
116,
34,
58,
123,
34,
99,
108,
105,
101,
110,
116,
78,
97,
109,
101,
34,
58,
34,
49,
34,
44,
34,
99,
108,
105,
101,
110,
116,
86,
101,
114,
115,
105,
111,
110,
34,
58,
34,
50,
46,
50,
48,
50,
49,
48,
54,
48,
54,
34,
125,
125,
44,
34,
112,
97,
114,
97,
109,
115,
34,
58,
34,
69,
103,
90,
50,
97,
87,
82,
108,
98,
51,
77,
37,
51,
68,
34,
125
],
"localization": {
"languageCode": "en",
"countryCode": "GB"
}
},
"response": {
"responseCode": 400,
"responseMessage": "",
"responseHeaders": {
"alt-svc": [
"h3-29\u003d\":443\"; ma\u003d2592000,h3-T051\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
],
"cache-control": [
"private"
],
"content-type": [
"application/json; charset\u003dUTF-8"
],
"date": [
"Tue, 08 Jun 2021 13:14:16 GMT"
],
"server": [
"ESF"
],
"vary": [
"Origin",
"X-Origin",
"Referer"
],
"x-content-type-options": [
"nosniff"
],
"x-frame-options": [
"SAMEORIGIN"
],
"x-xss-protection": [
"0"
]
},
"responseBody": "{\n \"error\": {\n \"code\": 400,\n \"message\": \"Request contains an invalid argument.\",\n \"errors\": [\n {\n \"message\": \"Request contains an invalid argument.\",\n \"domain\": \"global\",\n \"reason\": \"badRequest\"\n }\n ],\n \"status\": \"INVALID_ARGUMENT\"\n }\n}\n",
"latestUrl": "https://youtubei.googleapis.com/youtubei/v1/browse?key\u003dAIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
}
}

Some files were not shown because too many files have changed in this diff Show more