Extract YouTube's key
This commit is contained in:
parent
4c987a5302
commit
f345f667e2
2 changed files with 55 additions and 33 deletions
|
@ -64,6 +64,8 @@ public class YoutubeParsingHelper {
|
|||
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
|
||||
private static String clientVersion;
|
||||
|
||||
private static String key;
|
||||
|
||||
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
|
||||
private static String[] youtubeMusicKeys;
|
||||
|
||||
|
@ -214,39 +216,31 @@ public class YoutubeParsingHelper {
|
|||
return response.length() > 50; // ensure to have a valid response
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the client version from a page
|
||||
* @return
|
||||
* @throws ParsingException
|
||||
*/
|
||||
public static String getClientVersion() throws IOException, ExtractionException {
|
||||
if (!isNullOrEmpty(clientVersion)) return clientVersion;
|
||||
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
|
||||
|
||||
private static void getClientVersionAndKey() throws IOException, ExtractionException {
|
||||
final String url = "https://www.youtube.com/results?search_query=test";
|
||||
final String html = getDownloader().get(url).responseBody();
|
||||
JsonObject initialData = getInitialData(html);
|
||||
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
|
||||
final JsonObject initialData = getInitialData(html);
|
||||
final JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
|
||||
String shortClientVersion = null;
|
||||
|
||||
// try to get version from initial data first
|
||||
for (Object service : serviceTrackingParams) {
|
||||
JsonObject s = (JsonObject) service;
|
||||
for (final Object service : serviceTrackingParams) {
|
||||
final JsonObject s = (JsonObject) service;
|
||||
if (s.getString("service").equals("CSI")) {
|
||||
JsonArray params = s.getArray("params");
|
||||
for (Object param : params) {
|
||||
JsonObject p = (JsonObject) param;
|
||||
String key = p.getString("key");
|
||||
final JsonArray params = s.getArray("params");
|
||||
for (final Object param : params) {
|
||||
final JsonObject p = (JsonObject) param;
|
||||
final String key = p.getString("key");
|
||||
if (key != null && key.equals("cver")) {
|
||||
return clientVersion = p.getString("value");
|
||||
clientVersion = p.getString("value");
|
||||
}
|
||||
}
|
||||
} else if (s.getString("service").equals("ECATCHER")) {
|
||||
// fallback to get a shortened client version which does not contain the last two digits
|
||||
JsonArray params = s.getArray("params");
|
||||
for (Object param : params) {
|
||||
JsonObject p = (JsonObject) param;
|
||||
String key = p.getString("key");
|
||||
final JsonArray params = s.getArray("params");
|
||||
for (final Object param : params) {
|
||||
final JsonObject p = (JsonObject) param;
|
||||
final String key = p.getString("key");
|
||||
if (key != null && key.equals("client.version")) {
|
||||
shortClientVersion = p.getString("value");
|
||||
}
|
||||
|
@ -255,26 +249,54 @@ public class YoutubeParsingHelper {
|
|||
}
|
||||
|
||||
String contextClientVersion;
|
||||
String[] patterns = {
|
||||
final String[] patterns = {
|
||||
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
|
||||
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
|
||||
"client.version=([0-9\\.]+)"
|
||||
};
|
||||
for (String pattern : patterns) {
|
||||
for (final String pattern : patterns) {
|
||||
try {
|
||||
contextClientVersion = Parser.matchGroup1(pattern, html);
|
||||
if (!isNullOrEmpty(contextClientVersion)) {
|
||||
return clientVersion = contextClientVersion;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
clientVersion = contextClientVersion;
|
||||
}
|
||||
} catch (Parser.RegexException ignored) { }
|
||||
}
|
||||
|
||||
if (shortClientVersion != null) {
|
||||
return clientVersion = shortClientVersion;
|
||||
clientVersion = shortClientVersion;
|
||||
}
|
||||
|
||||
throw new ParsingException("Could not get client version");
|
||||
try {
|
||||
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
|
||||
} catch (Parser.RegexException e) {
|
||||
try {
|
||||
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
|
||||
} catch (Parser.RegexException ignored) { }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the client version
|
||||
*/
|
||||
public static String getClientVersion() throws IOException, ExtractionException {
|
||||
if (!isNullOrEmpty(clientVersion)) return clientVersion;
|
||||
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
|
||||
|
||||
getClientVersionAndKey();
|
||||
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version");
|
||||
return clientVersion;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the key
|
||||
*/
|
||||
public static String getKey() throws IOException, ExtractionException {
|
||||
if (!isNullOrEmpty(key)) return key;
|
||||
|
||||
getClientVersionAndKey();
|
||||
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract key");
|
||||
return key;
|
||||
}
|
||||
|
||||
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
|
||||
|
|
|
@ -28,6 +28,7 @@ import javax.annotation.Nonnull;
|
|||
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||
|
@ -107,7 +108,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
|
||||
@Nonnull
|
||||
@Override
|
||||
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
|
||||
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
|
||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||
|
||||
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
|
||||
|
@ -221,7 +222,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
+ "&itct=" + clickTrackingParams);
|
||||
}
|
||||
|
||||
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) {
|
||||
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException {
|
||||
if (isNullOrEmpty(continuationItemRenderer)) {
|
||||
return null;
|
||||
}
|
||||
|
@ -229,8 +230,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
|||
final String token = continuationItemRenderer.getObject("continuationEndpoint")
|
||||
.getObject("continuationCommand").getString("token");
|
||||
|
||||
// FIXME: Key needs to be extracted
|
||||
final String url = "https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
|
||||
final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
|
||||
|
||||
return new Page(url, token);
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue