Extract YouTube's key
This commit is contained in:
parent
4c987a5302
commit
f345f667e2
2 changed files with 55 additions and 33 deletions
|
@ -64,6 +64,8 @@ public class YoutubeParsingHelper {
|
||||||
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
|
private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
|
||||||
private static String clientVersion;
|
private static String clientVersion;
|
||||||
|
|
||||||
|
private static String key;
|
||||||
|
|
||||||
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
|
private static final String[] HARDCODED_YOUTUBE_MUSIC_KEYS = {"AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30", "67", "0.1"};
|
||||||
private static String[] youtubeMusicKeys;
|
private static String[] youtubeMusicKeys;
|
||||||
|
|
||||||
|
@ -214,39 +216,31 @@ public class YoutubeParsingHelper {
|
||||||
return response.length() > 50; // ensure to have a valid response
|
return response.length() > 50; // ensure to have a valid response
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private static void getClientVersionAndKey() throws IOException, ExtractionException {
|
||||||
* Get the client version from a page
|
|
||||||
* @return
|
|
||||||
* @throws ParsingException
|
|
||||||
*/
|
|
||||||
public static String getClientVersion() throws IOException, ExtractionException {
|
|
||||||
if (!isNullOrEmpty(clientVersion)) return clientVersion;
|
|
||||||
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
|
|
||||||
|
|
||||||
final String url = "https://www.youtube.com/results?search_query=test";
|
final String url = "https://www.youtube.com/results?search_query=test";
|
||||||
final String html = getDownloader().get(url).responseBody();
|
final String html = getDownloader().get(url).responseBody();
|
||||||
JsonObject initialData = getInitialData(html);
|
final JsonObject initialData = getInitialData(html);
|
||||||
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
|
final JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
|
||||||
String shortClientVersion = null;
|
String shortClientVersion = null;
|
||||||
|
|
||||||
// try to get version from initial data first
|
// try to get version from initial data first
|
||||||
for (Object service : serviceTrackingParams) {
|
for (final Object service : serviceTrackingParams) {
|
||||||
JsonObject s = (JsonObject) service;
|
final JsonObject s = (JsonObject) service;
|
||||||
if (s.getString("service").equals("CSI")) {
|
if (s.getString("service").equals("CSI")) {
|
||||||
JsonArray params = s.getArray("params");
|
final JsonArray params = s.getArray("params");
|
||||||
for (Object param : params) {
|
for (final Object param : params) {
|
||||||
JsonObject p = (JsonObject) param;
|
final JsonObject p = (JsonObject) param;
|
||||||
String key = p.getString("key");
|
final String key = p.getString("key");
|
||||||
if (key != null && key.equals("cver")) {
|
if (key != null && key.equals("cver")) {
|
||||||
return clientVersion = p.getString("value");
|
clientVersion = p.getString("value");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if (s.getString("service").equals("ECATCHER")) {
|
} else if (s.getString("service").equals("ECATCHER")) {
|
||||||
// fallback to get a shortened client version which does not contain the last two digits
|
// fallback to get a shortened client version which does not contain the last two digits
|
||||||
JsonArray params = s.getArray("params");
|
final JsonArray params = s.getArray("params");
|
||||||
for (Object param : params) {
|
for (final Object param : params) {
|
||||||
JsonObject p = (JsonObject) param;
|
final JsonObject p = (JsonObject) param;
|
||||||
String key = p.getString("key");
|
final String key = p.getString("key");
|
||||||
if (key != null && key.equals("client.version")) {
|
if (key != null && key.equals("client.version")) {
|
||||||
shortClientVersion = p.getString("value");
|
shortClientVersion = p.getString("value");
|
||||||
}
|
}
|
||||||
|
@ -255,26 +249,54 @@ public class YoutubeParsingHelper {
|
||||||
}
|
}
|
||||||
|
|
||||||
String contextClientVersion;
|
String contextClientVersion;
|
||||||
String[] patterns = {
|
final String[] patterns = {
|
||||||
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
|
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
|
||||||
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
|
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
|
||||||
"client.version=([0-9\\.]+)"
|
"client.version=([0-9\\.]+)"
|
||||||
};
|
};
|
||||||
for (String pattern : patterns) {
|
for (final String pattern : patterns) {
|
||||||
try {
|
try {
|
||||||
contextClientVersion = Parser.matchGroup1(pattern, html);
|
contextClientVersion = Parser.matchGroup1(pattern, html);
|
||||||
if (!isNullOrEmpty(contextClientVersion)) {
|
if (!isNullOrEmpty(contextClientVersion)) {
|
||||||
return clientVersion = contextClientVersion;
|
clientVersion = contextClientVersion;
|
||||||
}
|
|
||||||
} catch (Exception ignored) {
|
|
||||||
}
|
}
|
||||||
|
} catch (Parser.RegexException ignored) { }
|
||||||
}
|
}
|
||||||
|
|
||||||
if (shortClientVersion != null) {
|
if (shortClientVersion != null) {
|
||||||
return clientVersion = shortClientVersion;
|
clientVersion = shortClientVersion;
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new ParsingException("Could not get client version");
|
try {
|
||||||
|
key = Parser.matchGroup1("INNERTUBE_API_KEY\":\"([0-9a-zA-Z_-]+?)\"", html);
|
||||||
|
} catch (Parser.RegexException e) {
|
||||||
|
try {
|
||||||
|
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
|
||||||
|
} catch (Parser.RegexException ignored) { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the client version
|
||||||
|
*/
|
||||||
|
public static String getClientVersion() throws IOException, ExtractionException {
|
||||||
|
if (!isNullOrEmpty(clientVersion)) return clientVersion;
|
||||||
|
if (isHardcodedClientVersionValid()) return clientVersion = HARDCODED_CLIENT_VERSION;
|
||||||
|
|
||||||
|
getClientVersionAndKey();
|
||||||
|
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract client version");
|
||||||
|
return clientVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the key
|
||||||
|
*/
|
||||||
|
public static String getKey() throws IOException, ExtractionException {
|
||||||
|
if (!isNullOrEmpty(key)) return key;
|
||||||
|
|
||||||
|
getClientVersionAndKey();
|
||||||
|
if (isNullOrEmpty(key)) throw new ParsingException("Could not extract key");
|
||||||
|
return key;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
|
public static boolean areHardcodedYoutubeMusicKeysValid() throws IOException, ReCaptchaException {
|
||||||
|
|
|
@ -28,6 +28,7 @@ import javax.annotation.Nonnull;
|
||||||
|
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
|
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
|
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
|
||||||
|
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
|
||||||
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
|
||||||
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
|
||||||
|
@ -107,7 +108,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
||||||
|
|
||||||
@Nonnull
|
@Nonnull
|
||||||
@Override
|
@Override
|
||||||
public InfoItemsPage<InfoItem> getInitialPage() throws ExtractionException {
|
public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
|
||||||
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
final InfoItemsSearchCollector collector = new InfoItemsSearchCollector(getServiceId());
|
||||||
|
|
||||||
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
|
final JsonArray sections = initialData.getObject("contents").getObject("twoColumnSearchResultsRenderer")
|
||||||
|
@ -221,7 +222,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
||||||
+ "&itct=" + clickTrackingParams);
|
+ "&itct=" + clickTrackingParams);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) {
|
private Page getNewNextPageFrom(final JsonObject continuationItemRenderer) throws IOException, ExtractionException {
|
||||||
if (isNullOrEmpty(continuationItemRenderer)) {
|
if (isNullOrEmpty(continuationItemRenderer)) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -229,8 +230,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
|
||||||
final String token = continuationItemRenderer.getObject("continuationEndpoint")
|
final String token = continuationItemRenderer.getObject("continuationEndpoint")
|
||||||
.getObject("continuationCommand").getString("token");
|
.getObject("continuationCommand").getString("token");
|
||||||
|
|
||||||
// FIXME: Key needs to be extracted
|
final String url = "https://www.youtube.com/youtubei/v1/search?key=" + getKey();
|
||||||
final String url = "https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
|
|
||||||
|
|
||||||
return new Page(url, token);
|
return new Page(url, token);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue