Improve getClientVersion()

This commit is contained in:
wb9688 2020-02-26 15:22:59 +01:00
parent 15afbea3e1
commit 985c3ec877
4 changed files with 83 additions and 62 deletions

View File

@ -5,11 +5,9 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor; import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
@ -68,10 +66,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version", headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams"); throw new ParsingException("Could not parse json data for next streams");
@ -222,10 +218,8 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version", headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams"); throw new ParsingException("Could not parse json data for next streams");

View File

@ -43,10 +43,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version", headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams"); throw new ParsingException("Could not parse json data for next streams");
@ -202,10 +200,8 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version", headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams"); throw new ParsingException("Could not parse json data for next streams");

View File

@ -5,11 +5,9 @@ import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler; import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
@ -61,10 +59,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version", headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody(); final String response = getDownloader().get(url, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams"); throw new ParsingException("Could not parse json data for next streams");
@ -130,10 +126,8 @@ public class YoutubeSearchExtractor extends SearchExtractor {
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
// Use the hardcoded client version first to get JSON with a structure we know
// TODO: Use YoutubeParsingHelper.getClientVersion() as fallback
headers.put("X-YouTube-Client-Version", headers.put("X-YouTube-Client-Version",
Collections.singletonList(YoutubeParsingHelper.HARDCODED_CLIENT_VERSION)); Collections.singletonList(YoutubeParsingHelper.getClientVersion()));
final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody(); final String response = getDownloader().get(pageUrl, headers, getExtractorLocalization()).responseBody();
if (response.length() < 50) { // ensure to have a valid response if (response.length() < 50) { // ensure to have a valid response
throw new ParsingException("Could not parse json data for next streams"); throw new ParsingException("Could not parse json data for next streams");

View File

@ -5,6 +5,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
@ -16,7 +17,13 @@ import java.net.URL;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
/* /*
* Created by Christian Schabesberger on 02.03.16. * Created by Christian Schabesberger on 02.03.16.
@ -43,7 +50,9 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() { private YoutubeParsingHelper() {
} }
public static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00"; private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;
private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id="; private static final String FEED_BASE_CHANNEL_ID = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user="; private static final String FEED_BASE_USER = "https://www.youtube.com/feeds/videos.xml?user=";
@ -162,55 +171,83 @@ public class YoutubeParsingHelper {
/** /**
* Get the client version from a page * Get the client version from a page
* @param initialData
* @param html The page HTML
* @return * @return
* @throws ParsingException * @throws ParsingException
*/ */
public static String getClientVersion(JsonObject initialData, String html) throws ParsingException { public static String getClientVersion() throws ParsingException {
if (initialData == null) initialData = getInitialData(html); if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
String shortClientVersion = null;
// try to get version from initial data first // Test if hard-coded client version is valid
for (Object service : serviceTrackingParams) { try {
JsonObject s = (JsonObject) service; final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
if (s.getString("service").equals("CSI")) {
JsonArray params = s.getArray("params"); Map<String, List<String>> headers = new HashMap<>();
for (Object param: params) { headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
JsonObject p = (JsonObject) param; headers.put("X-YouTube-Client-Version",
String key = p.getString("key"); Collections.singletonList(HARDCODED_CLIENT_VERSION));
if (key != null && key.equals("cver")) { final String response = getDownloader().get(url, headers).responseBody();
return p.getString("value"); if (response.length() > 50) { // ensure to have a valid response
clientVersion = HARDCODED_CLIENT_VERSION;
return clientVersion;
}
} catch (Exception ignored) {}
// Try extracting it from YouTube's website otherwise
try {
final String url = "https://www.youtube.com/results?search_query=test";
final String html = getDownloader().get(url).responseBody();
JsonObject initialData = getInitialData(html);
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
String shortClientVersion = null;
// try to get version from initial data first
for (Object service : serviceTrackingParams) {
JsonObject s = (JsonObject) service;
if (s.getString("service").equals("CSI")) {
JsonArray params = s.getArray("params");
for (Object param : params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("cver")) {
clientVersion = p.getString("value");
return clientVersion;
}
} }
} } else if (s.getString("service").equals("ECATCHER")) {
} else if (s.getString("service").equals("ECATCHER")) { // fallback to get a shortened client version which does not contain the last do digits
// fallback to get a shortened client version which does not contain the last do digits JsonArray params = s.getArray("params");
JsonArray params = s.getArray("params"); for (Object param : params) {
for (Object param: params) { JsonObject p = (JsonObject) param;
JsonObject p = (JsonObject) param; String key = p.getString("key");
String key = p.getString("key"); if (key != null && key.equals("client.version")) {
if (key != null && key.equals("client.version")) { shortClientVersion = p.getString("value");
shortClientVersion = p.getString("value"); }
} }
} }
} }
}
String clientVersion; String contextClientVersion;
String[] patterns = { String[] patterns = {
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"", "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
"innertube_context_client_version\":\"([0-9\\.]+?)\"", "innertube_context_client_version\":\"([0-9\\.]+?)\"",
"client.version=([0-9\\.]+)" "client.version=([0-9\\.]+)"
}; };
for (String pattern: patterns) { for (String pattern : patterns) {
try { try {
clientVersion = Parser.matchGroup1(pattern, html); contextClientVersion = Parser.matchGroup1(pattern, html);
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
} catch (Exception ignored) {} clientVersion = contextClientVersion;
} return clientVersion;
}
} catch (Exception ignored) {
}
}
if (shortClientVersion != null) return shortClientVersion; if (shortClientVersion != null) {
clientVersion = shortClientVersion;
return clientVersion;
}
} catch (Exception ignored) {}
throw new ParsingException("Could not get client version"); throw new ParsingException("Could not get client version");
} }