CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[YouTube] Avoid crashing by letting exceptions bubble up

This commit is contained in:
Mauricio Colli 2020-02-29 18:42:43 -03:00 committed by TobiGr
parent e9644e6216
commit 342bdbb852
2 changed files with 58 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/linkHandler/YoutubeParsingHelper.java
View File

@ -5,7 +5,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
@ -22,12 +21,7 @@ import java.net.URL;
import java.net.URLDecoder; import java.net.URLDecoder;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Calendar; import java.util.*;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader; import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP; import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
@ -177,21 +171,16 @@ public class YoutubeParsingHelper {
} }
} }
public static boolean isHardcodedClientVersionValid() throws IOException { public static boolean isHardcodedClientVersionValid() throws IOException, ExtractionException {
try { final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
final String url = "https://www.youtube.com/results?search_query=test&pbj=1";
Map<String, List<String>> headers = new HashMap<>(); Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1")); headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", headers.put("X-YouTube-Client-Version",
Collections.singletonList(HARDCODED_CLIENT_VERSION)); Collections.singletonList(HARDCODED_CLIENT_VERSION));
final String response = getDownloader().get(url, headers).responseBody(); final String response = getDownloader().get(url, headers).responseBody();
if (response.length() > 50) { // ensure to have a valid response
return true;
}
} catch (ReCaptchaException ignored) {}
return false; return response.length() > 50; // ensure to have a valid response
} }
/** /**
@ -199,7 +188,7 @@ public class YoutubeParsingHelper {
* @return * @return
* @throws ParsingException * @throws ParsingException
*/ */
public static String getClientVersion() throws ParsingException, IOException { public static String getClientVersion() throws IOException, ExtractionException {
if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion; if (clientVersion != null && !clientVersion.isEmpty()) return clientVersion;
if (isHardcodedClientVersionValid()) { if (isHardcodedClientVersionValid()) {
@ -207,62 +196,59 @@ public class YoutubeParsingHelper {
return clientVersion; return clientVersion;
} }
// Try extracting it from YouTube's website otherwise final String url = "https://www.youtube.com/results?search_query=test";
try { final String html = getDownloader().get(url).responseBody();
final String url = "https://www.youtube.com/results?search_query=test"; JsonObject initialData = getInitialData(html);
final String html = getDownloader().get(url).responseBody(); JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
JsonObject initialData = getInitialData(html); String shortClientVersion = null;
JsonArray serviceTrackingParams = initialData.getObject("responseContext").getArray("serviceTrackingParams");
String shortClientVersion = null;
// try to get version from initial data first // try to get version from initial data first
for (Object service : serviceTrackingParams) { for (Object service : serviceTrackingParams) {
JsonObject s = (JsonObject) service; JsonObject s = (JsonObject) service;
if (s.getString("service").equals("CSI")) { if (s.getString("service").equals("CSI")) {
JsonArray params = s.getArray("params"); JsonArray params = s.getArray("params");
for (Object param : params) { for (Object param : params) {
JsonObject p = (JsonObject) param; JsonObject p = (JsonObject) param;
String key = p.getString("key"); String key = p.getString("key");
if (key != null && key.equals("cver")) { if (key != null && key.equals("cver")) {
clientVersion = p.getString("value"); clientVersion = p.getString("value");
return clientVersion;
}
}
} else if (s.getString("service").equals("ECATCHER")) {
// fallback to get a shortened client version which does not contain the last two digits
JsonArray params = s.getArray("params");
for (Object param : params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("client.version")) {
shortClientVersion = p.getString("value");
}
}
}
}
String contextClientVersion;
String[] patterns = {
"INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
"client.version=([0-9\\.]+)"
};
for (String pattern : patterns) {
try {
contextClientVersion = Parser.matchGroup1(pattern, html);
if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
clientVersion = contextClientVersion;
return clientVersion; return clientVersion;
} }
} catch (Exception ignored) { }
} else if (s.getString("service").equals("ECATCHER")) {
// fallback to get a shortened client version which does not contain the last two digits
JsonArray params = s.getArray("params");
for (Object param : params) {
JsonObject p = (JsonObject) param;
String key = p.getString("key");
if (key != null && key.equals("client.version")) {
shortClientVersion = p.getString("value");
}
} }
} }
}
if (shortClientVersion != null) { String contextClientVersion;
clientVersion = shortClientVersion; String[] patterns = {
return clientVersion; "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"",
"innertube_context_client_version\":\"([0-9\\.]+?)\"",
"client.version=([0-9\\.]+)"
};
for (String pattern : patterns) {
try {
contextClientVersion = Parser.matchGroup1(pattern, html);
if (contextClientVersion != null && !contextClientVersion.isEmpty()) {
clientVersion = contextClientVersion;
return clientVersion;
}
} catch (Exception ignored) {
} }
} catch (Exception ignored) {} }
if (shortClientVersion != null) {
clientVersion = shortClientVersion;
return clientVersion;
}
throw new ParsingException("Could not get client version"); throw new ParsingException("Could not get client version");
} }

3
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelperTest.java
View File

@ -4,6 +4,7 @@ import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.DownloaderTestImpl; import org.schabi.newpipe.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper; import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
import java.io.IOException; import java.io.IOException;
@ -17,7 +18,7 @@ public class YoutubeParsingHelperTest {
} }
@Test @Test
public void testIsHardcodedClientVersionValid() throws IOException { public void testIsHardcodedClientVersionValid() throws IOException, ExtractionException {
assertTrue("Hardcoded client version is not valid anymore", assertTrue("Hardcoded client version is not valid anymore",
YoutubeParsingHelper.isHardcodedClientVersionValid()); YoutubeParsingHelper.isHardcodedClientVersionValid());
} }