CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #786 from XiangRongLin/throttling_resilience

Browse source 
[Youtube] Make throttling decryption more resilient to api change
This commit is contained in:
Stypox 2022-03-16 11:03:16 +01:00 committed by GitHub
commit 40aa5104b1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 83 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

97
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java
View file

@ -38,7 +38,9 @@ public class YoutubeThrottlingDecrypter {
private static final Pattern FUNCTION_NAME_PATTERN = Pattern.compile(
"b=a\\.get\\(\"n\"\\)\\)&&\\(b=(\\S+)\\(b\\),a\\.set\\(\"n\",b\\)");
private static final Map<String, String> nParams = new HashMap<>();
private static final Map<String, String> N_PARAMS_CACHE = new HashMap<>();
private static String FUNCTION;
private static String FUNCTION_NAME;
private final String functionName;
private final String function;
@ -49,6 +51,8 @@ public class YoutubeThrottlingDecrypter {
* is requested.
* </p>
* Otherwise use the no-arg constructor which uses a constant value.
*
* @deprecated Use static function instead
*/
public YoutubeThrottlingDecrypter(final String videoId) throws ParsingException {
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
@ -57,6 +61,9 @@ public class YoutubeThrottlingDecrypter {
function = parseDecodeFunction(playerJsCode, functionName);
}
/**
* @deprecated Use static function instead
*/
public YoutubeThrottlingDecrypter() throws ParsingException {
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode();
@ -64,27 +71,50 @@ public class YoutubeThrottlingDecrypter {
function = parseDecodeFunction(playerJsCode, functionName);
}
private String parseDecodeFunctionName(final String playerJsCode)
throws Parser.RegexException {
String functionName = Parser.matchGroup1(FUNCTION_NAME_PATTERN, playerJsCode);
final int arrayStartBrace = functionName.indexOf("[");
if (arrayStartBrace > 0) {
final String arrayVarName = functionName.substring(0, arrayStartBrace);
final String order = functionName.substring(
arrayStartBrace + 1, functionName.indexOf("]"));
final int arrayNum = Integer.parseInt(order);
final Pattern arrayPattern = Pattern.compile(
String.format("var %s=\\[(.+?)\\];", arrayVarName));
final String arrayStr = Parser.matchGroup1(arrayPattern, playerJsCode);
final String[] names = arrayStr.split(",");
functionName = names[arrayNum];
/**
* <p>
* The videoId is only used to fetch the decryption function.
* It can be a constant value of any existing video.
* A constant value is discouraged, because it could allow tracking.
*/
public static String apply(final String url, final String videoId) throws ParsingException {
if (containsNParam(url)) {
if (FUNCTION == null) {
final String playerJsCode = YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
FUNCTION_NAME = parseDecodeFunctionName(playerJsCode);
FUNCTION = parseDecodeFunction(playerJsCode, FUNCTION_NAME);
}
return functionName;
final String oldNParam = parseNParam(url);
final String newNParam = decryptNParam(FUNCTION, FUNCTION_NAME, oldNParam);
return replaceNParam(url, oldNParam, newNParam);
} else {
return url;
}
}
private static String parseDecodeFunctionName(final String playerJsCode)
throws Parser.RegexException {
String functionName = Parser.matchGroup1(FUNCTION_NAME_PATTERN, playerJsCode);
final int arrayStartBrace = functionName.indexOf("[");
if (arrayStartBrace > 0) {
final String arrayVarName = functionName.substring(0, arrayStartBrace);
final String order = functionName.substring(
arrayStartBrace + 1, functionName.indexOf("]"));
final int arrayNum = Integer.parseInt(order);
final Pattern arrayPattern = Pattern.compile(
String.format("var %s=\\[(.+?)\\];", arrayVarName));
final String arrayStr = Parser.matchGroup1(arrayPattern, playerJsCode);
final String[] names = arrayStr.split(",");
functionName = names[arrayNum];
}
return functionName;
}
@Nonnull
private String parseDecodeFunction(final String playerJsCode, final String functionName)
private static String parseDecodeFunction(final String playerJsCode, final String functionName)
throws Parser.RegexException {
try {
return parseWithParenthesisMatching(playerJsCode, functionName);
@ -94,49 +124,50 @@ public class YoutubeThrottlingDecrypter {
}
@Nonnull
private String parseWithParenthesisMatching(final String playerJsCode, final String functionName) {
private static String parseWithParenthesisMatching(final String playerJsCode, final String functionName) {
final String functionBase = functionName + "=function";
return functionBase + StringUtils.matchToClosingParenthesis(playerJsCode, functionBase) + ";";
}
@Nonnull
private String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException {
private static String parseWithRegex(final String playerJsCode, final String functionName) throws Parser.RegexException {
final Pattern functionPattern = Pattern.compile(functionName + "=function(.*?}};)\n",
Pattern.DOTALL);
return "function " + functionName + Parser.matchGroup1(functionPattern, playerJsCode);
}
public String apply(final String url) throws Parser.RegexException {
@Deprecated
public String apply(final String url) throws ParsingException {
if (containsNParam(url)) {
final String oldNParam = parseNParam(url);
final String newNParam = decryptNParam(oldNParam);
final String newNParam = decryptNParam(function, functionName, oldNParam);
return replaceNParam(url, oldNParam, newNParam);
} else {
return url;
}
}
private boolean containsNParam(final String url) {
private static boolean containsNParam(final String url) {
return Parser.isMatch(N_PARAM_PATTERN, url);
}
private String parseNParam(final String url) throws Parser.RegexException {
private static String parseNParam(final String url) throws Parser.RegexException {
return Parser.matchGroup1(N_PARAM_PATTERN, url);
}
private String decryptNParam(final String nParam) {
if (nParams.containsKey(nParam)) {
return nParams.get(nParam);
private static String decryptNParam(final String function, final String functionName, final String nParam) {
if (N_PARAMS_CACHE.containsKey(nParam)) {
return N_PARAMS_CACHE.get(nParam);
}
final String decryptedNParam = JavaScript.run(function, functionName, nParam);
nParams.put(nParam, decryptedNParam);
N_PARAMS_CACHE.put(nParam, decryptedNParam);
return decryptedNParam;
}
@Nonnull
private String replaceNParam(@Nonnull final String url,
final String oldValue,
final String newValue) {
private static String replaceNParam(@Nonnull final String url,
final String oldValue,
final String newValue) {
return url.replace(oldValue, newValue);
}
@ -144,13 +175,13 @@ public class YoutubeThrottlingDecrypter {
* @return the number of the cached "n" query parameters.
*/
public static int getCacheSize() {
return nParams.size();
return N_PARAMS_CACHE.size();
}
/**
* Clears all stored "n" query parameters.
*/
public static void clearCache() {
nParams.clear();
N_PARAMS_CACHE.clear();
}
}

25
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file

@ -482,14 +482,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public List<AudioStream> getAudioStreams() throws ExtractionException {
assertPageFetched();
final List<AudioStream> audioStreams = new ArrayList<>();
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
ItagItem.ItagType.AUDIO).entrySet()) {
final ItagItem itag = entry.getValue();
String url = entry.getKey();
url = throttlingDecrypter.apply(url);
final String url = tryDecryption(entry.getKey(), getId());
final AudioStream audioStream = new AudioStream(url, itag);
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
@ -507,14 +505,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public List<VideoStream> getVideoStreams() throws ExtractionException {
assertPageFetched();
final List<VideoStream> videoStreams = new ArrayList<>();
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try {
for (final Map.Entry<String, ItagItem> entry : getItags(FORMATS,
ItagItem.ItagType.VIDEO).entrySet()) {
final ItagItem itag = entry.getValue();
String url = entry.getKey();
url = throttlingDecrypter.apply(url);
final String url = tryDecryption(entry.getKey(), getId());
final VideoStream videoStream = new VideoStream(url, false, itag);
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
@ -532,14 +528,12 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public List<VideoStream> getVideoOnlyStreams() throws ExtractionException {
assertPageFetched();
final List<VideoStream> videoOnlyStreams = new ArrayList<>();
final YoutubeThrottlingDecrypter throttlingDecrypter = new YoutubeThrottlingDecrypter(getId());
try {
for (final Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS,
ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
final ItagItem itag = entry.getValue();
String url = entry.getKey();
url = throttlingDecrypter.apply(url);
final String url = tryDecryption(entry.getKey(), getId());
final VideoStream videoStream = new VideoStream(url, true, itag);
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
@ -553,6 +547,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return videoOnlyStreams;
}
/**
* Try to decrypt url and fallback to given url, because decryption is not
* always needed.
* This way a breaking change from YouTube does not result in a broken extractor.
*/
private String tryDecryption(final String url, final String videoId) {
try {
return YoutubeThrottlingDecrypter.apply(url, videoId);
} catch (final ParsingException e) {
return url;
}
}
@Override
@Nonnull
public List<SubtitlesStream> getSubtitlesDefault() throws ParsingException {

6
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypterTest.java
View file

@ -28,7 +28,7 @@ public class YoutubeThrottlingDecrypterTest {
for (final String videoId : videoIds) {
try {
final String decryptedUrl = new YoutubeThrottlingDecrypter(videoId).apply(encryptedUrl);
final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, videoId);
assertNotEquals(encryptedUrl, decryptedUrl);
} catch (EvaluatorException e) {
fail("Failed to extract n param decrypt function for video " + videoId + "\n" + e);
@ -40,7 +40,7 @@ public class YoutubeThrottlingDecrypterTest {
public void testDecode__success() throws ParsingException {
// URL extracted from browser with the dev tools
final String encryptedUrl = "https://r6---sn-4g5ednek.googlevideo.com/videoplayback?expire=1626562120&ei=6AnzYO_YBpql1gLGkb_IBQ&ip=127.0.0.1&id=o-ANhBEf36Z5h-8U9DDddtPDqtS0ZNwf0XJAAigudKI2uI&itag=278&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278&source=youtube&requiressl=yes&vprv=1&mime=video%2Fwebm&ns=TvecOReN0vPuXb3j_zq157IG&gir=yes&clen=2915100&dur=270.203&lmt=1608157174907785&keepalive=yes&fexp=24001373,24007246&c=WEB&txp=5535432&n=N9BWSTFT7vvBJrvQ&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&alr=yes&sig=AOq0QJ8wRQIgW6XnUDKPDSxiT0_KE_tDDMpcaCJl2Un5p0Fu9qZNQGkCIQDWxsDHi_s2BEmRqIbd1C5g_gzfihB7RZLsScKWNMwzzA%3D%3D&cpn=9r2yt3BqcYmeb2Yu&cver=2.20210716.00.00&redirect_counter=1&cm2rm=sn-4g5ezy7s&cms_redirect=yes&mh=Y5&mm=34&mn=sn-4g5ednek&ms=ltu&mt=1626540524&mv=m&mvi=6&pl=43&lsparams=mh,mm,mn,ms,mv,mvi,pl&lsig=AG3C_xAwRQIhAIUzxTn9Vw1-vm-_7OQ5-0h1M6AZsY9Bx1FlCCTeMICzAiADtGggbn4Znsrh2EnvyOsGnYdRGcbxn4mW9JMOQiInDQ%3D%3D&range=259165-480735&rn=11&rbuf=20190";
final String decryptedUrl = new YoutubeThrottlingDecrypter().apply(encryptedUrl);
final String decryptedUrl = YoutubeThrottlingDecrypter.apply(encryptedUrl, "jE1USQrs1rw");
// The cipher function changes over time, so we just check if the n param changed.
assertNotEquals(encryptedUrl, decryptedUrl);
}
@ -48,7 +48,7 @@ public class YoutubeThrottlingDecrypterTest {
@Test
public void testDecode__noNParam__success() throws ParsingException {
final String noNParamUrl = "https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?expire=1626553257&ei=SefyYPmIFoKT1wLtqbjgCQ&ip=127.0.0.1&id=o-AIT5xGifsaEAdEOAb5vd06J9VNtm-KHHolnaZRGPjHZi&itag=140&source=youtube&requiressl=yes&mh=xO&mm=31%2C29&mn=sn-4g5ednsz%2Csn-4g5e6nsr&ms=au%2Crdu&mv=m&mvi=5&pl=24&initcwndbps=1322500&vprv=1&mime=audio%2Fmp4&ns=cA2SS5atEe0mH8tMwGTO4RIG&gir=yes&clen=3009275&dur=185.898&lmt=1626356984653961&mt=1626531173&fvip=5&keepalive=yes&fexp=24001373%2C24007246&beids=23886212&c=WEB&txp=6411222&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&lsig=AG3C_xAwRgIhAPueRlTutSlzPafxrqBmgZz5m7-Zfbw3QweDp3j4XO9SAiEA5tF7_ZCJFKmS-D6I1jlUURjpjoiTbsYyKuarV4u6E8Y%3D&sig=AOq0QJ8wRQIgRD_4WwkPeTEKGVSQqPsznMJGqq4nVJ8o1ChGBCgi4Y0CIQCZT3tI40YLKBWJCh2Q7AlvuUIpN0ficzdSElLeQpJdrw==";
final String decrypted = new YoutubeThrottlingDecrypter().apply(noNParamUrl);
final String decrypted = YoutubeThrottlingDecrypter.apply(noNParamUrl, "jE1USQrs1rw");
assertEquals(noNParamUrl, decrypted);
}