CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #891 from Theta-Dev/fix-throttling-decrypter

Browse source 
[YouTube] Fix extraction of more complex nsig functions
This commit is contained in:
AudricV 2022-08-12 18:24:21 +02:00 committed by GitHub
commit 6a885ef5ab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 112 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeThrottlingDecrypter.java
View file

@ -12,24 +12,24 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* YouTube's streaming URLs of HTML5 clients are protected with a cipher, which modifies their
* {@code n} query parameter.
*
* <p>
* YouTube's media is protected with a cipher,
* which modifies the "n" query parameter of it's video playback urls.
* This class handles extracting that "n" query parameter,
* applying the cipher on it and returning the resulting url which is not throttled.
* This class handles extracting that {@code n} query parameter, applying the cipher on it and
* returning the resulting URL which is not throttled.
* </p>
*
* <pre>
* https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=VVF2xyZLVRZZxHXZ&amp;other=other
* </pre>
* becomes
* <pre>
* https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=iHywZkMipkszqA&amp;other=other
* </pre>
* <br>
* <p>
* Decoding the "n" parameter is time intensive. For this reason, the results are cached.
* The cache can be cleared using {@link #clearCache()}
* For instance,
* {@code https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=VVF2xyZLVRZZxHXZ&other=other}
* becomes
* {@code https://r5---sn-4g5ednsz.googlevideo.com/videoplayback?n=iHywZkMipkszqA&other=other}.
* </p>
*
* <p>
* Decoding the {@code n} parameter is time intensive. For this reason, the results are cached.
* The cache can be cleared using {@link #clearCache()}.
* </p>
*
*/
@ -73,13 +73,35 @@ public class YoutubeThrottlingDecrypter {
}
/**
* Try to decrypt a YouTube streaming URL protected with a throttling parameter.
*
* <p>
* The videoId is only used to fetch the decryption function.
* It can be a constant value of any existing video.
* A constant value is discouraged, because it could allow tracking.
* If the streaming URL provided doesn't contain a throttling parameter, it is returned as it
* is; otherwise, the encrypted value is decrypted and this value is replaced by the decrypted
* one.
* </p>
*
* <p>
* If the JavaScript code has been not extracted, it is extracted with the given video ID using
* {@link YoutubeJavaScriptExtractor#extractJavaScriptCode(String)}.
* </p>
*
* @param streamingUrl The streaming URL to decrypt, if needed.
* @param videoId A video ID, used to fetch the JavaScript code to get the decryption
* function. It can be a constant value of any existing video, but a
* constant value is discouraged, because it could allow tracking.
* @return A streaming URL with the decrypted parameter or the streaming URL itself if no
* throttling parameter has been found
* @throws ParsingException If the streaming URL contains a throttling parameter and its
* decryption failed
*/
public static String apply(final String url, final String videoId) throws ParsingException {
if (containsNParam(url)) {
public static String apply(@Nonnull final String streamingUrl,
@Nonnull final String videoId) throws ParsingException {
if (!containsNParam(streamingUrl)) {
return streamingUrl;
}
try {
if (FUNCTION == null) {
final String playerJsCode
= YoutubeJavaScriptExtractor.extractJavaScriptCode(videoId);
@ -88,11 +110,11 @@ public class YoutubeThrottlingDecrypter {
FUNCTION = parseDecodeFunction(playerJsCode, FUNCTION_NAME);
}
final String oldNParam = parseNParam(url);
final String oldNParam = parseNParam(streamingUrl);
final String newNParam = decryptNParam(FUNCTION, FUNCTION_NAME, oldNParam);
return replaceNParam(url, oldNParam, newNParam);
} else {
return url;
return replaceNParam(streamingUrl, oldNParam, newNParam);
} catch (final Exception e) {
throw new ParsingException("Could not parse, decrypt or replace n parameter", e);
}
}

16
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file

@ -602,15 +602,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
/**
* Try to decrypt url and fallback to given url, because decryption is not
* always needed.
* Try to decrypt a streaming URL and fallback to the given URL, because decryption may fail if
* YouTube do breaking changes.
*
* <p>
* This way a breaking change from YouTube does not result in a broken extractor.
* </p>
*
* @param streamingUrl the streaming URL to decrypt with {@link YoutubeThrottlingDecrypter}
* @param videoId the video ID to use when extracting JavaScript player code, if needed
*/
private String tryDecryptUrl(final String url, final String videoId) {
private String tryDecryptUrl(final String streamingUrl, final String videoId) {
try {
return YoutubeThrottlingDecrypter.apply(url, videoId);
return YoutubeThrottlingDecrypter.apply(streamingUrl, videoId);
} catch (final ParsingException e) {
return url;
return streamingUrl;
}
}

50
extractor/src/main/java/org/schabi/newpipe/extractor/utils/StringUtils.java
View file

@ -23,14 +23,13 @@ public final class StringUtils {
}
startIndex += start.length();
int endIndex = startIndex;
while (string.charAt(endIndex) != '{') {
++endIndex;
}
int endIndex = findNextParenthesis(string, startIndex, true);
++endIndex;
int openParenthesis = 1;
while (openParenthesis > 0) {
endIndex = findNextParenthesis(string, endIndex, false);
switch (string.charAt(endIndex)) {
case '{':
++openParenthesis;
@ -46,4 +45,47 @@ public final class StringUtils {
return string.substring(startIndex, endIndex);
}
private static int findNextParenthesis(@Nonnull final String string,
final int offset,
final boolean onlyOpen) {
boolean lastEscaped = false;
char quote = ' ';
for (int i = offset; i < string.length(); i++) {
boolean thisEscaped = false;
final char c = string.charAt(i);
switch (c) {
case '{':
if (quote == ' ') {
return i;
}
break;
case '}':
if (!onlyOpen && quote == ' ') {
return i;
}
break;
case '\\':
if (!lastEscaped) {
thisEscaped = true;
}
break;
case '\'':
case '"':
if (!lastEscaped) {
if (quote == ' ') {
quote = c;
} else if (quote == c) {
quote = ' ';
}
}
}
lastEscaped = thisEscaped;
}
return -1;
}
}

10
extractor/src/test/java/org/schabi/newpipe/extractor/utils/StringUtilsTest.java
View file

@ -58,4 +58,14 @@ public class StringUtilsTest {
assertEquals(expected, substring);
}
@Test
void find_closing_with_quotes() {
final String expected = "{return \",}\\\"/\"}";
final String string = "function(d){return \",}\\\"/\"}";
final String substring = matchToClosingParenthesis(string, "function(d)");
assertEquals(expected, substring);
}
}