CypherpunkSamurai/NewPipeExtractor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

polish strings

Browse source
This commit is contained in:
bopol 2020-10-26 16:32:39 +01:00
parent 14c0c37d64
commit 01f49e8f66
1 changed files with 42 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file

@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParser;
import org.mozilla.javascript.Context; import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function; import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject; import org.mozilla.javascript.ScriptableObject;
@ -36,6 +35,8 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
@ -49,9 +50,6 @@ import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
@ -84,8 +82,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
// Exceptions // Exceptions
//////////////////////////////////////////////////////////////////////////*/ //////////////////////////////////////////////////////////////////////////*/
public class DecryptException extends ParsingException { public class DeobfuscateException extends ParsingException {
DecryptException(String message, Throwable cause) { DeobfuscateException(String message, Throwable cause) {
super(message, cause); super(message, cause);
} }
} }
@ -156,12 +154,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en")); TimeAgoParser timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.fromLocalizationCode("en"));
Calendar parsedTime = timeAgoParser.parse(time).date(); Calendar parsedTime = timeAgoParser.parse(time).date();
return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime()); return new SimpleDateFormat("yyyy-MM-dd").format(parsedTime.getTime());
} catch (Exception ignored) {} } catch (Exception ignored) {
}
try { // Premiered Feb 21, 2020 try { // Premiered Feb 21, 2020
Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time); Date d = new SimpleDateFormat("MMM dd, YYYY", Locale.ENGLISH).parse(time);
return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime()); return new SimpleDateFormat("yyyy-MM-dd").format(d.getTime());
} catch (Exception ignored) {} } catch (Exception ignored) {
}
} }
try { try {
@ -169,7 +169,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse( Date d = new SimpleDateFormat("dd MMM yyyy", Locale.ENGLISH).parse(
getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText"))); getTextFromObject(getVideoPrimaryInfoRenderer().getObject("dateText")));
return new SimpleDateFormat("yyyy-MM-dd").format(d); return new SimpleDateFormat("yyyy-MM-dd").format(d);
} catch (Exception ignored) {} } catch (Exception ignored) {
}
throw new ParsingException("Could not get upload date"); throw new ParsingException("Could not get upload date");
} }
@ -368,7 +369,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
try { try {
uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner") uploaderName = getTextFromObject(getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getObject("title")); .getObject("videoOwnerRenderer").getObject("title"));
} catch (ParsingException ignored) { } } catch (ParsingException ignored) {
}
if (isNullOrEmpty(uploaderName)) { if (isNullOrEmpty(uploaderName)) {
uploaderName = playerResponse.getObject("videoDetails").getString("author"); uploaderName = playerResponse.getObject("videoDetails").getString("author");
@ -436,11 +438,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
if (!dashManifestUrl.contains("/signature/")) { if (!dashManifestUrl.contains("/signature/")) {
String encryptedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl); String obfuscatedSig = Parser.matchGroup1("/s/([a-fA-F0-9\\.]+)", dashManifestUrl);
String decryptedSig; String deobfuscatedSig;
decryptedSig = decryptSignature(encryptedSig, decryptionCode); deobfuscatedSig = deobfuscateSignature(obfuscatedSig, deobfuscationCode);
dashManifestUrl = dashManifestUrl.replace("/s/" + encryptedSig, "/signature/" + decryptedSig); dashManifestUrl = dashManifestUrl.replace("/s/" + obfuscatedSig, "/signature/" + deobfuscatedSig);
} }
return dashManifestUrl; return dashManifestUrl;
@ -630,7 +632,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private static final String FORMATS = "formats"; private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats"; private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String HTTPS = "https:"; private static final String HTTPS = "https:";
private static final String DECRYPTION_FUNC_NAME = "decrypt"; private static final String DEOBFUSCATION_FUNC_NAME = "decrypt";
private final static String[] REGEXES = { private final static String[] REGEXES = {
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)", "(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
@ -640,7 +642,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
"\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(" "\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\("
}; };
private volatile String decryptionCode = ""; private volatile String deobfuscationCode = "";
@Override @Override
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException { public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
@ -695,8 +697,8 @@ public class YoutubeStreamExtractor extends StreamExtractor {
throw new ContentNotAvailableException("Got error: \"" + reason + "\""); throw new ContentNotAvailableException("Got error: \"" + reason + "\"");
} }
if (decryptionCode.isEmpty()) { if (deobfuscationCode.isEmpty()) {
decryptionCode = loadDecryptionCode(playerUrl); deobfuscationCode = loadDeobfuscationCode(playerUrl);
} }
if (subtitlesInfos.isEmpty()) { if (subtitlesInfos.isEmpty()) {
@ -716,7 +718,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
private String getPlayerUrl(final JsonObject playerConfig) throws ParsingException { private String getPlayerUrl(final JsonObject playerConfig) throws ParsingException {
// The Youtube service needs to be initialized by downloading the // The Youtube service needs to be initialized by downloading the
// js-Youtube-player. This is done in order to get the algorithm // js-Youtube-player. This is done in order to get the algorithm
// for decrypting cryptic signatures inside certain stream URLs. // for deobfuscating cryptic signatures inside certain stream URLs.
final String playerUrl = playerConfig.getObject("assets").getString("js"); final String playerUrl = playerConfig.getObject("assets").getString("js");
if (playerUrl == null) { if (playerUrl == null) {
@ -768,11 +770,11 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} catch (IOException e) { } catch (IOException e) {
throw new ParsingException( throw new ParsingException(
"Could load decryption code form restricted video for the Youtube service.", e); "Could load deobfuscation code form restricted video for the Youtube service.", e);
} }
} }
private String loadDecryptionCode(String playerUrl) throws DecryptException { private String loadDeobfuscationCode(String playerUrl) throws DeobfuscateException {
try { try {
Downloader downloader = NewPipe.getDownloader(); Downloader downloader = NewPipe.getDownloader();
if (!playerUrl.contains("https://youtube.com")) { if (!playerUrl.contains("https://youtube.com")) {
@ -782,49 +784,49 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody(); final String playerCode = downloader.get(playerUrl, getExtractorLocalization()).responseBody();
final String decryptionFunctionName = getDecryptionFuncName(playerCode); final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
final String functionPattern = "(" final String functionPattern = "("
+ decryptionFunctionName.replace("$", "\\$") + deobfuscationFunctionName.replace("$", "\\$")
+ "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})"; + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
final String decryptionFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";"; final String deobfuscateFunction = "var " + Parser.matchGroup1(functionPattern, playerCode) + ";";
final String helperObjectName = final String helperObjectName =
Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", decryptionFunction); Parser.matchGroup1(";([A-Za-z0-9_\\$]{2})\\...\\(", deobfuscateFunction);
final String helperPattern = final String helperPattern =
"(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)"; "(var " + helperObjectName.replace("$", "\\$") + "=\\{.+?\\}\\};)";
final String helperObject = final String helperObject =
Parser.matchGroup1(helperPattern, playerCode.replace("\n", "")); Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));
final String callerFunction = final String callerFunction =
"function " + DECRYPTION_FUNC_NAME + "(a){return " + decryptionFunctionName + "(a);}"; "function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}";
return helperObject + decryptionFunction + callerFunction; return helperObject + deobfuscateFunction + callerFunction;
} catch (IOException ioe) { } catch (IOException ioe) {
throw new DecryptException("Could not load decrypt function", ioe); throw new DeobfuscateException("Could not load deobfuscate function", ioe);
} catch (Exception e) { } catch (Exception e) {
throw new DecryptException("Could not parse decrypt function ", e); throw new DeobfuscateException("Could not parse deobfuscate function ", e);
} }
} }
private String decryptSignature(String encryptedSig, String decryptionCode) throws DecryptException { private String deobfuscateSignature(String obfuscatedSig, String deobfuscationCode) throws DeobfuscateException {
final Context context = Context.enter(); final Context context = Context.enter();
context.setOptimizationLevel(-1); context.setOptimizationLevel(-1);
final Object result; final Object result;
try { try {
final ScriptableObject scope = context.initSafeStandardObjects(); final ScriptableObject scope = context.initSafeStandardObjects();
context.evaluateString(scope, decryptionCode, "decryptionCode", 1, null); context.evaluateString(scope, deobfuscationCode, "decryptionCode", 1, null);
final Function decryptionFunc = (Function) scope.get("decrypt", scope); final Function deobfuscateFunc = (Function) scope.get("decrypt", scope);
result = decryptionFunc.call(context, scope, scope, new Object[]{encryptedSig}); result = deobfuscateFunc.call(context, scope, scope, new Object[]{obfuscatedSig});
} catch (Exception e) { } catch (Exception e) {
throw new DecryptException("Could not get decrypt signature", e); throw new DeobfuscateException("Could not get deobfuscate signature", e);
} finally { } finally {
Context.exit(); Context.exit();
} }
return result == null ? "" : result.toString(); return result == null ? "" : result.toString();
} }
private String getDecryptionFuncName(final String playerCode) throws DecryptException { private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null; Parser.RegexException exception = null;
for (final String regex : REGEXES) { for (final String regex : REGEXES) {
try { try {
@ -835,7 +837,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
} }
} }
} }
throw new DecryptException("Could not find decrypt function with any of the given patterns.", exception); throw new DeobfuscateException("Could not find deobfuscate function with any of the given patterns.", exception);
} }
@Nonnull @Nonnull
@ -989,18 +991,19 @@ public class YoutubeStreamExtractor extends StreamExtractor {
if (formatData.has("url")) { if (formatData.has("url")) {
streamUrl = formatData.getString("url"); streamUrl = formatData.getString("url");
} else { } else {
// this url has an encrypted signature // this url has an obfuscated signature
final String cipherString = formatData.has("cipher") final String cipherString = formatData.has("cipher")
? formatData.getString("cipher") ? formatData.getString("cipher")
: formatData.getString("signatureCipher"); : formatData.getString("signatureCipher");
final Map<String, String> cipher = Parser.compatParseMap(cipherString); final Map<String, String> cipher = Parser.compatParseMap(cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ decryptSignature(cipher.get("s"), decryptionCode); + deobfuscateSignature(cipher.get("s"), deobfuscationCode);
} }
urlAndItags.put(streamUrl, itagItem); urlAndItags.put(streamUrl, itagItem);
} }
} catch (UnsupportedEncodingException ignored) {} } catch (UnsupportedEncodingException ignored) {
}
} }
} }