Implement blake3 proxy query hash

This commit is contained in:
Kavin 2023-11-20 05:17:29 +00:00
parent 2c5f6ed6af
commit 268e599421
No known key found for this signature in database
GPG Key ID: 6E4598CA5C92C41F
5 changed files with 97 additions and 2 deletions

View File

@ -15,7 +15,7 @@ dependencies {
implementation 'commons-io:commons-io:2.14.0'
implementation 'it.unimi.dsi:fastutil-core:8.5.12'
implementation 'commons-codec:commons-codec:1.16.0'
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
implementation 'org.bouncycastle:bcprov-jdk18on:1.77'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:8cf9a4aef0919df2ef1baafd30ab5bfefefc0844'
implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7'
implementation 'com.fasterxml.jackson.core:jackson-core:2.16.0'

View File

@ -6,6 +6,9 @@ HTTP_WORKERS:2
# Proxy
PROXY_PART:https://pipedproxy-cdg.kavin.rocks
# Proxy Hash Secret
#PROXY_HASH_SECRET:INSERT_HERE
# Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080
#REQWEST_PROXY: socks5://127.0.0.1:1080
# Optional proxy username and password

View File

@ -13,6 +13,7 @@ import me.kavin.piped.utils.obj.db.PlaylistVideo;
import me.kavin.piped.utils.obj.db.PubSub;
import me.kavin.piped.utils.obj.db.Video;
import okhttp3.OkHttpClient;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.hibernate.Session;
import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.NewPipe;
@ -21,6 +22,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import java.security.Security;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue;
@ -33,6 +35,9 @@ public class Main {
public static void main(String[] args) throws Exception {
Security.setProperty("crypto.policy", "unlimited");
Security.addProvider(new BouncyCastleProvider());
NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT, Multithreading.getCachedExecutor());
YoutubeStreamExtractor.forceFetchAndroidClient(true);
YoutubeStreamExtractor.forceFetchIosClient(true);

View File

@ -25,7 +25,9 @@ import java.io.File;
import java.io.FileReader;
import java.net.InetSocketAddress;
import java.net.ProxySelector;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.regex.Pattern;
@ -40,6 +42,8 @@ public class Constants {
public static final String IMAGE_PROXY_PART;
public static final byte[] PROXY_HASH_SECRET;
public static final String CAPTCHA_BASE_URL, CAPTCHA_API_KEY;
public static final StreamingService YOUTUBE_SERVICE;
@ -127,6 +131,7 @@ public class Constants {
String.valueOf(Runtime.getRuntime().availableProcessors()));
PROXY_PART = getProperty(prop, "PROXY_PART");
IMAGE_PROXY_PART = getProperty(prop, "IMAGE_PROXY_PART", PROXY_PART);
PROXY_HASH_SECRET = Optional.ofNullable(getProperty(prop, "PROXY_HASH_SECRET")).map(s -> s.getBytes(StandardCharsets.UTF_8)).orElse(null);
CAPTCHA_BASE_URL = getProperty(prop, "CAPTCHA_BASE_URL");
CAPTCHA_API_KEY = getProperty(prop, "CAPTCHA_API_KEY");
PUBLIC_URL = getProperty(prop, "API_URL");

View File

@ -1,6 +1,7 @@
package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang3.StringUtils;
import org.schabi.newpipe.extractor.Image;
@ -9,7 +10,14 @@ import java.net.URL;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import static me.kavin.piped.consts.Constants.PROXY_HASH_SECRET;
public class URLUtils {
@ -65,13 +73,87 @@ public class URLUtils {
boolean hasQuery = query != null;
Comparator<List<String>> listComparator = (o1, o2) -> {
for (int i = 0; i < Math.min(o1.size(), o2.size()); i++) {
int result = o1.get(i).compareTo(o2.get(i));
if (result != 0) {
return result;
}
}
return Integer.compare(o1.size(), o2.size()); // compare list sizes if all elements are equal
};
Set<List<String>> queryPairs = new TreeSet<>(listComparator);
if (hasQuery) {
String[] pairs = query.split("&");
for (String pair : pairs) {
int idx = pair.indexOf("=");
queryPairs.add(List.of(
silentDecode(pair.substring(0, idx)),
silentDecode(pair.substring(idx + 1))
));
}
}
// look for host param, and add it if it doesn't exist
boolean hasHost = false;
for (List<String> pair : queryPairs) {
if (pair.get(0).equals("host")) {
hasHost = true;
break;
}
}
if (!hasHost) {
queryPairs.add(List.of("host", host));
}
if (PROXY_HASH_SECRET != null)
try {
MessageDigest md = MessageDigest.getInstance("BLAKE3-256");
for (List<String> pair : queryPairs) {
md.update(pair.get(0).getBytes(StandardCharsets.UTF_8));
md.update(pair.get(1).getBytes(StandardCharsets.UTF_8));
}
md.update(PROXY_HASH_SECRET);
queryPairs.add(List.of("qhash", Hex.encodeHexString(md.digest()).substring(0, 8)));
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
String path = url.getPath();
if (path.contains("=")) {
path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw");
}
return proxy + path + (hasQuery ? "?" + query + "&host=" : "?host=") + silentEncode(host);
String newUrl = proxy + path;
StringBuilder qstring = null;
if (hasQuery) {
for (List<String> pair : queryPairs) {
if (qstring == null) {
qstring = new StringBuilder();
} else {
qstring.append("&");
}
qstring.append(pair.get(0));
qstring.append("=");
qstring.append(pair.get(1));
}
}
if (qstring != null) {
newUrl += "?" + qstring;
}
return newUrl;
}
}