diff --git a/build.gradle b/build.gradle index 5949672..535b051 100644 --- a/build.gradle +++ b/build.gradle @@ -15,12 +15,12 @@ dependencies { implementation 'commons-io:commons-io:2.14.0' implementation 'it.unimi.dsi:fastutil-core:8.5.12' implementation 'commons-codec:commons-codec:1.16.0' - implementation 'org.bouncycastle:bcprov-jdk15on:1.70' + implementation 'org.bouncycastle:bcprov-jdk18on:1.77' implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:8cf9a4aef0919df2ef1baafd30ab5bfefefc0844' implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' - implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2' - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2' + implementation 'com.fasterxml.jackson.core:jackson-core:2.16.0' + implementation 'com.fasterxml.jackson.core:jackson-annotations:2.16.0' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' implementation 'com.rometools:rome:2.1.0' implementation 'com.rometools:rome-modules:2.1.0' implementation 'org.jsoup:jsoup:1.16.1' @@ -33,8 +33,8 @@ dependencies { implementation 'org.postgresql:postgresql:42.6.0' implementation 'org.hibernate:hibernate-core:6.3.1.Final' implementation 'org.hibernate:hibernate-hikaricp:6.3.1.Final' - implementation 'org.liquibase:liquibase-core:4.23.2' - implementation('org.liquibase.ext:liquibase-yugabytedb:4.23.2') { exclude group: 'org.liquibase' } + implementation 'org.liquibase:liquibase-core:4.25.0' + implementation('org.liquibase.ext:liquibase-yugabytedb:4.24.0') { exclude group: 'org.liquibase' } implementation 'com.zaxxer:HikariCP:5.0.1' implementation 'org.springframework.security:spring-security-crypto:6.1.4' implementation 'commons-logging:commons-logging:1.2' diff --git a/config.properties b/config.properties index 9b5fed0..4b99288 100644 --- a/config.properties +++ b/config.properties @@ -6,6 +6,9 @@ HTTP_WORKERS:2 # Proxy PROXY_PART:https://pipedproxy-cdg.kavin.rocks +# Proxy Hash Secret +#PROXY_HASH_SECRET:INSERT_HERE + # Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080 #REQWEST_PROXY: socks5://127.0.0.1:1080 # Optional proxy username and password diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index b80d31f..934dc8c 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -13,6 +13,7 @@ import me.kavin.piped.utils.obj.db.PlaylistVideo; import me.kavin.piped.utils.obj.db.PubSub; import me.kavin.piped.utils.obj.db.Video; import okhttp3.OkHttpClient; +import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.hibernate.Session; import org.hibernate.StatelessSession; import org.schabi.newpipe.extractor.NewPipe; @@ -21,6 +22,7 @@ import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; +import java.security.Security; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; @@ -33,6 +35,9 @@ public class Main { public static void main(String[] args) throws Exception { + Security.setProperty("crypto.policy", "unlimited"); + Security.addProvider(new BouncyCastleProvider()); + NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT, Multithreading.getCachedExecutor()); YoutubeStreamExtractor.forceFetchAndroidClient(true); YoutubeStreamExtractor.forceFetchIosClient(true); diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index 516ece6..840069a 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -25,7 +25,9 @@ import java.io.File; import java.io.FileReader; import java.net.InetSocketAddress; import java.net.ProxySelector; +import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Optional; import java.util.Properties; import java.util.regex.Pattern; @@ -40,6 +42,8 @@ public class Constants { public static final String IMAGE_PROXY_PART; + public static final byte[] PROXY_HASH_SECRET; + public static final String CAPTCHA_BASE_URL, CAPTCHA_API_KEY; public static final StreamingService YOUTUBE_SERVICE; @@ -127,6 +131,7 @@ public class Constants { String.valueOf(Runtime.getRuntime().availableProcessors())); PROXY_PART = getProperty(prop, "PROXY_PART"); IMAGE_PROXY_PART = getProperty(prop, "IMAGE_PROXY_PART", PROXY_PART); + PROXY_HASH_SECRET = Optional.ofNullable(getProperty(prop, "PROXY_HASH_SECRET")).map(s -> s.getBytes(StandardCharsets.UTF_8)).orElse(null); CAPTCHA_BASE_URL = getProperty(prop, "CAPTCHA_BASE_URL"); CAPTCHA_API_KEY = getProperty(prop, "CAPTCHA_API_KEY"); PUBLIC_URL = getProperty(prop, "API_URL"); diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index a36f3e9..7a8184a 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -1,6 +1,7 @@ package me.kavin.piped.utils; import me.kavin.piped.consts.Constants; +import org.apache.commons.codec.binary.Hex; import org.apache.commons.lang3.StringUtils; import org.schabi.newpipe.extractor.Image; @@ -9,7 +10,14 @@ import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Comparator; import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import static me.kavin.piped.consts.Constants.PROXY_HASH_SECRET; public class URLUtils { @@ -65,13 +73,84 @@ public class URLUtils { boolean hasQuery = query != null; + Comparator> listComparator = (o1, o2) -> { + for (int i = 0; i < Math.min(o1.size(), o2.size()); i++) { + int result = o1.get(i).compareTo(o2.get(i)); + if (result != 0) { + return result; + } + } + return Integer.compare(o1.size(), o2.size()); // compare list sizes if all elements are equal + }; + + Set> queryPairs = new TreeSet<>(listComparator); + + if (hasQuery) { + String[] pairs = query.split("&"); + + for (String pair : pairs) { + int idx = pair.indexOf("="); + queryPairs.add(List.of( + silentDecode(pair.substring(0, idx)), + silentDecode(pair.substring(idx + 1)) + )); + } + } + + // look for host param, and add it if it doesn't exist + boolean hasHost = false; + for (List pair : queryPairs) { + if (pair.get(0).equals("host")) { + hasHost = true; + break; + } + } + if (!hasHost) { + queryPairs.add(List.of("host", host)); + } + String path = url.getPath(); if (path.contains("=")) { path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw"); } - return proxy + path + (hasQuery ? "?" + query + "&host=" : "?host=") + silentEncode(host); + if (PROXY_HASH_SECRET != null) + try { + MessageDigest md = MessageDigest.getInstance("BLAKE3-256"); + for (List pair : queryPairs) { + md.update(pair.get(0).getBytes(StandardCharsets.UTF_8)); + md.update(pair.get(1).getBytes(StandardCharsets.UTF_8)); + } + + md.update(path.getBytes(StandardCharsets.UTF_8)); + + md.update(PROXY_HASH_SECRET); + + queryPairs.add(List.of("qhash", Hex.encodeHexString(md.digest()).substring(0, 8))); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + } + + String newUrl = proxy + path; + + StringBuilder qstring = null; + + for (List pair : queryPairs) { + if (qstring == null) { + qstring = new StringBuilder(); + } else { + qstring.append("&"); + } + + qstring.append(silentEncode(pair.get(0))); + qstring.append("="); + qstring.append(silentEncode(pair.get(1))); + } + + newUrl += "?" + qstring; + + return newUrl; } }