TeamPiped-mirror
/
Piped-Backend
mirror of https://github.com/TeamPiped/Piped-Backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into add-back-hsqldb

This commit is contained in:
Andrea Spacca 2023-11-21 15:55:53 +09:00
parent cb17f9bdc8 a977355ebf
commit 0348fa0d2e
5 changed files with 99 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
build.gradle
View File

@ -15,12 +15,12 @@ dependencies {
implementation 'commons-io:commons-io:2.14.0'
implementation 'it.unimi.dsi:fastutil-core:8.5.12'
implementation 'commons-codec:commons-codec:1.16.0'
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
implementation 'org.bouncycastle:bcprov-jdk18on:1.77'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:8cf9a4aef0919df2ef1baafd30ab5bfefefc0844'
implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7'
implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
implementation 'com.fasterxml.jackson.core:jackson-core:2.16.0'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.16.0'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0'
implementation 'com.rometools:rome:2.1.0'
implementation 'com.rometools:rome-modules:2.1.0'
implementation 'org.jsoup:jsoup:1.16.1'
@ -33,8 +33,8 @@ dependencies {
implementation 'org.postgresql:postgresql:42.6.0'
implementation 'org.hibernate:hibernate-core:6.3.1.Final'
implementation 'org.hibernate:hibernate-hikaricp:6.3.1.Final'
implementation 'org.liquibase:liquibase-core:4.23.2'
implementation('org.liquibase.ext:liquibase-yugabytedb:4.23.2') { exclude group: 'org.liquibase' }
implementation 'org.liquibase:liquibase-core:4.25.0'
implementation('org.liquibase.ext:liquibase-yugabytedb:4.24.0') { exclude group: 'org.liquibase' }
implementation 'com.zaxxer:HikariCP:5.0.1'
implementation 'org.springframework.security:spring-security-crypto:6.1.4'
implementation 'commons-logging:commons-logging:1.2'

3
config.properties
View File

@ -6,6 +6,9 @@ HTTP_WORKERS:2
# Proxy
PROXY_PART:https://pipedproxy-cdg.kavin.rocks
# Proxy Hash Secret
#PROXY_HASH_SECRET:INSERT_HERE
# Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080
#REQWEST_PROXY: socks5://127.0.0.1:1080
# Optional proxy username and password

5
src/main/java/me/kavin/piped/Main.java
View File

@ -13,6 +13,7 @@ import me.kavin.piped.utils.obj.db.PlaylistVideo;
import me.kavin.piped.utils.obj.db.PubSub;
import me.kavin.piped.utils.obj.db.Video;
import okhttp3.OkHttpClient;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.hibernate.Session;
import org.hibernate.StatelessSession;
import org.schabi.newpipe.extractor.NewPipe;
@ -21,6 +22,7 @@ import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import java.security.Security;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentLinkedQueue;
@ -33,6 +35,9 @@ public class Main {
public static void main(String[] args) throws Exception {
Security.setProperty("crypto.policy", "unlimited");
Security.addProvider(new BouncyCastleProvider());
NewPipe.init(new DownloaderImpl(), new Localization("en", "US"), ContentCountry.DEFAULT, Multithreading.getCachedExecutor());
YoutubeStreamExtractor.forceFetchAndroidClient(true);
YoutubeStreamExtractor.forceFetchIosClient(true);

5
src/main/java/me/kavin/piped/consts/Constants.java
View File

@ -25,7 +25,9 @@ import java.io.File;
import java.io.FileReader;
import java.net.InetSocketAddress;
import java.net.ProxySelector;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.regex.Pattern;
@ -40,6 +42,8 @@ public class Constants {
public static final String IMAGE_PROXY_PART;
public static final byte[] PROXY_HASH_SECRET;
public static final String CAPTCHA_BASE_URL, CAPTCHA_API_KEY;
public static final StreamingService YOUTUBE_SERVICE;
@ -127,6 +131,7 @@ public class Constants {
String.valueOf(Runtime.getRuntime().availableProcessors()));
PROXY_PART = getProperty(prop, "PROXY_PART");
IMAGE_PROXY_PART = getProperty(prop, "IMAGE_PROXY_PART", PROXY_PART);
PROXY_HASH_SECRET = Optional.ofNullable(getProperty(prop, "PROXY_HASH_SECRET")).map(s -> s.getBytes(StandardCharsets.UTF_8)).orElse(null);
CAPTCHA_BASE_URL = getProperty(prop, "CAPTCHA_BASE_URL");
CAPTCHA_API_KEY = getProperty(prop, "CAPTCHA_API_KEY");
PUBLIC_URL = getProperty(prop, "API_URL");

81
src/main/java/me/kavin/piped/utils/URLUtils.java
View File

@ -1,6 +1,7 @@
package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang3.StringUtils;
import org.schabi.newpipe.extractor.Image;
@ -9,7 +10,14 @@ import java.net.URL;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import static me.kavin.piped.consts.Constants.PROXY_HASH_SECRET;
public class URLUtils {
@ -65,13 +73,84 @@ public class URLUtils {
boolean hasQuery = query != null;
Comparator<List<String>> listComparator = (o1, o2) -> {
for (int i = 0; i < Math.min(o1.size(), o2.size()); i++) {
int result = o1.get(i).compareTo(o2.get(i));
if (result != 0) {
return result;
}
}
return Integer.compare(o1.size(), o2.size()); // compare list sizes if all elements are equal
};
Set<List<String>> queryPairs = new TreeSet<>(listComparator);
if (hasQuery) {
String[] pairs = query.split("&");
for (String pair : pairs) {
int idx = pair.indexOf("=");
queryPairs.add(List.of(
silentDecode(pair.substring(0, idx)),
silentDecode(pair.substring(idx + 1))
));
}
}
// look for host param, and add it if it doesn't exist
boolean hasHost = false;
for (List<String> pair : queryPairs) {
if (pair.get(0).equals("host")) {
hasHost = true;
break;
}
}
if (!hasHost) {
queryPairs.add(List.of("host", host));
}
String path = url.getPath();
if (path.contains("=")) {
path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw");
}
return proxy + path + (hasQuery ? "?" + query + "&host=" : "?host=") + silentEncode(host);
if (PROXY_HASH_SECRET != null)
try {
MessageDigest md = MessageDigest.getInstance("BLAKE3-256");
for (List<String> pair : queryPairs) {
md.update(pair.get(0).getBytes(StandardCharsets.UTF_8));
md.update(pair.get(1).getBytes(StandardCharsets.UTF_8));
}
md.update(path.getBytes(StandardCharsets.UTF_8));
md.update(PROXY_HASH_SECRET);
queryPairs.add(List.of("qhash", Hex.encodeHexString(md.digest()).substring(0, 8)));
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
String newUrl = proxy + path;
StringBuilder qstring = null;
for (List<String> pair : queryPairs) {
if (qstring == null) {
qstring = new StringBuilder();
} else {
qstring.append("&");
}
qstring.append(silentEncode(pair.get(0)));
qstring.append("=");
qstring.append(silentEncode(pair.get(1)));
}
newUrl += "?" + qstring;
return newUrl;
}
}