TeamPiped-mirror
/
Piped-Backend
mirror of https://github.com/TeamPiped/Piped-Backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use reqwest4j instead of okhttp.

This commit is contained in:
Kavin 2023-01-29 12:16:57 +00:00
parent 9f06b7eec2
commit 36af3af096
No known key found for this signature in database
GPG Key ID: 49451E4482CC5BCD
9 changed files with 129 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
build.gradle
View File

@ -1,60 +1,61 @@
plugins {
id "com.github.johnrengelman.shadow" version "7.1.2"
id "java"
id "io.freefair.lombok" version "6.6.1"
id "eclipse"
}
repositories {
mavenCentral()
maven { url 'https://jitpack.io' }
}
dependencies {
implementation 'org.apache.commons:commons-lang3:3.12.0'
implementation 'org.apache.commons:commons-text:1.10.0'
implementation 'commons-io:commons-io:2.11.0'
implementation 'it.unimi.dsi:fastutil-core:8.5.11'
implementation 'commons-codec:commons-codec:1.15'
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:85bff7662e56f5570dd0e5f8b666b4ffaf57a3b0'
implementation 'com.github.FireMasterK:nanojson:885effc25ad9ff920d43a4b09849719926ed89b2'
implementation 'com.fasterxml.jackson.core:jackson-core:2.14.1'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.14.1'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1'
implementation 'com.rometools:rome:1.18.0'
implementation 'org.jsoup:jsoup:1.15.3'
implementation 'io.activej:activej-common:5.4.3'
implementation 'io.activej:activej-http:5.4.3'
implementation 'io.activej:activej-boot:5.4.3'
implementation 'io.activej:activej-specializer:5.4.3'
implementation 'io.activej:activej-launchers-http:5.4.3'
implementation 'org.hsqldb:hsqldb:2.7.1'
implementation 'org.postgresql:postgresql:42.5.1'
implementation 'org.hibernate:hibernate-core:6.1.6.Final'
implementation 'org.hibernate:hibernate-hikaricp:6.1.6.Final'
implementation 'com.zaxxer:HikariCP:5.0.1'
implementation 'org.springframework.security:spring-security-crypto:6.0.1'
implementation 'commons-logging:commons-logging:1.2'
implementation(platform("com.squareup.okhttp3:okhttp-bom:4.10.0"))
implementation 'com.squareup.okhttp3:okhttp'
implementation 'com.squareup.okhttp3:okhttp-brotli'
implementation 'io.sentry:sentry:6.11.0'
}
shadowJar {
// minimize()
}
jar {
manifest {
attributes(
'Main-Class': 'me.kavin.piped.Main'
)
}
}
group = 'me.kavin.piped'
version = '1.0'
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
plugins {
id "com.github.johnrengelman.shadow" version "7.1.2"
id "java"
id "io.freefair.lombok" version "6.6.1"
id "eclipse"
}
repositories {
mavenCentral()
maven { url 'https://jitpack.io' }
}
dependencies {
implementation 'org.apache.commons:commons-lang3:3.12.0'
implementation 'org.apache.commons:commons-text:1.10.0'
implementation 'commons-io:commons-io:2.11.0'
implementation 'it.unimi.dsi:fastutil-core:8.5.11'
implementation 'commons-codec:commons-codec:1.15'
implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:85bff7662e56f5570dd0e5f8b666b4ffaf57a3b0'
implementation 'com.github.FireMasterK:nanojson:885effc25ad9ff920d43a4b09849719926ed89b2'
implementation 'com.fasterxml.jackson.core:jackson-core:2.14.1'
implementation 'com.fasterxml.jackson.core:jackson-annotations:2.14.1'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1'
implementation 'com.rometools:rome:1.18.0'
implementation 'org.jsoup:jsoup:1.15.3'
implementation 'io.activej:activej-common:5.4.3'
implementation 'io.activej:activej-http:5.4.3'
implementation 'io.activej:activej-boot:5.4.3'
implementation 'io.activej:activej-specializer:5.4.3'
implementation 'io.activej:activej-launchers-http:5.4.3'
implementation 'org.hsqldb:hsqldb:2.7.1'
implementation 'org.postgresql:postgresql:42.5.1'
implementation 'org.hibernate:hibernate-core:6.1.6.Final'
implementation 'org.hibernate:hibernate-hikaricp:6.1.6.Final'
implementation 'com.zaxxer:HikariCP:5.0.1'
implementation 'org.springframework.security:spring-security-crypto:6.0.1'
implementation 'commons-logging:commons-logging:1.2'
implementation(platform("com.squareup.okhttp3:okhttp-bom:4.10.0"))
implementation 'com.squareup.okhttp3:okhttp'
implementation 'com.squareup.okhttp3:okhttp-brotli'
implementation 'io.sentry:sentry:6.11.0'
implementation 'rocks.kavin:reqwest4j:1.0'
}
shadowJar {
// minimize()
}
jar {
manifest {
attributes(
'Main-Class': 'me.kavin.piped.Main'
)
}
}
group = 'me.kavin.piped'
version = '1.0'
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17

4
settings.gradle
View File

@ -1,2 +1,2 @@
rootProject.name = 'piped'
rootProject.name = 'piped'

2
src/main/java/me/kavin/piped/consts/Constants.java
View File

@ -29,7 +29,7 @@ import java.util.regex.Pattern;
public class Constants {
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0";
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0";
public static final int PORT;
public static final String HTTP_WORKERS;

35
src/main/java/me/kavin/piped/utils/DownloaderImpl.java
View File

@ -1,10 +1,10 @@
package me.kavin.piped.utils;
import com.grack.nanojson.JsonParserException;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import me.kavin.piped.consts.Constants;
import me.kavin.piped.utils.obj.SolvedCaptcha;
import okhttp3.FormBody;
import okhttp3.RequestBody;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
@ -12,9 +12,12 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.io.IOException;
import java.net.HttpCookie;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public class DownloaderImpl extends Downloader {
@ -31,23 +34,16 @@ public class DownloaderImpl extends Downloader {
// TODO: HTTP/3 aka QUIC
var bytes = request.dataToSend();
RequestBody body = null;
if (bytes != null)
body = RequestBody.create(bytes);
var builder = new okhttp3.Request.Builder()
.url(request.url())
.method(request.httpMethod(), body)
.header("User-Agent", Constants.USER_AGENT);
Map<String, String> headers = new Object2ObjectOpenHashMap<>();
if (saved_cookie != null && !saved_cookie.hasExpired())
builder.header("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue());
headers.put("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue());
request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value)));
request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value)));
var resp = Constants.h2client.newCall(builder.build()).execute();
var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
if (resp.code() == 429) {
if (resp.status() == 429) {
synchronized (cookie_lock) {
@ -55,14 +51,14 @@ public class DownloaderImpl extends Downloader {
|| (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
saved_cookie = null;
String redir_url = String.valueOf(resp.request().url());
String redir_url = String.valueOf(resp.finalUrl());
if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
var formBuilder = new FormBody.Builder();
String sitekey = null, data_s = null;
for (Element el : Jsoup.parse(resp.body().string()).selectFirst("form").children()) {
for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
String name;
if (!(name = el.tagName()).equals("script")) {
if (name.equals("input"))
@ -105,11 +101,10 @@ public class DownloaderImpl extends Downloader {
}
var response = new Response(resp.code(), resp.message(), resp.headers().toMultimap(), resp.body().string(),
String.valueOf(resp.request().url()));
Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
.collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
resp.close();
return response;
return new Response(resp.status(), null, headerMap, new String(resp.body()),
resp.finalUrl());
}
}

29
src/main/java/me/kavin/piped/utils/LbryHelper.java
View File

@ -1,16 +1,13 @@
package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants;
import okhttp3.MediaType;
import okhttp3.Request;
import okhttp3.RequestBody;
import org.apache.commons.lang3.StringUtils;
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.io.IOException;
import java.util.Map;
import static me.kavin.piped.consts.Constants.h2client;
import static me.kavin.piped.consts.Constants.mapper;
import static me.kavin.piped.utils.RequestUtils.sendGet;
import static me.kavin.piped.utils.URLUtils.silentEncode;
public class LbryHelper {
@ -20,7 +17,7 @@ public class LbryHelper {
if (Constants.DISABLE_LBRY)
return null;
return mapper.readTree(sendGet("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId)))
return RequestUtils.sendGetJson("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId))
.at("/data/videos")
.path(videoId)
.asText(null);
@ -32,11 +29,9 @@ public class LbryHelper {
if (StringUtils.isEmpty(lbryId))
return null;
var request = new Request.Builder()
.url("https://api.na-backend.odysee.com/api/v1/proxy?m=get")
.post(RequestBody.create(mapper.writeValueAsBytes(
var resp = ReqwestUtils.fetch("https://api.na-backend.odysee.com/api/v1/proxy?m=get", "POST",
mapper.writeValueAsBytes(
mapper.createObjectNode()
.put("id", System.currentTimeMillis())
.put("id", System.currentTimeMillis())
.put("jsonrpc", "2.0")
.put("method", "get")
@ -45,15 +40,11 @@ public class LbryHelper {
.put("uri", "lbry://" + lbryId)
.put("save_file", true)
)
), MediaType.get("application/json")))
.build();
try (var resp = h2client.newCall(request).execute()) {
if (resp.isSuccessful()) {
return mapper.readTree(resp.body().byteStream())
.at("/result/streaming_url")
.asText(null);
}
), Map.of("Content-Type", "application/json"));
if (resp.status() / 100 == 2) {
return mapper.readTree(resp.body())
.at("/result/streaming_url")
.asText(null);
}
return null;

22
src/main/java/me/kavin/piped/utils/PubSubHelper.java
View File

@ -3,11 +3,12 @@ package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants;
import me.kavin.piped.utils.obj.db.PubSub;
import okhttp3.FormBody;
import okhttp3.Request;
import okio.Buffer;
import org.hibernate.StatelessSession;
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.io.IOException;
import java.util.Objects;
import java.util.Map;
import java.util.concurrent.TimeUnit;
public class PubSubHelper {
@ -23,9 +24,6 @@ public class PubSubHelper {
String callback = Constants.PUBSUB_URL + "/webhooks/pubsub";
String topic = "https://www.youtube.com/xml/feeds/videos.xml?channel_id=" + channelId;
var builder = new Request.Builder()
.url(Constants.PUBSUB_HUB_URL);
var formBuilder = new FormBody.Builder();
formBuilder.add("hub.callback", callback);
@ -42,16 +40,16 @@ public class PubSubHelper {
tr.commit();
}
try (var resp = Constants.h2client
.newCall(builder.post(formBuilder.build())
.build()).execute()) {
// write form to read later
var buffer = new Buffer();
formBuilder.build().writeTo(buffer);
if (resp.code() != 202)
System.out.println("Failed to subscribe: " + resp.code() + "\n" + Objects.requireNonNull(resp.body()).string());
var resp = ReqwestUtils.fetch(callback, "POST", buffer.readByteArray(), Map.of());
if (resp.status() != 202)
System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
}
}
}
public static void updatePubSub(String channelId) {

34
src/main/java/me/kavin/piped/utils/RequestUtils.java
View File

@ -4,38 +4,38 @@ import com.fasterxml.jackson.databind.JsonNode;
import me.kavin.piped.consts.Constants;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import rocks.kavin.reqwest4j.ReqwestUtils;
import rocks.kavin.reqwest4j.Response;
import java.io.IOException;
import java.util.Map;
import static me.kavin.piped.consts.Constants.mapper;
public class RequestUtils {
public static Response sendGetRaw(String url) throws IOException {
return sendGetRaw(url, Constants.USER_AGENT);
}
public static Response sendGetRaw(String url, String ua) throws IOException {
var request = new Request.Builder().header("User-Agent", ua).url(url).build();
return Constants.h2client.newCall(request).execute();
return ReqwestUtils.fetch(url, "GET", null, Map.of());
}
public static String sendGet(String url) throws IOException {
return sendGet(url, Constants.USER_AGENT);
return new String(
ReqwestUtils.fetch(url, "GET", null, Map.of())
.body()
);
}
public static String sendGet(String url, String ua) throws IOException {
var response = sendGetRaw(url, ua);
var responseString = response.body().string();
response.close();
return responseString;
return new String(
ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
.body()
);
}
public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException {
try (var resp = client.newCall(request).execute()) {
try {
return Constants.mapper.readTree(resp.body().byteStream());
return mapper.readTree(resp.body().byteStream());
} catch (Exception e) {
if (!resp.isSuccessful())
ExceptionHandler.handle(e);
@ -49,6 +49,8 @@ public class RequestUtils {
}
public static JsonNode sendGetJson(String url) throws IOException {
return sendGetJson(url, Constants.USER_AGENT);
return mapper.readTree(ReqwestUtils.fetch(url, "GET", null, Map.of()).body());
}
}

13
src/main/java/me/kavin/piped/utils/RydHelper.java
View File

@ -14,15 +14,14 @@ public class RydHelper {
if (Constants.DISABLE_RYD)
return -1;
try (var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId)) {
var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId);
if (!resp.isSuccessful())
return -1;
if (resp.status() / 100 != 2)
return -1;
return mapper.readTree(resp.body().byteStream())
.path("rating")
.asDouble(-1);
return mapper.readTree(resp.body())
.path("rating")
.asDouble(-1);
}
}
}

10
src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java
View File

@ -25,11 +25,13 @@ public class SponsorBlockUtils {
String hash = toSha256(id);
for (String url : Constants.SPONSORBLOCK_SERVERS) {
try (var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
+ "?categories=" + URLUtils.silentEncode(categories))) {
try {
if (resp.code() == 200) {
JsonArray jArray = JsonParser.array().from(resp.body().string());
var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
+ "?categories=" + URLUtils.silentEncode(categories));
if (resp.status() == 200) {
JsonArray jArray = JsonParser.array().from(new String(resp.body()));
jArray.removeIf(jObject -> !((JsonObject) jObject).getString("videoID").equalsIgnoreCase(id));