TeamPiped-mirror/Piped-Backend
1
0
Fork 0
mirror of https://github.com/TeamPiped/Piped-Backend.git synced 2024-08-14 23:51:41 +00:00
Code Issues Projects Releases Packages Wiki Activity

reqwest4j is now async first

Browse source
This commit is contained in:
Kavin 2023-06-30 14:13:16 +01:00
parent 02765fd33e
commit 990388f0d4
No known key found for this signature in database
GPG key ID: 6E4598CA5C92C41F
11 changed files with 125 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build.gradle
View file

@ -40,7 +40,7 @@ dependencies {
implementation 'com.squareup.okhttp3:okhttp'
implementation 'com.squareup.okhttp3:okhttp-brotli'
implementation 'io.sentry:sentry:6.24.0'
implementation 'rocks.kavin:reqwest4j:1.0.4'
implementation 'rocks.kavin:reqwest4j:1.0.5'
implementation 'io.minio:minio:8.5.4'
}

2
src/main/java/me/kavin/piped/consts/Constants.java
View file

@ -199,7 +199,7 @@ public class Constants {
h2_no_redir_client = builder_noredir.build();
String temp = null;
try {
var html = RequestUtils.sendGet("https://www.youtube.com/");
var html = RequestUtils.sendGet("https://www.youtube.com/").get();
var regex = Pattern.compile("GL\":\"([A-Z]{2})\"", Pattern.MULTILINE);
var matcher = regex.matcher(html);
if (matcher.find()) {

2
src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java
View file

@ -150,7 +150,7 @@ public class StreamHandlers {
List<String> allowedCountries = new ObjectArrayList<>();
{
var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId);
var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId).get();
if (!restrictedTree.get("restricted").asBoolean()) {
assert exception != null;
throw (Exception) exception;

5
src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java
View file

@ -28,7 +28,7 @@ public class UserHandlers {
private static final Argon2PasswordEncoder argon2PasswordEncoder = Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8();
private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder();
public static byte[] registerResponse(String user, String pass) throws IOException {
public static byte[] registerResponse(String user, String pass) throws Exception {
if (Constants.DISABLE_REGISTRATION)
ExceptionHandler.throwErrorResponse(new DisabledRegistrationResponse());
@ -57,7 +57,8 @@ public class UserHandlers {
String suffix = sha1Hash.substring(5);
String[] entries = RequestUtils
.sendGet("https://api.pwnedpasswords.com/range/" + prefix, "github.com/TeamPiped/Piped-Backend")
.split("\n");
.thenApplyAsync(str -> str.split("\n"))
.get();
for (String entry : entries)
if (StringUtils.substringBefore(entry, ":").equals(suffix))
ExceptionHandler.throwErrorResponse(new CompromisedPasswordResponse());

142
src/main/java/me/kavin/piped/utils/DownloaderImpl.java
View file

@ -1,13 +1,6 @@
package me.kavin.piped.utils;
import com.grack.nanojson.JsonParserException;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import me.kavin.piped.consts.Constants;
import me.kavin.piped.utils.obj.SolvedCaptcha;
import okhttp3.FormBody;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response;
@ -18,7 +11,7 @@ import java.io.IOException;
import java.net.HttpCookie;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.ExecutionException;
public class DownloaderImpl extends Downloader {
@ -41,70 +34,85 @@ public class DownloaderImpl extends Downloader {
request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value)));
var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
var future = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
if (resp.status() == 429) {
// Recaptcha solver code
// Commented out, as it hasn't been ported to reqwest4j yet
// Also, this was last seen a long time back
synchronized (cookie_lock) {
// future.thenAcceptAsync(resp -> {
// if (resp.status() == 429) {
// synchronized (cookie_lock) {
//
// if (saved_cookie != null && saved_cookie.hasExpired()
// || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
// saved_cookie = null;
//
// String redir_url = String.valueOf(resp.finalUrl());
//
// if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
//
// var formBuilder = new FormBody.Builder();
// String sitekey = null, data_s = null;
//
// for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
// String name;
// if (!(name = el.tagName()).equals("script")) {
// if (name.equals("input"))
// formBuilder.add(el.attr("name"), el.attr("value"));
// else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
// sitekey = el.attr("data-sitekey");
// data_s = el.attr("data-s");
// }
// }
// }
//
// if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
// ExceptionHandler.handle(new ReCaptchaException("Could not get recaptcha", redir_url));
//
// SolvedCaptcha solved = null;
//
// try {
// solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
// } catch (JsonParserException | InterruptedException | IOException e) {
// e.printStackTrace();
// }
//
// formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
//
// var formReqBuilder = new okhttp3.Request.Builder()
// .url("https://www.google.com/sorry/index")
// .header("User-Agent", Constants.USER_AGENT)
// .post(formBuilder.build());
//
// okhttp3.Response formResponse;
// try {
// formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
//
// saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
// .substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
// .get(0);
// cookie_received = System.currentTimeMillis();
// }
// }
// }
// }, Multithreading.getCachedExecutor());
if (saved_cookie != null && saved_cookie.hasExpired()
|| (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
saved_cookie = null;
var responseFuture = future.thenApplyAsync(resp -> {
Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
.collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
String redir_url = String.valueOf(resp.finalUrl());
if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
var formBuilder = new FormBody.Builder();
String sitekey = null, data_s = null;
for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
String name;
if (!(name = el.tagName()).equals("script")) {
if (name.equals("input"))
formBuilder.add(el.attr("name"), el.attr("value"));
else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
sitekey = el.attr("data-sitekey");
data_s = el.attr("data-s");
}
}
}
if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
throw new ReCaptchaException("Could not get recaptcha", redir_url);
SolvedCaptcha solved = null;
try {
solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
} catch (JsonParserException | InterruptedException e) {
e.printStackTrace();
}
formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
var formReqBuilder = new okhttp3.Request.Builder()
.url("https://www.google.com/sorry/index")
.header("User-Agent", Constants.USER_AGENT)
.post(formBuilder.build());
var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
.substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
.get(0);
cookie_received = System.currentTimeMillis();
}
if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired.
execute(request);
}
return new Response(resp.status(), null, headerMap, new String(resp.body()),
resp.finalUrl());
}, Multithreading.getCachedExecutor());
try {
return responseFuture.get();
} catch (InterruptedException | ExecutionException e) {
throw new IOException(e);
}
Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
.collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
return new Response(resp.status(), null, headerMap, new String(resp.body()),
resp.finalUrl());
}
}

27
src/main/java/me/kavin/piped/utils/LbryHelper.java
View file

@ -3,8 +3,8 @@ package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants;
import org.apache.commons.lang3.StringUtils;
import rocks.kavin.reqwest4j.ReqwestUtils;
import rocks.kavin.reqwest4j.Response;
import java.io.IOException;
import java.net.URI;
import java.util.Map;
@ -13,19 +13,20 @@ import static me.kavin.piped.utils.URLUtils.silentEncode;
public class LbryHelper {
public static String getLBRYId(String videoId) throws IOException {
public static String getLBRYId(String videoId) throws Exception {
if (Constants.DISABLE_LBRY)
return null;
return RequestUtils.sendGetJson("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId))
.at("/data/videos")
.path(videoId)
.asText(null);
.thenApplyAsync(json -> json.at("/data/videos")
.path(videoId)
.asText(null)
).get();
}
public static String getLBRYStreamURL(String lbryId)
throws IOException {
throws Exception {
if (StringUtils.isEmpty(lbryId))
return null;
@ -41,7 +42,7 @@ public class LbryHelper {
.put("uri", "lbry://" + lbryId)
.put("save_file", true)
)
), Map.of("Content-Type", "application/json"));
), Map.of("Content-Type", "application/json")).get();
if (resp.status() / 100 == 2) {
return mapper.readTree(resp.body())
.at("/result/streaming_url")
@ -60,12 +61,12 @@ public class LbryHelper {
// LBRY provides non UTF-8 characters in the URL, which causes issues
streamUrl = new URI(streamUrl).toASCIIString();
var resp = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of(
"Origin", "https://odysee.com",
"Referer", "https://odysee.com/"
));
final String lastLocation = resp.finalUrl();
final String lastLocation = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of(
"Origin", "https://odysee.com",
"Referer", "https://odysee.com/"
))
.thenApply(Response::finalUrl)
.get();
return streamUrl.equals(lastLocation) ? null : lastLocation;
}

10
src/main/java/me/kavin/piped/utils/PubSubHelper.java
View file

@ -44,11 +44,11 @@ public class PubSubHelper {
var buffer = new Buffer();
formBuilder.build().writeTo(buffer);
var resp = ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of());
if (resp.status() != 202)
System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of())
.thenAccept(resp -> {
if (resp.status() != 202)
System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
});
}
}

38
src/main/java/me/kavin/piped/utils/RequestUtils.java
View file

@ -1,7 +1,6 @@
package me.kavin.piped.utils;
import com.fasterxml.jackson.databind.JsonNode;
import me.kavin.piped.consts.Constants;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import rocks.kavin.reqwest4j.ReqwestUtils;
@ -9,27 +8,26 @@ import rocks.kavin.reqwest4j.Response;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import static me.kavin.piped.consts.Constants.mapper;
public class RequestUtils {
public static Response sendGetRaw(String url) throws IOException {
public static CompletableFuture<Response> sendGetRaw(String url) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of());
}
public static String sendGet(String url) throws IOException {
return new String(
ReqwestUtils.fetch(url, "GET", null, Map.of())
.body()
);
public static CompletableFuture<String> sendGet(String url) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of())
.thenApply(Response::body)
.thenApplyAsync(String::new);
}
public static String sendGet(String url, String ua) throws IOException {
return new String(
ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
.body()
);
public static CompletableFuture<String> sendGet(String url, String ua) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
.thenApply(Response::body)
.thenApplyAsync(String::new);
}
public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException {
@ -44,13 +42,13 @@ public class RequestUtils {
}
}
public static JsonNode sendGetJson(String url, String ua) throws IOException {
return getJsonNode(Constants.h2client, new Request.Builder().header("User-Agent", ua).url(url).build());
}
public static JsonNode sendGetJson(String url) throws IOException {
return mapper.readTree(ReqwestUtils.fetch(url, "GET", null, Map.of()).body());
public static CompletableFuture<JsonNode> sendGetJson(String url) throws Exception {
return ReqwestUtils.fetch(url, "GET", null, Map.of()).thenApply(Response::body).thenApplyAsync(resp -> {
try {
return mapper.readTree(resp);
} catch (Exception e) {
throw new RuntimeException("Failed to parse JSON", e);
}
}, Multithreading.getCachedExecutor());
}
}

18
src/main/java/me/kavin/piped/utils/RydHelper.java
View file

@ -3,25 +3,17 @@ package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants;
import java.io.IOException;
import static me.kavin.piped.consts.Constants.mapper;
import static me.kavin.piped.utils.RequestUtils.sendGetRaw;
import static me.kavin.piped.utils.RequestUtils.sendGetJson;
public class RydHelper {
public static double getDislikeRating(String videoId) throws IOException {
public static double getDislikeRating(String videoId) throws Exception {
if (Constants.DISABLE_RYD)
return -1;
var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId);
if (resp.status() / 100 != 2)
return -1;
return mapper.readTree(resp.body())
.path("rating")
.asDouble(-1);
return sendGetJson(Constants.RYD_PROXY_URL + "/votes/" + videoId)
.thenApply(tree -> tree.path("rating").asDouble(-1))
.get();
}
}

2
src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java
View file

@ -24,7 +24,7 @@ public class SponsorBlockUtils {
try {
var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
+ "?categories=" + URLUtils.silentEncode(categories));
+ "?categories=" + URLUtils.silentEncode(categories)).get();
if (resp.status() == 200) {
var any = mapper.readTree(resp.body());

2
src/main/java/me/kavin/piped/utils/obj/MatrixHelper.java
View file

@ -66,7 +66,7 @@ public class MatrixHelper {
.asText();
}
AUTHORIZED_USERS = (ArrayNode) mapper.readTree(RequestUtils.sendGet("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json"));
AUTHORIZED_USERS = (ArrayNode) RequestUtils.sendGetJson("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json").get();
} catch (Exception e) {
ExceptionHandler.handle(e);