From 990388f0d41a3cc08eb5a10143d57b41b49b50c8 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Fri, 30 Jun 2023 14:13:16 +0100 Subject: [PATCH 1/2] reqwest4j is now async first --- build.gradle | 2 +- .../java/me/kavin/piped/consts/Constants.java | 2 +- .../piped/server/handlers/StreamHandlers.java | 2 +- .../server/handlers/auth/UserHandlers.java | 5 +- .../me/kavin/piped/utils/DownloaderImpl.java | 142 +++++++++--------- .../java/me/kavin/piped/utils/LbryHelper.java | 27 ++-- .../me/kavin/piped/utils/PubSubHelper.java | 10 +- .../me/kavin/piped/utils/RequestUtils.java | 38 +++-- .../java/me/kavin/piped/utils/RydHelper.java | 18 +-- .../kavin/piped/utils/SponsorBlockUtils.java | 2 +- .../kavin/piped/utils/obj/MatrixHelper.java | 2 +- 11 files changed, 125 insertions(+), 125 deletions(-) diff --git a/build.gradle b/build.gradle index b5aab76..e957149 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,7 @@ dependencies { implementation 'com.squareup.okhttp3:okhttp' implementation 'com.squareup.okhttp3:okhttp-brotli' implementation 'io.sentry:sentry:6.24.0' - implementation 'rocks.kavin:reqwest4j:1.0.4' + implementation 'rocks.kavin:reqwest4j:1.0.5' implementation 'io.minio:minio:8.5.4' } diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index 478b99e..4f338c0 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -199,7 +199,7 @@ public class Constants { h2_no_redir_client = builder_noredir.build(); String temp = null; try { - var html = RequestUtils.sendGet("https://www.youtube.com/"); + var html = RequestUtils.sendGet("https://www.youtube.com/").get(); var regex = Pattern.compile("GL\":\"([A-Z]{2})\"", Pattern.MULTILINE); var matcher = regex.matcher(html); if (matcher.find()) { diff --git a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java index 6eb6744..a6e50e3 100644 --- a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java @@ -150,7 +150,7 @@ public class StreamHandlers { List allowedCountries = new ObjectArrayList<>(); { - var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId); + var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId).get(); if (!restrictedTree.get("restricted").asBoolean()) { assert exception != null; throw (Exception) exception; diff --git a/src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java b/src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java index ad36481..3e0bfe5 100644 --- a/src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java @@ -28,7 +28,7 @@ public class UserHandlers { private static final Argon2PasswordEncoder argon2PasswordEncoder = Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8(); private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder(); - public static byte[] registerResponse(String user, String pass) throws IOException { + public static byte[] registerResponse(String user, String pass) throws Exception { if (Constants.DISABLE_REGISTRATION) ExceptionHandler.throwErrorResponse(new DisabledRegistrationResponse()); @@ -57,7 +57,8 @@ public class UserHandlers { String suffix = sha1Hash.substring(5); String[] entries = RequestUtils .sendGet("https://api.pwnedpasswords.com/range/" + prefix, "github.com/TeamPiped/Piped-Backend") - .split("\n"); + .thenApplyAsync(str -> str.split("\n")) + .get(); for (String entry : entries) if (StringUtils.substringBefore(entry, ":").equals(suffix)) ExceptionHandler.throwErrorResponse(new CompromisedPasswordResponse()); diff --git a/src/main/java/me/kavin/piped/utils/DownloaderImpl.java b/src/main/java/me/kavin/piped/utils/DownloaderImpl.java index ecf1250..aef67cb 100644 --- a/src/main/java/me/kavin/piped/utils/DownloaderImpl.java +++ b/src/main/java/me/kavin/piped/utils/DownloaderImpl.java @@ -1,13 +1,6 @@ package me.kavin.piped.utils; -import com.grack.nanojson.JsonParserException; import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; -import me.kavin.piped.consts.Constants; -import me.kavin.piped.utils.obj.SolvedCaptcha; -import okhttp3.FormBody; -import org.apache.commons.lang3.StringUtils; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Request; import org.schabi.newpipe.extractor.downloader.Response; @@ -18,7 +11,7 @@ import java.io.IOException; import java.net.HttpCookie; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.ExecutionException; public class DownloaderImpl extends Downloader { @@ -41,70 +34,85 @@ public class DownloaderImpl extends Downloader { request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value))); - var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers); + var future = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers); - if (resp.status() == 429) { + // Recaptcha solver code + // Commented out, as it hasn't been ported to reqwest4j yet + // Also, this was last seen a long time back - synchronized (cookie_lock) { +// future.thenAcceptAsync(resp -> { +// if (resp.status() == 429) { +// synchronized (cookie_lock) { +// +// if (saved_cookie != null && saved_cookie.hasExpired() +// || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30))) +// saved_cookie = null; +// +// String redir_url = String.valueOf(resp.finalUrl()); +// +// if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) { +// +// var formBuilder = new FormBody.Builder(); +// String sitekey = null, data_s = null; +// +// for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) { +// String name; +// if (!(name = el.tagName()).equals("script")) { +// if (name.equals("input")) +// formBuilder.add(el.attr("name"), el.attr("value")); +// else if (name.equals("div") && el.attr("id").equals("recaptcha")) { +// sitekey = el.attr("data-sitekey"); +// data_s = el.attr("data-s"); +// } +// } +// } +// +// if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s)) +// ExceptionHandler.handle(new ReCaptchaException("Could not get recaptcha", redir_url)); +// +// SolvedCaptcha solved = null; +// +// try { +// solved = CaptchaSolver.solve(redir_url, sitekey, data_s); +// } catch (JsonParserException | InterruptedException | IOException e) { +// e.printStackTrace(); +// } +// +// formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse()); +// +// var formReqBuilder = new okhttp3.Request.Builder() +// .url("https://www.google.com/sorry/index") +// .header("User-Agent", Constants.USER_AGENT) +// .post(formBuilder.build()); +// +// okhttp3.Response formResponse; +// try { +// formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute(); +// } catch (IOException e) { +// throw new RuntimeException(e); +// } +// +// saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils +// .substringAfter(formResponse.headers().get("Location"), "google_abuse="))) +// .get(0); +// cookie_received = System.currentTimeMillis(); +// } +// } +// } +// }, Multithreading.getCachedExecutor()); - if (saved_cookie != null && saved_cookie.hasExpired() - || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30))) - saved_cookie = null; + var responseFuture = future.thenApplyAsync(resp -> { + Map> headerMap = resp.headers().entrySet().stream() + .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll); - String redir_url = String.valueOf(resp.finalUrl()); - - if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) { - - var formBuilder = new FormBody.Builder(); - String sitekey = null, data_s = null; - - for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) { - String name; - if (!(name = el.tagName()).equals("script")) { - if (name.equals("input")) - formBuilder.add(el.attr("name"), el.attr("value")); - else if (name.equals("div") && el.attr("id").equals("recaptcha")) { - sitekey = el.attr("data-sitekey"); - data_s = el.attr("data-s"); - } - } - } - if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s)) - throw new ReCaptchaException("Could not get recaptcha", redir_url); - - SolvedCaptcha solved = null; - - try { - solved = CaptchaSolver.solve(redir_url, sitekey, data_s); - } catch (JsonParserException | InterruptedException e) { - e.printStackTrace(); - } - - formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse()); - - var formReqBuilder = new okhttp3.Request.Builder() - .url("https://www.google.com/sorry/index") - .header("User-Agent", Constants.USER_AGENT) - .post(formBuilder.build()); - - var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute(); - - saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils - .substringAfter(formResponse.headers().get("Location"), "google_abuse="))) - .get(0); - cookie_received = System.currentTimeMillis(); - } - - if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired. - execute(request); - } + return new Response(resp.status(), null, headerMap, new String(resp.body()), + resp.finalUrl()); + }, Multithreading.getCachedExecutor()); + try { + return responseFuture.get(); + } catch (InterruptedException | ExecutionException e) { + throw new IOException(e); } - - Map> headerMap = resp.headers().entrySet().stream() - .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll); - - return new Response(resp.status(), null, headerMap, new String(resp.body()), - resp.finalUrl()); } } diff --git a/src/main/java/me/kavin/piped/utils/LbryHelper.java b/src/main/java/me/kavin/piped/utils/LbryHelper.java index af92e45..52c0dfc 100644 --- a/src/main/java/me/kavin/piped/utils/LbryHelper.java +++ b/src/main/java/me/kavin/piped/utils/LbryHelper.java @@ -3,8 +3,8 @@ package me.kavin.piped.utils; import me.kavin.piped.consts.Constants; import org.apache.commons.lang3.StringUtils; import rocks.kavin.reqwest4j.ReqwestUtils; +import rocks.kavin.reqwest4j.Response; -import java.io.IOException; import java.net.URI; import java.util.Map; @@ -13,19 +13,20 @@ import static me.kavin.piped.utils.URLUtils.silentEncode; public class LbryHelper { - public static String getLBRYId(String videoId) throws IOException { + public static String getLBRYId(String videoId) throws Exception { if (Constants.DISABLE_LBRY) return null; return RequestUtils.sendGetJson("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId)) - .at("/data/videos") - .path(videoId) - .asText(null); + .thenApplyAsync(json -> json.at("/data/videos") + .path(videoId) + .asText(null) + ).get(); } public static String getLBRYStreamURL(String lbryId) - throws IOException { + throws Exception { if (StringUtils.isEmpty(lbryId)) return null; @@ -41,7 +42,7 @@ public class LbryHelper { .put("uri", "lbry://" + lbryId) .put("save_file", true) ) - ), Map.of("Content-Type", "application/json")); + ), Map.of("Content-Type", "application/json")).get(); if (resp.status() / 100 == 2) { return mapper.readTree(resp.body()) .at("/result/streaming_url") @@ -60,12 +61,12 @@ public class LbryHelper { // LBRY provides non UTF-8 characters in the URL, which causes issues streamUrl = new URI(streamUrl).toASCIIString(); - var resp = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of( - "Origin", "https://odysee.com", - "Referer", "https://odysee.com/" - )); - - final String lastLocation = resp.finalUrl(); + final String lastLocation = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of( + "Origin", "https://odysee.com", + "Referer", "https://odysee.com/" + )) + .thenApply(Response::finalUrl) + .get(); return streamUrl.equals(lastLocation) ? null : lastLocation; } diff --git a/src/main/java/me/kavin/piped/utils/PubSubHelper.java b/src/main/java/me/kavin/piped/utils/PubSubHelper.java index 63e141a..7b07a38 100644 --- a/src/main/java/me/kavin/piped/utils/PubSubHelper.java +++ b/src/main/java/me/kavin/piped/utils/PubSubHelper.java @@ -44,11 +44,11 @@ public class PubSubHelper { var buffer = new Buffer(); formBuilder.build().writeTo(buffer); - var resp = ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of()); - - if (resp.status() != 202) - System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body())); - + ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of()) + .thenAccept(resp -> { + if (resp.status() != 202) + System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body())); + }); } } diff --git a/src/main/java/me/kavin/piped/utils/RequestUtils.java b/src/main/java/me/kavin/piped/utils/RequestUtils.java index 6285c8c..d886418 100644 --- a/src/main/java/me/kavin/piped/utils/RequestUtils.java +++ b/src/main/java/me/kavin/piped/utils/RequestUtils.java @@ -1,7 +1,6 @@ package me.kavin.piped.utils; import com.fasterxml.jackson.databind.JsonNode; -import me.kavin.piped.consts.Constants; import okhttp3.OkHttpClient; import okhttp3.Request; import rocks.kavin.reqwest4j.ReqwestUtils; @@ -9,27 +8,26 @@ import rocks.kavin.reqwest4j.Response; import java.io.IOException; import java.util.Map; +import java.util.concurrent.CompletableFuture; import static me.kavin.piped.consts.Constants.mapper; public class RequestUtils { - public static Response sendGetRaw(String url) throws IOException { + public static CompletableFuture sendGetRaw(String url) throws Exception { return ReqwestUtils.fetch(url, "GET", null, Map.of()); } - public static String sendGet(String url) throws IOException { - return new String( - ReqwestUtils.fetch(url, "GET", null, Map.of()) - .body() - ); + public static CompletableFuture sendGet(String url) throws Exception { + return ReqwestUtils.fetch(url, "GET", null, Map.of()) + .thenApply(Response::body) + .thenApplyAsync(String::new); } - public static String sendGet(String url, String ua) throws IOException { - return new String( - ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua)) - .body() - ); + public static CompletableFuture sendGet(String url, String ua) throws Exception { + return ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua)) + .thenApply(Response::body) + .thenApplyAsync(String::new); } public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException { @@ -44,13 +42,13 @@ public class RequestUtils { } } - public static JsonNode sendGetJson(String url, String ua) throws IOException { - return getJsonNode(Constants.h2client, new Request.Builder().header("User-Agent", ua).url(url).build()); - } - - public static JsonNode sendGetJson(String url) throws IOException { - - return mapper.readTree(ReqwestUtils.fetch(url, "GET", null, Map.of()).body()); - + public static CompletableFuture sendGetJson(String url) throws Exception { + return ReqwestUtils.fetch(url, "GET", null, Map.of()).thenApply(Response::body).thenApplyAsync(resp -> { + try { + return mapper.readTree(resp); + } catch (Exception e) { + throw new RuntimeException("Failed to parse JSON", e); + } + }, Multithreading.getCachedExecutor()); } } diff --git a/src/main/java/me/kavin/piped/utils/RydHelper.java b/src/main/java/me/kavin/piped/utils/RydHelper.java index 44c34ce..d1bfb83 100644 --- a/src/main/java/me/kavin/piped/utils/RydHelper.java +++ b/src/main/java/me/kavin/piped/utils/RydHelper.java @@ -3,25 +3,17 @@ package me.kavin.piped.utils; import me.kavin.piped.consts.Constants; -import java.io.IOException; - -import static me.kavin.piped.consts.Constants.mapper; -import static me.kavin.piped.utils.RequestUtils.sendGetRaw; +import static me.kavin.piped.utils.RequestUtils.sendGetJson; public class RydHelper { - public static double getDislikeRating(String videoId) throws IOException { + public static double getDislikeRating(String videoId) throws Exception { if (Constants.DISABLE_RYD) return -1; - var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId); - - if (resp.status() / 100 != 2) - return -1; - - return mapper.readTree(resp.body()) - .path("rating") - .asDouble(-1); + return sendGetJson(Constants.RYD_PROXY_URL + "/votes/" + videoId) + .thenApply(tree -> tree.path("rating").asDouble(-1)) + .get(); } } diff --git a/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java b/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java index cd27a16..bfe580f 100644 --- a/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java +++ b/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java @@ -24,7 +24,7 @@ public class SponsorBlockUtils { try { var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4)) - + "?categories=" + URLUtils.silentEncode(categories)); + + "?categories=" + URLUtils.silentEncode(categories)).get(); if (resp.status() == 200) { var any = mapper.readTree(resp.body()); diff --git a/src/main/java/me/kavin/piped/utils/obj/MatrixHelper.java b/src/main/java/me/kavin/piped/utils/obj/MatrixHelper.java index 3ecceaa..173a5cb 100644 --- a/src/main/java/me/kavin/piped/utils/obj/MatrixHelper.java +++ b/src/main/java/me/kavin/piped/utils/obj/MatrixHelper.java @@ -66,7 +66,7 @@ public class MatrixHelper { .asText(); } - AUTHORIZED_USERS = (ArrayNode) mapper.readTree(RequestUtils.sendGet("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json")); + AUTHORIZED_USERS = (ArrayNode) RequestUtils.sendGetJson("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json").get(); } catch (Exception e) { ExceptionHandler.handle(e); From d707beb85acc6645d007284d7d9cc4796e7f2763 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Fri, 30 Jun 2023 14:46:01 +0100 Subject: [PATCH 2/2] Add exception handler to pubsub --- src/main/java/me/kavin/piped/utils/PubSubHelper.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/me/kavin/piped/utils/PubSubHelper.java b/src/main/java/me/kavin/piped/utils/PubSubHelper.java index 7b07a38..23976b7 100644 --- a/src/main/java/me/kavin/piped/utils/PubSubHelper.java +++ b/src/main/java/me/kavin/piped/utils/PubSubHelper.java @@ -48,6 +48,10 @@ public class PubSubHelper { .thenAccept(resp -> { if (resp.status() != 202) System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body())); + }) + .exceptionally(e -> { + ExceptionHandler.handle((Exception) e); + return null; }); } }