Merge pull request #636 from TeamPiped/reqwest4j-async
reqwest4j is now async first
This commit is contained in:
Kavin
GitHub
commit
d31371e35b
|
|
@ -40,7 +40,7 @@ dependencies {
|
|||
implementation 'com.squareup.okhttp3:okhttp'
|
||||
implementation 'com.squareup.okhttp3:okhttp-brotli'
|
||||
implementation 'io.sentry:sentry:6.24.0'
|
||||
implementation 'rocks.kavin:reqwest4j:1.0.4'
|
||||
implementation 'rocks.kavin:reqwest4j:1.0.5'
|
||||
implementation 'io.minio:minio:8.5.4'
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ public class Constants {
|
|||
h2_no_redir_client = builder_noredir.build();
|
||||
String temp = null;
|
||||
try {
|
||||
var html = RequestUtils.sendGet("https://www.youtube.com/");
|
||||
var html = RequestUtils.sendGet("https://www.youtube.com/").get();
|
||||
var regex = Pattern.compile("GL\":\"([A-Z]{2})\"", Pattern.MULTILINE);
|
||||
var matcher = regex.matcher(html);
|
||||
if (matcher.find()) {
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ public class StreamHandlers {
|
|||
List<String> allowedCountries = new ObjectArrayList<>();
|
||||
|
||||
{
|
||||
var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId);
|
||||
var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId).get();
|
||||
if (!restrictedTree.get("restricted").asBoolean()) {
|
||||
assert exception != null;
|
||||
throw (Exception) exception;
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ public class UserHandlers {
|
|||
private static final Argon2PasswordEncoder argon2PasswordEncoder = Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8();
|
||||
private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder();
|
||||
|
||||
public static byte[] registerResponse(String user, String pass) throws IOException {
|
||||
public static byte[] registerResponse(String user, String pass) throws Exception {
|
||||
|
||||
if (Constants.DISABLE_REGISTRATION)
|
||||
ExceptionHandler.throwErrorResponse(new DisabledRegistrationResponse());
|
||||
|
|
@ -57,7 +57,8 @@ public class UserHandlers {
|
|||
String suffix = sha1Hash.substring(5);
|
||||
String[] entries = RequestUtils
|
||||
.sendGet("https://api.pwnedpasswords.com/range/" + prefix, "github.com/TeamPiped/Piped-Backend")
|
||||
.split("\n");
|
||||
.thenApplyAsync(str -> str.split("\n"))
|
||||
.get();
|
||||
for (String entry : entries)
|
||||
if (StringUtils.substringBefore(entry, ":").equals(suffix))
|
||||
ExceptionHandler.throwErrorResponse(new CompromisedPasswordResponse());
|
||||
|
|
|
|||
|
|
@ -1,13 +1,6 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import com.grack.nanojson.JsonParserException;
|
||||
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import me.kavin.piped.utils.obj.SolvedCaptcha;
|
||||
import okhttp3.FormBody;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.schabi.newpipe.extractor.downloader.Downloader;
|
||||
import org.schabi.newpipe.extractor.downloader.Request;
|
||||
import org.schabi.newpipe.extractor.downloader.Response;
|
||||
|
|
@ -18,7 +11,7 @@ import java.io.IOException;
|
|||
import java.net.HttpCookie;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
public class DownloaderImpl extends Downloader {
|
||||
|
||||
|
|
@ -41,70 +34,85 @@ public class DownloaderImpl extends Downloader {
|
|||
|
||||
request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value)));
|
||||
|
||||
var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
|
||||
var future = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
|
||||
|
||||
if (resp.status() == 429) {
|
||||
// Recaptcha solver code
|
||||
// Commented out, as it hasn't been ported to reqwest4j yet
|
||||
// Also, this was last seen a long time back
|
||||
|
||||
synchronized (cookie_lock) {
|
||||
// future.thenAcceptAsync(resp -> {
|
||||
// if (resp.status() == 429) {
|
||||
// synchronized (cookie_lock) {
|
||||
//
|
||||
// if (saved_cookie != null && saved_cookie.hasExpired()
|
||||
// || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
|
||||
// saved_cookie = null;
|
||||
//
|
||||
// String redir_url = String.valueOf(resp.finalUrl());
|
||||
//
|
||||
// if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
|
||||
//
|
||||
// var formBuilder = new FormBody.Builder();
|
||||
// String sitekey = null, data_s = null;
|
||||
//
|
||||
// for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
|
||||
// String name;
|
||||
// if (!(name = el.tagName()).equals("script")) {
|
||||
// if (name.equals("input"))
|
||||
// formBuilder.add(el.attr("name"), el.attr("value"));
|
||||
// else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
|
||||
// sitekey = el.attr("data-sitekey");
|
||||
// data_s = el.attr("data-s");
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
|
||||
// ExceptionHandler.handle(new ReCaptchaException("Could not get recaptcha", redir_url));
|
||||
//
|
||||
// SolvedCaptcha solved = null;
|
||||
//
|
||||
// try {
|
||||
// solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
|
||||
// } catch (JsonParserException | InterruptedException | IOException e) {
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
//
|
||||
// formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
|
||||
//
|
||||
// var formReqBuilder = new okhttp3.Request.Builder()
|
||||
// .url("https://www.google.com/sorry/index")
|
||||
// .header("User-Agent", Constants.USER_AGENT)
|
||||
// .post(formBuilder.build());
|
||||
//
|
||||
// okhttp3.Response formResponse;
|
||||
// try {
|
||||
// formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
|
||||
// } catch (IOException e) {
|
||||
// throw new RuntimeException(e);
|
||||
// }
|
||||
//
|
||||
// saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
|
||||
// .substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
|
||||
// .get(0);
|
||||
// cookie_received = System.currentTimeMillis();
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }, Multithreading.getCachedExecutor());
|
||||
|
||||
if (saved_cookie != null && saved_cookie.hasExpired()
|
||||
|| (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
|
||||
saved_cookie = null;
|
||||
var responseFuture = future.thenApplyAsync(resp -> {
|
||||
Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
|
||||
.collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
|
||||
|
||||
String redir_url = String.valueOf(resp.finalUrl());
|
||||
|
||||
if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
|
||||
|
||||
var formBuilder = new FormBody.Builder();
|
||||
String sitekey = null, data_s = null;
|
||||
|
||||
for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
|
||||
String name;
|
||||
if (!(name = el.tagName()).equals("script")) {
|
||||
if (name.equals("input"))
|
||||
formBuilder.add(el.attr("name"), el.attr("value"));
|
||||
else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
|
||||
sitekey = el.attr("data-sitekey");
|
||||
data_s = el.attr("data-s");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
|
||||
throw new ReCaptchaException("Could not get recaptcha", redir_url);
|
||||
|
||||
SolvedCaptcha solved = null;
|
||||
|
||||
try {
|
||||
solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
|
||||
} catch (JsonParserException | InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
|
||||
|
||||
var formReqBuilder = new okhttp3.Request.Builder()
|
||||
.url("https://www.google.com/sorry/index")
|
||||
.header("User-Agent", Constants.USER_AGENT)
|
||||
.post(formBuilder.build());
|
||||
|
||||
var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
|
||||
|
||||
saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
|
||||
.substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
|
||||
.get(0);
|
||||
cookie_received = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired.
|
||||
execute(request);
|
||||
}
|
||||
return new Response(resp.status(), null, headerMap, new String(resp.body()),
|
||||
resp.finalUrl());
|
||||
}, Multithreading.getCachedExecutor());
|
||||
|
||||
try {
|
||||
return responseFuture.get();
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
|
||||
Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
|
||||
.collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
|
||||
|
||||
return new Response(resp.status(), null, headerMap, new String(resp.body()),
|
||||
resp.finalUrl());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ package me.kavin.piped.utils;
|
|||
import me.kavin.piped.consts.Constants;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import rocks.kavin.reqwest4j.ReqwestUtils;
|
||||
import rocks.kavin.reqwest4j.Response;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -13,19 +13,20 @@ import static me.kavin.piped.utils.URLUtils.silentEncode;
|
|||
|
||||
public class LbryHelper {
|
||||
|
||||
public static String getLBRYId(String videoId) throws IOException {
|
||||
public static String getLBRYId(String videoId) throws Exception {
|
||||
|
||||
if (Constants.DISABLE_LBRY)
|
||||
return null;
|
||||
|
||||
return RequestUtils.sendGetJson("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId))
|
||||
.at("/data/videos")
|
||||
.path(videoId)
|
||||
.asText(null);
|
||||
.thenApplyAsync(json -> json.at("/data/videos")
|
||||
.path(videoId)
|
||||
.asText(null)
|
||||
).get();
|
||||
}
|
||||
|
||||
public static String getLBRYStreamURL(String lbryId)
|
||||
throws IOException {
|
||||
throws Exception {
|
||||
|
||||
if (StringUtils.isEmpty(lbryId))
|
||||
return null;
|
||||
|
|
@ -41,7 +42,7 @@ public class LbryHelper {
|
|||
.put("uri", "lbry://" + lbryId)
|
||||
.put("save_file", true)
|
||||
)
|
||||
), Map.of("Content-Type", "application/json"));
|
||||
), Map.of("Content-Type", "application/json")).get();
|
||||
if (resp.status() / 100 == 2) {
|
||||
return mapper.readTree(resp.body())
|
||||
.at("/result/streaming_url")
|
||||
|
|
@ -60,12 +61,12 @@ public class LbryHelper {
|
|||
// LBRY provides non UTF-8 characters in the URL, which causes issues
|
||||
streamUrl = new URI(streamUrl).toASCIIString();
|
||||
|
||||
var resp = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of(
|
||||
"Origin", "https://odysee.com",
|
||||
"Referer", "https://odysee.com/"
|
||||
));
|
||||
|
||||
final String lastLocation = resp.finalUrl();
|
||||
final String lastLocation = ReqwestUtils.fetch(streamUrl, "HEAD", null, Map.of(
|
||||
"Origin", "https://odysee.com",
|
||||
"Referer", "https://odysee.com/"
|
||||
))
|
||||
.thenApply(Response::finalUrl)
|
||||
.get();
|
||||
|
||||
return streamUrl.equals(lastLocation) ? null : lastLocation;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,11 +44,15 @@ public class PubSubHelper {
|
|||
var buffer = new Buffer();
|
||||
formBuilder.build().writeTo(buffer);
|
||||
|
||||
var resp = ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of());
|
||||
|
||||
if (resp.status() != 202)
|
||||
System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
|
||||
|
||||
ReqwestUtils.fetch(Constants.PUBSUB_HUB_URL, "POST", buffer.readByteArray(), Map.of())
|
||||
.thenAccept(resp -> {
|
||||
if (resp.status() != 202)
|
||||
System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
|
||||
})
|
||||
.exceptionally(e -> {
|
||||
ExceptionHandler.handle((Exception) e);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
package me.kavin.piped.utils;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import me.kavin.piped.consts.Constants;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
import rocks.kavin.reqwest4j.ReqwestUtils;
|
||||
|
|
@ -9,27 +8,26 @@ import rocks.kavin.reqwest4j.Response;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
|
||||
import static me.kavin.piped.consts.Constants.mapper;
|
||||
|
||||
public class RequestUtils {
|
||||
|
||||
public static Response sendGetRaw(String url) throws IOException {
|
||||
public static CompletableFuture<Response> sendGetRaw(String url) throws Exception {
|
||||
return ReqwestUtils.fetch(url, "GET", null, Map.of());
|
||||
}
|
||||
|
||||
public static String sendGet(String url) throws IOException {
|
||||
return new String(
|
||||
ReqwestUtils.fetch(url, "GET", null, Map.of())
|
||||
.body()
|
||||
);
|
||||
public static CompletableFuture<String> sendGet(String url) throws Exception {
|
||||
return ReqwestUtils.fetch(url, "GET", null, Map.of())
|
||||
.thenApply(Response::body)
|
||||
.thenApplyAsync(String::new);
|
||||
}
|
||||
|
||||
public static String sendGet(String url, String ua) throws IOException {
|
||||
return new String(
|
||||
ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
|
||||
.body()
|
||||
);
|
||||
public static CompletableFuture<String> sendGet(String url, String ua) throws Exception {
|
||||
return ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
|
||||
.thenApply(Response::body)
|
||||
.thenApplyAsync(String::new);
|
||||
}
|
||||
|
||||
public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException {
|
||||
|
|
@ -44,13 +42,13 @@ public class RequestUtils {
|
|||
}
|
||||
}
|
||||
|
||||
public static JsonNode sendGetJson(String url, String ua) throws IOException {
|
||||
return getJsonNode(Constants.h2client, new Request.Builder().header("User-Agent", ua).url(url).build());
|
||||
}
|
||||
|
||||
public static JsonNode sendGetJson(String url) throws IOException {
|
||||
|
||||
return mapper.readTree(ReqwestUtils.fetch(url, "GET", null, Map.of()).body());
|
||||
|
||||
public static CompletableFuture<JsonNode> sendGetJson(String url) throws Exception {
|
||||
return ReqwestUtils.fetch(url, "GET", null, Map.of()).thenApply(Response::body).thenApplyAsync(resp -> {
|
||||
try {
|
||||
return mapper.readTree(resp);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Failed to parse JSON", e);
|
||||
}
|
||||
}, Multithreading.getCachedExecutor());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,25 +3,17 @@ package me.kavin.piped.utils;
|
|||
|
||||
import me.kavin.piped.consts.Constants;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import static me.kavin.piped.consts.Constants.mapper;
|
||||
import static me.kavin.piped.utils.RequestUtils.sendGetRaw;
|
||||
import static me.kavin.piped.utils.RequestUtils.sendGetJson;
|
||||
|
||||
public class RydHelper {
|
||||
public static double getDislikeRating(String videoId) throws IOException {
|
||||
public static double getDislikeRating(String videoId) throws Exception {
|
||||
|
||||
if (Constants.DISABLE_RYD)
|
||||
return -1;
|
||||
|
||||
var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId);
|
||||
|
||||
if (resp.status() / 100 != 2)
|
||||
return -1;
|
||||
|
||||
return mapper.readTree(resp.body())
|
||||
.path("rating")
|
||||
.asDouble(-1);
|
||||
return sendGetJson(Constants.RYD_PROXY_URL + "/votes/" + videoId)
|
||||
.thenApply(tree -> tree.path("rating").asDouble(-1))
|
||||
.get();
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ public class SponsorBlockUtils {
|
|||
try {
|
||||
|
||||
var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
|
||||
+ "?categories=" + URLUtils.silentEncode(categories));
|
||||
+ "?categories=" + URLUtils.silentEncode(categories)).get();
|
||||
|
||||
if (resp.status() == 200) {
|
||||
var any = mapper.readTree(resp.body());
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ public class MatrixHelper {
|
|||
.asText();
|
||||
}
|
||||
|
||||
AUTHORIZED_USERS = (ArrayNode) mapper.readTree(RequestUtils.sendGet("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json"));
|
||||
AUTHORIZED_USERS = (ArrayNode) RequestUtils.sendGetJson("https://raw.githubusercontent.com/TeamPiped/piped-federation/main/authorized-users.json").get();
|
||||
|
||||
} catch (Exception e) {
|
||||
ExceptionHandler.handle(e);
|
||||
|
|
|
|||
Loading…
Reference in New Issue