Use reqwest4j instead of okhttp.

This commit is contained in:
Kavin 2023-01-29 12:16:57 +00:00
parent 9f06b7eec2
commit 36af3af096
No known key found for this signature in database
GPG Key ID: 49451E4482CC5BCD
9 changed files with 129 additions and 141 deletions

View File

@ -40,6 +40,7 @@ dependencies {
implementation 'com.squareup.okhttp3:okhttp' implementation 'com.squareup.okhttp3:okhttp'
implementation 'com.squareup.okhttp3:okhttp-brotli' implementation 'com.squareup.okhttp3:okhttp-brotli'
implementation 'io.sentry:sentry:6.11.0' implementation 'io.sentry:sentry:6.11.0'
implementation 'rocks.kavin:reqwest4j:1.0'
} }
shadowJar { shadowJar {

View File

@ -29,7 +29,7 @@ import java.util.regex.Pattern;
public class Constants { public class Constants {
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0"; public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:102.0) Gecko/20100101 Firefox/102.0";
public static final int PORT; public static final int PORT;
public static final String HTTP_WORKERS; public static final String HTTP_WORKERS;

View File

@ -1,10 +1,10 @@
package me.kavin.piped.utils; package me.kavin.piped.utils;
import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonParserException;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import me.kavin.piped.consts.Constants; import me.kavin.piped.consts.Constants;
import me.kavin.piped.utils.obj.SolvedCaptcha; import me.kavin.piped.utils.obj.SolvedCaptcha;
import okhttp3.FormBody; import okhttp3.FormBody;
import okhttp3.RequestBody;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@ -12,9 +12,12 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Request; import org.schabi.newpipe.extractor.downloader.Request;
import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.io.IOException; import java.io.IOException;
import java.net.HttpCookie; import java.net.HttpCookie;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
public class DownloaderImpl extends Downloader { public class DownloaderImpl extends Downloader {
@ -31,23 +34,16 @@ public class DownloaderImpl extends Downloader {
// TODO: HTTP/3 aka QUIC // TODO: HTTP/3 aka QUIC
var bytes = request.dataToSend(); var bytes = request.dataToSend();
RequestBody body = null; Map<String, String> headers = new Object2ObjectOpenHashMap<>();
if (bytes != null)
body = RequestBody.create(bytes);
var builder = new okhttp3.Request.Builder()
.url(request.url())
.method(request.httpMethod(), body)
.header("User-Agent", Constants.USER_AGENT);
if (saved_cookie != null && !saved_cookie.hasExpired()) if (saved_cookie != null && !saved_cookie.hasExpired())
builder.header("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue()); headers.put("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue());
request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value))); request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value)));
var resp = Constants.h2client.newCall(builder.build()).execute(); var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
if (resp.code() == 429) { if (resp.status() == 429) {
synchronized (cookie_lock) { synchronized (cookie_lock) {
@ -55,14 +51,14 @@ public class DownloaderImpl extends Downloader {
|| (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30))) || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
saved_cookie = null; saved_cookie = null;
String redir_url = String.valueOf(resp.request().url()); String redir_url = String.valueOf(resp.finalUrl());
if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) { if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
var formBuilder = new FormBody.Builder(); var formBuilder = new FormBody.Builder();
String sitekey = null, data_s = null; String sitekey = null, data_s = null;
for (Element el : Jsoup.parse(resp.body().string()).selectFirst("form").children()) { for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
String name; String name;
if (!(name = el.tagName()).equals("script")) { if (!(name = el.tagName()).equals("script")) {
if (name.equals("input")) if (name.equals("input"))
@ -105,11 +101,10 @@ public class DownloaderImpl extends Downloader {
} }
var response = new Response(resp.code(), resp.message(), resp.headers().toMultimap(), resp.body().string(), Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
String.valueOf(resp.request().url())); .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
resp.close(); return new Response(resp.status(), null, headerMap, new String(resp.body()),
resp.finalUrl());
return response;
} }
} }

View File

@ -1,16 +1,13 @@
package me.kavin.piped.utils; package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants; import me.kavin.piped.consts.Constants;
import okhttp3.MediaType;
import okhttp3.Request;
import okhttp3.RequestBody;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.io.IOException; import java.io.IOException;
import java.util.Map;
import static me.kavin.piped.consts.Constants.h2client;
import static me.kavin.piped.consts.Constants.mapper; import static me.kavin.piped.consts.Constants.mapper;
import static me.kavin.piped.utils.RequestUtils.sendGet;
import static me.kavin.piped.utils.URLUtils.silentEncode; import static me.kavin.piped.utils.URLUtils.silentEncode;
public class LbryHelper { public class LbryHelper {
@ -20,7 +17,7 @@ public class LbryHelper {
if (Constants.DISABLE_LBRY) if (Constants.DISABLE_LBRY)
return null; return null;
return mapper.readTree(sendGet("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId))) return RequestUtils.sendGetJson("https://api.lbry.com/yt/resolve?video_ids=" + silentEncode(videoId))
.at("/data/videos") .at("/data/videos")
.path(videoId) .path(videoId)
.asText(null); .asText(null);
@ -32,11 +29,9 @@ public class LbryHelper {
if (StringUtils.isEmpty(lbryId)) if (StringUtils.isEmpty(lbryId))
return null; return null;
var request = new Request.Builder() var resp = ReqwestUtils.fetch("https://api.na-backend.odysee.com/api/v1/proxy?m=get", "POST",
.url("https://api.na-backend.odysee.com/api/v1/proxy?m=get") mapper.writeValueAsBytes(
.post(RequestBody.create(mapper.writeValueAsBytes(
mapper.createObjectNode() mapper.createObjectNode()
.put("id", System.currentTimeMillis())
.put("id", System.currentTimeMillis()) .put("id", System.currentTimeMillis())
.put("jsonrpc", "2.0") .put("jsonrpc", "2.0")
.put("method", "get") .put("method", "get")
@ -45,16 +40,12 @@ public class LbryHelper {
.put("uri", "lbry://" + lbryId) .put("uri", "lbry://" + lbryId)
.put("save_file", true) .put("save_file", true)
) )
), MediaType.get("application/json"))) ), Map.of("Content-Type", "application/json"));
.build(); if (resp.status() / 100 == 2) {
return mapper.readTree(resp.body())
try (var resp = h2client.newCall(request).execute()) {
if (resp.isSuccessful()) {
return mapper.readTree(resp.body().byteStream())
.at("/result/streaming_url") .at("/result/streaming_url")
.asText(null); .asText(null);
} }
}
return null; return null;
} }

View File

@ -3,11 +3,12 @@ package me.kavin.piped.utils;
import me.kavin.piped.consts.Constants; import me.kavin.piped.consts.Constants;
import me.kavin.piped.utils.obj.db.PubSub; import me.kavin.piped.utils.obj.db.PubSub;
import okhttp3.FormBody; import okhttp3.FormBody;
import okhttp3.Request; import okio.Buffer;
import org.hibernate.StatelessSession; import org.hibernate.StatelessSession;
import rocks.kavin.reqwest4j.ReqwestUtils;
import java.io.IOException; import java.io.IOException;
import java.util.Objects; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
public class PubSubHelper { public class PubSubHelper {
@ -23,9 +24,6 @@ public class PubSubHelper {
String callback = Constants.PUBSUB_URL + "/webhooks/pubsub"; String callback = Constants.PUBSUB_URL + "/webhooks/pubsub";
String topic = "https://www.youtube.com/xml/feeds/videos.xml?channel_id=" + channelId; String topic = "https://www.youtube.com/xml/feeds/videos.xml?channel_id=" + channelId;
var builder = new Request.Builder()
.url(Constants.PUBSUB_HUB_URL);
var formBuilder = new FormBody.Builder(); var formBuilder = new FormBody.Builder();
formBuilder.add("hub.callback", callback); formBuilder.add("hub.callback", callback);
@ -42,18 +40,18 @@ public class PubSubHelper {
tr.commit(); tr.commit();
} }
try (var resp = Constants.h2client // write form to read later
.newCall(builder.post(formBuilder.build()) var buffer = new Buffer();
.build()).execute()) { formBuilder.build().writeTo(buffer);
if (resp.code() != 202) var resp = ReqwestUtils.fetch(callback, "POST", buffer.readByteArray(), Map.of());
System.out.println("Failed to subscribe: " + resp.code() + "\n" + Objects.requireNonNull(resp.body()).string());
if (resp.status() != 202)
System.out.println("Failed to subscribe: " + resp.status() + "\n" + new String(resp.body()));
} }
} }
}
public static void updatePubSub(String channelId) { public static void updatePubSub(String channelId) {
var pubsub = DatabaseHelper.getPubSubFromId(channelId); var pubsub = DatabaseHelper.getPubSubFromId(channelId);
try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) {

View File

@ -4,38 +4,38 @@ import com.fasterxml.jackson.databind.JsonNode;
import me.kavin.piped.consts.Constants; import me.kavin.piped.consts.Constants;
import okhttp3.OkHttpClient; import okhttp3.OkHttpClient;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import rocks.kavin.reqwest4j.ReqwestUtils;
import rocks.kavin.reqwest4j.Response;
import java.io.IOException; import java.io.IOException;
import java.util.Map;
import static me.kavin.piped.consts.Constants.mapper;
public class RequestUtils { public class RequestUtils {
public static Response sendGetRaw(String url) throws IOException { public static Response sendGetRaw(String url) throws IOException {
return sendGetRaw(url, Constants.USER_AGENT); return ReqwestUtils.fetch(url, "GET", null, Map.of());
}
public static Response sendGetRaw(String url, String ua) throws IOException {
var request = new Request.Builder().header("User-Agent", ua).url(url).build();
return Constants.h2client.newCall(request).execute();
} }
public static String sendGet(String url) throws IOException { public static String sendGet(String url) throws IOException {
return sendGet(url, Constants.USER_AGENT); return new String(
ReqwestUtils.fetch(url, "GET", null, Map.of())
.body()
);
} }
public static String sendGet(String url, String ua) throws IOException { public static String sendGet(String url, String ua) throws IOException {
return new String(
var response = sendGetRaw(url, ua); ReqwestUtils.fetch(url, "GET", null, Map.of("User-Agent", ua))
var responseString = response.body().string(); .body()
response.close(); );
return responseString;
} }
public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException { public static JsonNode getJsonNode(OkHttpClient client, Request request) throws IOException {
try (var resp = client.newCall(request).execute()) { try (var resp = client.newCall(request).execute()) {
try { try {
return Constants.mapper.readTree(resp.body().byteStream()); return mapper.readTree(resp.body().byteStream());
} catch (Exception e) { } catch (Exception e) {
if (!resp.isSuccessful()) if (!resp.isSuccessful())
ExceptionHandler.handle(e); ExceptionHandler.handle(e);
@ -49,6 +49,8 @@ public class RequestUtils {
} }
public static JsonNode sendGetJson(String url) throws IOException { public static JsonNode sendGetJson(String url) throws IOException {
return sendGetJson(url, Constants.USER_AGENT);
return mapper.readTree(ReqwestUtils.fetch(url, "GET", null, Map.of()).body());
} }
} }

View File

@ -14,15 +14,14 @@ public class RydHelper {
if (Constants.DISABLE_RYD) if (Constants.DISABLE_RYD)
return -1; return -1;
try (var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId)) { var resp = sendGetRaw(Constants.RYD_PROXY_URL + "/votes/" + videoId);
if (!resp.isSuccessful()) if (resp.status() / 100 != 2)
return -1; return -1;
return mapper.readTree(resp.body().byteStream()) return mapper.readTree(resp.body())
.path("rating") .path("rating")
.asDouble(-1); .asDouble(-1);
} }
} }
}

View File

@ -25,11 +25,13 @@ public class SponsorBlockUtils {
String hash = toSha256(id); String hash = toSha256(id);
for (String url : Constants.SPONSORBLOCK_SERVERS) { for (String url : Constants.SPONSORBLOCK_SERVERS) {
try (var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4)) try {
+ "?categories=" + URLUtils.silentEncode(categories))) {
if (resp.code() == 200) { var resp = RequestUtils.sendGetRaw(url + "/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
JsonArray jArray = JsonParser.array().from(resp.body().string()); + "?categories=" + URLUtils.silentEncode(categories));
if (resp.status() == 200) {
JsonArray jArray = JsonParser.array().from(new String(resp.body()));
jArray.removeIf(jObject -> !((JsonObject) jObject).getString("videoID").equalsIgnoreCase(id)); jArray.removeIf(jObject -> !((JsonObject) jObject).getString("videoID").equalsIgnoreCase(id));