From c07cf5fd1f6b6cb0290418795c8eb6cbea193d9b Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Wed, 2 Feb 2022 21:05:22 +0000 Subject: [PATCH] Refactor Utility methods and migrate to OkHttp (#179) --- build.gradle | 3 + gradle/wrapper/gradle-wrapper.properties | 2 +- .../java/me/kavin/piped/ServerLauncher.java | 49 ++- .../java/me/kavin/piped/consts/Constants.java | 18 +- .../me/kavin/piped/utils/CaptchaSolver.java | 39 ++- .../piped/utils/CustomServletDecorator.java | 14 +- .../me/kavin/piped/utils/DownloaderImpl.java | 107 +++---- .../kavin/piped/utils/ExceptionHandler.java | 9 +- .../java/me/kavin/piped/utils/LbryHelper.java | 39 +++ .../me/kavin/piped/utils/RequestUtils.java | 17 +- .../me/kavin/piped/utils/ResponseHelper.java | 284 +++++------------- .../kavin/piped/utils/SponsorBlockUtils.java | 32 +- .../java/me/kavin/piped/utils/URLUtils.java | 42 ++- .../me/kavin/piped/utils/obj/db/Channel.java | 2 +- .../me/kavin/piped/utils/obj/db/Video.java | 9 +- 15 files changed, 276 insertions(+), 390 deletions(-) create mode 100644 src/main/java/me/kavin/piped/utils/LbryHelper.java diff --git a/build.gradle b/build.gradle index 4ba9df8..49e03dd 100644 --- a/build.gradle +++ b/build.gradle @@ -38,6 +38,9 @@ dependencies { implementation 'com.zaxxer:HikariCP:5.0.1' implementation 'org.springframework.security:spring-security-crypto:5.6.1' implementation 'commons-logging:commons-logging:1.2' + implementation(platform("com.squareup.okhttp3:okhttp-bom:4.9.3")) + implementation("com.squareup.okhttp3:okhttp") + implementation("com.squareup.okhttp3:okhttp-brotli") } shadowJar { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 55c6e6e..ab27071 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://downloads.gradle.org/distributions/gradle-7.2-bin.zip +distributionUrl=https\://downloads.gradle.org/distributions/gradle-7.3.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/me/kavin/piped/ServerLauncher.java b/src/main/java/me/kavin/piped/ServerLauncher.java index 34c1954..71893c6 100644 --- a/src/main/java/me/kavin/piped/ServerLauncher.java +++ b/src/main/java/me/kavin/piped/ServerLauncher.java @@ -1,28 +1,8 @@ package me.kavin.piped; -import static io.activej.config.converter.ConfigConverters.ofInetSocketAddress; -import static io.activej.http.HttpHeaders.AUTHORIZATION; -import static io.activej.http.HttpHeaders.CACHE_CONTROL; -import static io.activej.http.HttpHeaders.CONTENT_TYPE; -import static io.activej.http.HttpHeaders.LINK; -import static io.activej.http.HttpHeaders.LOCATION; -import static io.activej.http.HttpMethod.GET; -import static io.activej.http.HttpMethod.POST; - -import java.io.ByteArrayInputStream; -import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.Executor; - -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.hibernate.Session; -import org.jetbrains.annotations.NotNull; -import org.xml.sax.InputSource; - import com.fasterxml.jackson.core.JsonProcessingException; import com.rometools.rome.feed.synd.SyndFeed; import com.rometools.rome.io.SyndFeedInput; - import io.activej.config.Config; import io.activej.http.AsyncServlet; import io.activej.http.HttpMethod; @@ -33,15 +13,25 @@ import io.activej.inject.module.AbstractModule; import io.activej.inject.module.Module; import io.activej.launchers.http.MultithreadedHttpServerLauncher; import me.kavin.piped.consts.Constants; -import me.kavin.piped.utils.CustomServletDecorator; -import me.kavin.piped.utils.DatabaseSessionFactory; -import me.kavin.piped.utils.ExceptionHandler; -import me.kavin.piped.utils.Multithreading; -import me.kavin.piped.utils.ResponseHelper; -import me.kavin.piped.utils.SponsorBlockUtils; +import me.kavin.piped.utils.*; import me.kavin.piped.utils.resp.ErrorResponse; import me.kavin.piped.utils.resp.LoginRequest; import me.kavin.piped.utils.resp.SubscriptionUpdateRequest; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.hibernate.Session; +import org.jetbrains.annotations.NotNull; +import org.xml.sax.InputSource; + +import java.io.ByteArrayInputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.concurrent.Executor; + +import static io.activej.config.converter.ConfigConverters.ofInetSocketAddress; +import static io.activej.http.HttpHeaders.*; +import static io.activej.http.HttpMethod.GET; +import static io.activej.http.HttpMethod.POST; public class ServerLauncher extends MultithreadedHttpServerLauncher { @@ -55,9 +45,8 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher { RoutingServlet router = RoutingServlet.create() .map(HttpMethod.OPTIONS, "/*", request -> HttpResponse.ofCode(200)) - .map(GET, "/webhooks/pubsub", request -> { - return HttpResponse.ok200().withPlainText(request.getQueryParameter("hub.challenge")); - }).map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> { + .map(GET, "/webhooks/pubsub", request -> HttpResponse.ok200().withPlainText(Objects.requireNonNull(request.getQueryParameter("hub.challenge")))) + .map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> { try { SyndFeed feed = new SyndFeedInput().build( @@ -320,7 +309,7 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher { } private @NotNull HttpResponse getRawResponse(int code, byte[] body, String contentType, String cache, - boolean prefetchProxy) { + boolean prefetchProxy) { HttpResponse response = HttpResponse.ofCode(code).withBody(body).withHeader(CONTENT_TYPE, contentType) .withHeader(CACHE_CONTROL, cache); if (prefetchProxy) diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index 234687a..963ccbb 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -3,6 +3,8 @@ package me.kavin.piped.consts; import com.fasterxml.jackson.databind.ObjectMapper; import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import me.kavin.piped.utils.PageMixin; +import okhttp3.OkHttpClient; +import okhttp3.brotli.BrotliInterceptor; import org.apache.commons.lang3.StringUtils; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.Page; @@ -11,10 +13,6 @@ import org.schabi.newpipe.extractor.StreamingService; import java.io.FileReader; import java.net.InetSocketAddress; import java.net.ProxySelector; -import java.net.http.HttpClient; -import java.net.http.HttpClient.Builder; -import java.net.http.HttpClient.Redirect; -import java.net.http.HttpClient.Version; import java.util.Properties; public class Constants { @@ -36,8 +34,8 @@ public class Constants { public static final String FRONTEND_URL; - public static final HttpClient h2client; - public static final HttpClient h2_no_redir_client; + public static final OkHttpClient h2client; + public static final OkHttpClient h2_no_redir_client; public static final boolean COMPROMISED_PASSWORD_CHECK; @@ -76,8 +74,12 @@ public class Constants { if (key.startsWith("hibernate")) hibernateProperties.put(key, value); }); - Builder builder = HttpClient.newBuilder().followRedirects(Redirect.NORMAL).version(Version.HTTP_1_1); - Builder builder_noredir = HttpClient.newBuilder().followRedirects(Redirect.NEVER).version(Version.HTTP_1_1); + OkHttpClient.Builder builder = new OkHttpClient.Builder() + .followRedirects(true) + .addInterceptor(BrotliInterceptor.INSTANCE); + OkHttpClient.Builder builder_noredir = new OkHttpClient.Builder() + .followRedirects(false) + .addInterceptor(BrotliInterceptor.INSTANCE); if (HTTP_PROXY != null && HTTP_PROXY.contains(":")) { String host = StringUtils.substringBefore(HTTP_PROXY, ":"); String port = StringUtils.substringAfter(HTTP_PROXY, ":"); diff --git a/src/main/java/me/kavin/piped/utils/CaptchaSolver.java b/src/main/java/me/kavin/piped/utils/CaptchaSolver.java index ac867cc..2032928 100644 --- a/src/main/java/me/kavin/piped/utils/CaptchaSolver.java +++ b/src/main/java/me/kavin/piped/utils/CaptchaSolver.java @@ -1,21 +1,18 @@ package me.kavin.piped.utils; -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpRequest; -import java.net.http.HttpRequest.BodyPublishers; -import java.net.http.HttpRequest.Builder; -import java.net.http.HttpResponse.BodyHandlers; -import java.util.Map; - import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; import com.grack.nanojson.JsonParserException; import com.grack.nanojson.JsonWriter; - import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import me.kavin.piped.consts.Constants; import me.kavin.piped.utils.obj.SolvedCaptcha; +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; + +import java.io.IOException; +import java.util.Map; public class CaptchaSolver { @@ -29,9 +26,8 @@ public class CaptchaSolver { } private static int createTask(String url, String sitekey, String data_s) - throws JsonParserException, IOException, InterruptedException { + throws JsonParserException, IOException { - Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "createTask")); JsonObject jObject = new JsonObject(); jObject.put("clientKey", Constants.CAPTCHA_API_KEY); { @@ -43,17 +39,16 @@ public class CaptchaSolver { jObject.put("task", task); } - builder.method("POST", BodyPublishers.ofString(JsonWriter.string(jObject))); - - builder.header("Content-Type", "application/json"); + var builder = new Request.Builder().url(Constants.CAPTCHA_BASE_URL + "createTask") + .post(RequestBody.create(JsonWriter.string(jObject), MediaType.get("application/json"))); JsonObject taskObj = JsonParser.object() - .from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body()); + .from(Constants.h2client.newCall(builder.build()).execute().body().byteStream()); return taskObj.getInt("taskId"); } - private static final SolvedCaptcha waitForSolve(int taskId) + private static SolvedCaptcha waitForSolve(int taskId) throws JsonParserException, IOException, InterruptedException { String body = JsonWriter.string( @@ -61,15 +56,15 @@ public class CaptchaSolver { SolvedCaptcha solved = null; - outer: while (true) { - Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "getTaskResult")); - - builder.method("POST", BodyPublishers.ofString(body)); + while (true) { + var builder = new Request.Builder() + .url(Constants.CAPTCHA_BASE_URL + "getTaskResult") + .post(RequestBody.create(body, MediaType.get("application/json"))); builder.header("Content-Type", "application/json"); JsonObject captchaObj = JsonParser.object() - .from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body()); + .from(Constants.h2client.newCall(builder.build()).execute().body().byteStream()); if (captchaObj.getInt("errorId") != 0) break; @@ -87,7 +82,7 @@ public class CaptchaSolver { }); solved = new SolvedCaptcha(cookies, captchaResp); - break outer; + break; } } diff --git a/src/main/java/me/kavin/piped/utils/CustomServletDecorator.java b/src/main/java/me/kavin/piped/utils/CustomServletDecorator.java index 39c5c9b..53b6ee5 100644 --- a/src/main/java/me/kavin/piped/utils/CustomServletDecorator.java +++ b/src/main/java/me/kavin/piped/utils/CustomServletDecorator.java @@ -1,18 +1,12 @@ package me.kavin.piped.utils; +import io.activej.http.*; +import io.activej.promise.Promisable; +import org.jetbrains.annotations.NotNull; + import static io.activej.http.HttpHeaders.ACCESS_CONTROL_ALLOW_HEADERS; import static io.activej.http.HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN; -import org.jetbrains.annotations.NotNull; - -import io.activej.http.AsyncServlet; -import io.activej.http.HttpHeader; -import io.activej.http.HttpHeaderValue; -import io.activej.http.HttpHeaders; -import io.activej.http.HttpRequest; -import io.activej.http.HttpResponse; -import io.activej.promise.Promisable; - public class CustomServletDecorator implements AsyncServlet { private static final HttpHeader HEADER = HttpHeaders.of("Server-Timing"); diff --git a/src/main/java/me/kavin/piped/utils/DownloaderImpl.java b/src/main/java/me/kavin/piped/utils/DownloaderImpl.java index a2a8632..01bfd94 100644 --- a/src/main/java/me/kavin/piped/utils/DownloaderImpl.java +++ b/src/main/java/me/kavin/piped/utils/DownloaderImpl.java @@ -1,18 +1,13 @@ package me.kavin.piped.utils; -import java.io.IOException; -import java.net.HttpCookie; -import java.net.URI; -import java.net.http.HttpRequest; -import java.net.http.HttpRequest.BodyPublisher; -import java.net.http.HttpRequest.BodyPublishers; -import java.net.http.HttpRequest.Builder; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.util.Map; -import java.util.concurrent.TimeUnit; - +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.Scheduler; +import com.grack.nanojson.JsonParserException; +import me.kavin.piped.consts.Constants; +import me.kavin.piped.utils.obj.SolvedCaptcha; +import okhttp3.FormBody; +import okhttp3.RequestBody; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jsoup.Jsoup; @@ -22,13 +17,9 @@ import org.schabi.newpipe.extractor.downloader.Request; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.LoadingCache; -import com.grack.nanojson.JsonParserException; - -import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; -import me.kavin.piped.consts.Constants; -import me.kavin.piped.utils.obj.SolvedCaptcha; +import java.io.IOException; +import java.net.HttpCookie; +import java.util.concurrent.TimeUnit; public class DownloaderImpl extends Downloader { @@ -52,30 +43,24 @@ public class DownloaderImpl extends Downloader { public Response executeRequest(Request request) throws IOException, ReCaptchaException { // TODO: HTTP/3 aka QUIC - Builder builder = HttpRequest.newBuilder(URI.create(request.url())); + var bytes = request.dataToSend(); + RequestBody body = null; + if (bytes != null) + body = RequestBody.create(bytes); - byte[] data = request.dataToSend(); - BodyPublisher publisher = data == null ? HttpRequest.BodyPublishers.noBody() - : HttpRequest.BodyPublishers.ofByteArray(data); - - builder.method(request.httpMethod(), publisher); - - builder.setHeader("User-Agent", Constants.USER_AGENT); + var builder = new okhttp3.Request.Builder() + .url(request.url()) + .method(request.httpMethod(), body) + .header("User-Agent", Constants.USER_AGENT); if (saved_cookie != null && !saved_cookie.hasExpired()) - builder.setHeader("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue()); + builder.header("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue()); request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value))); - HttpResponse response = null; + var response = Constants.h2client.newCall(builder.build()).execute(); - try { - response = Constants.h2client.send(builder.build(), BodyHandlers.ofString()); - } catch (InterruptedException e) { - // ignored - } - - if (response.statusCode() == 429) { + if (response.code() == 429) { synchronized (cookie_lock) { @@ -83,25 +68,25 @@ public class DownloaderImpl extends Downloader { || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30))) saved_cookie = null; - String redir_url = String.valueOf(response.request().uri()); + String redir_url = String.valueOf(response.request().url()); if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) { - Map formParams = new Object2ObjectOpenHashMap<>(); + var formBuilder = new FormBody.Builder(); String sitekey = null, data_s = null; - for (Element el : Jsoup.parse(response.body()).selectFirst("form").children()) { + for (Element el : Jsoup.parse(response.body().string()).selectFirst("form").children()) { String name; if (!(name = el.tagName()).equals("script")) { if (name.equals("input")) - formParams.put(el.attr("name"), el.attr("value")); + formBuilder.add(el.attr("name"), el.attr("value")); else if (name.equals("div") && el.attr("id").equals("recaptcha")) { sitekey = el.attr("data-sitekey"); data_s = el.attr("data-s"); } } } - if (sitekey == null || data_s == null) + if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s)) throw new ReCaptchaException("Could not get recaptcha", redir_url); SolvedCaptcha solved = null; @@ -112,35 +97,19 @@ public class DownloaderImpl extends Downloader { e.printStackTrace(); } - formParams.put("g-recaptcha-response", solved.getRecaptchaResponse()); + formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse()); - Builder formBuilder = HttpRequest.newBuilder(URI.create("https://www.google.com/sorry/index")); + var formReqBuilder = new okhttp3.Request.Builder() + .url("https://www.google.com/sorry/index") + .header("User-Agent", Constants.USER_AGENT) + .post(formBuilder.build()); - formBuilder.setHeader("User-Agent", Constants.USER_AGENT); + var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute(); - StringBuilder formBody = new StringBuilder(); - - formParams.forEach((name, value) -> { - formBody.append(name + "=" + URLUtils.silentEncode(value) + "&"); - }); - - formBuilder.header("content-type", "application/x-www-form-urlencoded"); - - formBuilder.method("POST", - BodyPublishers.ofString(String.valueOf(formBody.substring(0, formBody.length() - 1)))); - - try { - HttpResponse formResponse = Constants.h2_no_redir_client.send(formBuilder.build(), - BodyHandlers.ofString()); - - saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils - .substringAfter(formResponse.headers().firstValue("Location").get(), "google_abuse="))) - .get(0); - cookie_received = System.currentTimeMillis(); - - } catch (InterruptedException e) { - e.printStackTrace(); - } + saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils + .substringAfter(formResponse.headers().get("Location"), "google_abuse="))) + .get(0); + cookie_received = System.currentTimeMillis(); } if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired. @@ -149,7 +118,7 @@ public class DownloaderImpl extends Downloader { } - return new Response(response.statusCode(), "UNDEFINED", response.headers().map(), response.body(), - String.valueOf(response.uri())); + return new Response(response.code(), response.message(), response.headers().toMultimap(), response.body().string(), + String.valueOf(response.request().url())); } } diff --git a/src/main/java/me/kavin/piped/utils/ExceptionHandler.java b/src/main/java/me/kavin/piped/utils/ExceptionHandler.java index cca316c..31253cd 100644 --- a/src/main/java/me/kavin/piped/utils/ExceptionHandler.java +++ b/src/main/java/me/kavin/piped/utils/ExceptionHandler.java @@ -1,12 +1,10 @@ package me.kavin.piped.utils; +import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; + import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; -import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException; -import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; -import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException; - public class ExceptionHandler { public static Exception handle(Exception e) { @@ -18,8 +16,7 @@ public class ExceptionHandler { if (e.getCause() != null && (e instanceof ExecutionException || e instanceof CompletionException)) e = (Exception) e.getCause(); - if (!(e instanceof AgeRestrictedContentException || e instanceof ContentNotAvailableException - || e instanceof GeographicRestrictionException)) { + if (!(e instanceof ContentNotAvailableException)) { if (path != null) System.err.println("An error occoured in the path: " + path); e.printStackTrace(); diff --git a/src/main/java/me/kavin/piped/utils/LbryHelper.java b/src/main/java/me/kavin/piped/utils/LbryHelper.java new file mode 100644 index 0000000..1fc3156 --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/LbryHelper.java @@ -0,0 +1,39 @@ +package me.kavin.piped.utils; + +import me.kavin.piped.consts.Constants; +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import org.json.JSONObject; + +import java.io.IOException; + +public class LbryHelper { + + public static String getLBRYId(String videoId) throws IOException, InterruptedException { + return new JSONObject( + RequestUtils.sendGet("https://api.lbry.com/yt/resolve?video_ids=" + URLUtils.silentEncode(videoId)) + ).getJSONObject("data").getJSONObject("videos").optString(videoId, null); + } + + public static String getLBRYStreamURL(String lbryId) + throws IOException { + var request = new Request.Builder() + .url("https://api.lbry.tv/api/v1/proxy?m=get") + .post(RequestBody.create(String.valueOf( + new JSONObject().put("id", System.currentTimeMillis()) + .put("jsonrpc", "2.0") + .put("method", "get") + .put("params", + new JSONObject() + .put("uri", "lbry://" + lbryId) + .put("save_file", true))) + , MediaType.get("application/json"))) + .build(); + + return new JSONObject( + Constants.h2client.newCall(request).execute().body().string() + ).getJSONObject("result").getString("streaming_url"); + + } +} diff --git a/src/main/java/me/kavin/piped/utils/RequestUtils.java b/src/main/java/me/kavin/piped/utils/RequestUtils.java index d52bb09..2ea523d 100644 --- a/src/main/java/me/kavin/piped/utils/RequestUtils.java +++ b/src/main/java/me/kavin/piped/utils/RequestUtils.java @@ -1,23 +1,20 @@ package me.kavin.piped.utils; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse.BodyHandlers; - import me.kavin.piped.consts.Constants; +import okhttp3.Request; + +import java.io.IOException; public class RequestUtils { - public static String sendGet(String url) throws IOException, InterruptedException, URISyntaxException { + public static String sendGet(String url) throws IOException { return sendGet(url, Constants.USER_AGENT); } - public static String sendGet(String url, String ua) throws IOException, InterruptedException, URISyntaxException { + public static String sendGet(String url, String ua) throws IOException { - HttpRequest request = HttpRequest.newBuilder(new URI(url)).GET().setHeader("User-Agent", ua).build(); + var request = new Request.Builder().header("User-Agent", ua).url(url).build(); - return Constants.h2client.send(request, BodyHandlers.ofString()).body(); + return Constants.h2client.newCall(request).execute().body().string(); } } diff --git a/src/main/java/me/kavin/piped/utils/ResponseHelper.java b/src/main/java/me/kavin/piped/utils/ResponseHelper.java index 1eb3833..5f35950 100644 --- a/src/main/java/me/kavin/piped/utils/ResponseHelper.java +++ b/src/main/java/me/kavin/piped/utils/ResponseHelper.java @@ -1,41 +1,28 @@ package me.kavin.piped.utils; -import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.http.HttpRequest; -import java.net.http.HttpRequest.BodyPublishers; -import java.net.http.HttpRequest.Builder; -import java.net.http.HttpResponse; -import java.net.http.HttpResponse.BodyHandlers; -import java.nio.charset.StandardCharsets; -import java.security.NoSuchAlgorithmException; -import java.security.spec.InvalidKeySpecException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.Root; - +import com.fasterxml.jackson.core.JsonProcessingException; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.Scheduler; +import com.rometools.rome.feed.synd.*; +import com.rometools.rome.io.FeedException; +import com.rometools.rome.io.SyndFeedOutput; +import it.unimi.dsi.fastutil.objects.ObjectArrayList; +import me.kavin.piped.consts.Constants; +import me.kavin.piped.ipfs.IPFS; +import me.kavin.piped.utils.obj.*; +import me.kavin.piped.utils.obj.db.PubSub; +import me.kavin.piped.utils.obj.db.User; +import me.kavin.piped.utils.obj.db.Video; +import me.kavin.piped.utils.obj.search.SearchChannel; +import me.kavin.piped.utils.obj.search.SearchPlaylist; +import me.kavin.piped.utils.resp.*; +import okhttp3.FormBody; +import okhttp3.Request; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.hibernate.Session; -import org.json.JSONObject; import org.schabi.newpipe.extractor.InfoItem; import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage; import org.schabi.newpipe.extractor.Page; @@ -44,7 +31,6 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem; import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.kiosk.KioskExtractor; import org.schabi.newpipe.extractor.kiosk.KioskInfo; import org.schabi.newpipe.extractor.kiosk.KioskList; @@ -58,49 +44,19 @@ import org.schabi.newpipe.extractor.stream.StreamType; import org.springframework.security.crypto.argon2.Argon2PasswordEncoder; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.LoadingCache; -import com.rometools.rome.feed.synd.SyndEntry; -import com.rometools.rome.feed.synd.SyndEntryImpl; -import com.rometools.rome.feed.synd.SyndFeed; -import com.rometools.rome.feed.synd.SyndFeedImpl; -import com.rometools.rome.feed.synd.SyndPerson; -import com.rometools.rome.feed.synd.SyndPersonImpl; -import com.rometools.rome.io.FeedException; -import com.rometools.rome.io.SyndFeedOutput; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Root; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; -import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.objects.ObjectArrayList; -import me.kavin.piped.consts.Constants; -import me.kavin.piped.ipfs.IPFS; -import me.kavin.piped.utils.obj.Channel; -import me.kavin.piped.utils.obj.ChapterSegment; -import me.kavin.piped.utils.obj.Comment; -import me.kavin.piped.utils.obj.CommentsPage; -import me.kavin.piped.utils.obj.FeedItem; -import me.kavin.piped.utils.obj.PipedStream; -import me.kavin.piped.utils.obj.Playlist; -import me.kavin.piped.utils.obj.SearchResults; -import me.kavin.piped.utils.obj.StreamItem; -import me.kavin.piped.utils.obj.Streams; -import me.kavin.piped.utils.obj.StreamsPage; -import me.kavin.piped.utils.obj.SubscriptionChannel; -import me.kavin.piped.utils.obj.Subtitle; -import me.kavin.piped.utils.obj.db.PubSub; -import me.kavin.piped.utils.obj.db.User; -import me.kavin.piped.utils.obj.db.Video; -import me.kavin.piped.utils.obj.search.SearchChannel; -import me.kavin.piped.utils.obj.search.SearchPlaylist; -import me.kavin.piped.utils.resp.AcceptedResponse; -import me.kavin.piped.utils.resp.AlreadyRegisteredResponse; -import me.kavin.piped.utils.resp.AuthenticationFailureResponse; -import me.kavin.piped.utils.resp.CompromisedPasswordResponse; -import me.kavin.piped.utils.resp.DisabledRegistrationResponse; -import me.kavin.piped.utils.resp.IncorrectCredentialsResponse; -import me.kavin.piped.utils.resp.InvalidRequestResponse; -import me.kavin.piped.utils.resp.LoginResponse; -import me.kavin.piped.utils.resp.SubscribeStatusResponse; +import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; +import static me.kavin.piped.utils.URLUtils.rewriteURL; +import static me.kavin.piped.utils.URLUtils.substringYouTube; public class ResponseHelper { @@ -122,7 +78,7 @@ public class ResponseHelper { CompletableFuture futureLbryId = CompletableFuture.supplyAsync(() -> { try { - return getLBRYId(videoId); + return LbryHelper.getLBRYId(videoId); } catch (Exception e) { ExceptionHandler.handle(e); } @@ -135,7 +91,7 @@ public class ResponseHelper { lbryId = futureLbryId.completeOnTimeout(null, 2, TimeUnit.SECONDS).get(); - return getLBRYStreamURL(lbryId); + return LbryHelper.getLBRYStreamURL(lbryId); } catch (Exception e) { ExceptionHandler.handle(e); } @@ -194,15 +150,10 @@ public class ResponseHelper { info.getRelatedItems().forEach(o -> relatedStreams.add(collectRelatedStream(o))); - List segments = new ObjectArrayList<>(); - - info.getStreamSegments().forEach(segment -> segments - .add(new ChapterSegment(segment.getTitle(), segment.getPreviewUrl(), segment.getStartTimeSeconds()))); - long time = info.getUploadDate() != null ? info.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : System.currentTimeMillis(); - if (info.getUploadDate() != null && System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(10)) + if (info.getUploadDate() != null && System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) updateViews(info.getId(), info.getViewCount(), time, false); final Streams streams = new Streams(info.getName(), info.getDescription().getContent(), @@ -243,7 +194,7 @@ public class ResponseHelper { long time = item.getUploadDate() != null ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : System.currentTimeMillis(); - if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(10)) + if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) updateViews(item.getUrl().substring("https://www.youtube.com/watch?v=".length()), item.getViewCount(), time, true); } @@ -269,7 +220,7 @@ public class ResponseHelper { } public static final byte[] channelPageResponse(String channelId, String prevpageStr) - throws IOException, ExtractionException, InterruptedException { + throws IOException, ExtractionException { Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class); @@ -293,7 +244,7 @@ public class ResponseHelper { } public static final byte[] trendingResponse(String region) - throws ParsingException, ExtractionException, IOException { + throws ExtractionException, IOException { if (region == null) return Constants.mapper.writeValueAsBytes(new InvalidRequestResponse()); @@ -312,7 +263,7 @@ public class ResponseHelper { } public static final byte[] playlistResponse(String playlistId) - throws IOException, ExtractionException, InterruptedException { + throws IOException, ExtractionException { final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId); @@ -337,7 +288,7 @@ public class ResponseHelper { } public static final byte[] playlistPageResponse(String playlistId, String prevpageStr) - throws IOException, ExtractionException, InterruptedException { + throws IOException, ExtractionException { Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class); @@ -361,7 +312,7 @@ public class ResponseHelper { } public static final byte[] playlistRSSResponse(String playlistId) - throws IOException, ExtractionException, InterruptedException, FeedException { + throws IOException, ExtractionException, FeedException { final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId); @@ -392,22 +343,24 @@ public class ResponseHelper { } public static final byte[] suggestionsResponse(String query) - throws JsonProcessingException, IOException, ExtractionException { + throws IOException, ExtractionException { return Constants.mapper.writeValueAsBytes(YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query)); } public static final byte[] opensearchSuggestionsResponse(String query) - throws JsonProcessingException, IOException, ExtractionException { + throws IOException, ExtractionException { - return Constants.mapper.writeValueAsBytes( - Arrays.asList(query, YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query))); + return Constants.mapper.writeValueAsBytes(Arrays.asList( + query, + YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query) + )); } public static final byte[] searchResponse(String q, String filter) - throws IOException, ExtractionException, InterruptedException { + throws IOException, ExtractionException { final SearchInfo info = SearchInfo.getInfo(YOUTUBE_SERVICE, YOUTUBE_SERVICE.getSearchQHFactory().fromQuery(q, Collections.singletonList(filter), null)); @@ -443,7 +396,7 @@ public class ResponseHelper { } public static final byte[] searchPageResponse(String q, String filter, String prevpageStr) - throws IOException, ExtractionException, InterruptedException { + throws IOException, ExtractionException { Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class); @@ -552,8 +505,8 @@ public class ResponseHelper { private static final Argon2PasswordEncoder argon2PasswordEncoder = new Argon2PasswordEncoder(); - public static final byte[] registerResponse(String user, String pass) throws IOException, NoSuchAlgorithmException, - InvalidKeySpecException, InterruptedException, URISyntaxException { + public static final byte[] registerResponse(String user, String pass) throws IOException, + InterruptedException, URISyntaxException { if (Constants.DISABLE_REGISTRATION) return Constants.mapper.writeValueAsBytes(new DisabledRegistrationResponse()); @@ -602,7 +555,7 @@ public class ResponseHelper { private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder(); public static final byte[] loginResponse(String user, String pass) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + throws IOException { if (user == null || pass == null) return Constants.mapper.writeValueAsBytes(new InvalidRequestResponse()); @@ -637,7 +590,7 @@ public class ResponseHelper { } public static final byte[] subscribeResponse(String session, String channelId) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + throws IOException { Session s = DatabaseSessionFactory.createSession(); @@ -684,7 +637,7 @@ public class ResponseHelper { long time = item.getUploadDate() != null ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : System.currentTimeMillis(); - if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(10)) + if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) handleNewVideo(item.getUrl(), time, channel, sess); } } @@ -703,7 +656,7 @@ public class ResponseHelper { } public static final byte[] unsubscribeResponse(String session, String channelId) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + throws IOException { Session s = DatabaseSessionFactory.createSession(); @@ -725,7 +678,7 @@ public class ResponseHelper { } public static final byte[] isSubscribedResponse(String session, String channelId) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + throws IOException { Session s = DatabaseSessionFactory.createSession(); @@ -747,7 +700,7 @@ public class ResponseHelper { } public static final byte[] feedResponse(String session) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + throws IOException { Session s = DatabaseSessionFactory.createSession(); @@ -774,7 +727,7 @@ public class ResponseHelper { }); - Collections.sort(feedItems, (a, b) -> (int) (b.uploaded - a.uploaded)); + feedItems.sort((a, b) -> (int) (b.uploaded - a.uploaded)); s.close(); @@ -788,7 +741,7 @@ public class ResponseHelper { } public static final byte[] feedResponseRSS(String session) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException, FeedException { + throws IOException, FeedException { Session s = DatabaseSessionFactory.createSession(); @@ -811,8 +764,7 @@ public class ResponseHelper { .setParameter("user", user.getId()).addEntity("Video", Video.class) .addEntity("Channel", me.kavin.piped.utils.obj.db.Channel.class).getResultList(); - Collections.sort(queryResults, - (a, b) -> (int) (((Video) b[0]).getUploaded() - ((Video) a[0]).getUploaded())); + queryResults.sort((a, b) -> (int) (((Video) b[0]).getUploaded() - ((Video) a[0]).getUploaded())); final List entries = new ObjectArrayList<>(); @@ -850,7 +802,7 @@ public class ResponseHelper { } public static final byte[] importResponse(String session, String[] channelIds, boolean override) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + throws IOException { Session s = DatabaseSessionFactory.createSession(); @@ -884,7 +836,7 @@ public class ResponseHelper { me.kavin.piped.utils.obj.db.Channel channel = DatabaseHelper.getChannelFromId(sess, channelId); if (channel == null) { - ChannelInfo info = null; + ChannelInfo info; try { info = ChannelInfo.getInfo("https://youtube.com/channel/" + channelId); @@ -910,7 +862,7 @@ public class ResponseHelper { long time = item.getUploadDate() != null ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : System.currentTimeMillis(); - if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(10)) + if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) handleNewVideo(item.getUrl(), time, channel, sess); } @@ -939,7 +891,7 @@ public class ResponseHelper { } public static final byte[] subscriptionsResponse(String session) - throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { + throws IOException { Session s = DatabaseSessionFactory.createSession(); @@ -954,12 +906,10 @@ public class ResponseHelper { List channels = DatabaseHelper.getChannelFromIds(s, user.getSubscribed()); - channels.forEach(channel -> { - subscriptionItems.add(new SubscriptionChannel("/channel/" + channel.getUploaderId(), - channel.getUploader(), rewriteURL(channel.getUploaderAvatar()), channel.isVerified())); - }); + channels.forEach(channel -> subscriptionItems.add(new SubscriptionChannel("/channel/" + channel.getUploaderId(), + channel.getUploader(), rewriteURL(channel.getUploaderAvatar()), channel.isVerified()))); - Collections.sort(subscriptionItems, (a, b) -> (a.name.compareTo(b.name))); + subscriptionItems.sort(Comparator.comparing(o -> o.name)); } s.close(); @@ -977,37 +927,11 @@ public class ResponseHelper { Session s = DatabaseSessionFactory.createSession(); - long registered = ((Long) s.createQuery("select count(*) from User").uniqueResult()).longValue(); + long registered = (Long) s.createQuery("select count(*) from User").uniqueResult(); s.close(); - return String.format("https://img.shields.io/badge/Registered%%20Users-%s-blue", String.valueOf(registered)); - } - - private static final String getLBRYId(String videoId) throws IOException, InterruptedException { - return new JSONObject(Constants.h2client.send(HttpRequest - .newBuilder(URI.create("https://api.lbry.com/yt/resolve?video_ids=" + URLUtils.silentEncode(videoId))) - .setHeader("User-Agent", Constants.USER_AGENT).build(), BodyHandlers.ofString()).body()) - .getJSONObject("data").getJSONObject("videos").optString(videoId); - } - - private static final String getLBRYStreamURL(String lbryId) - throws IOException, InterruptedException, ExecutionException { - - if (lbryId != null && !lbryId.isEmpty()) - return new JSONObject( - Constants.h2client.send( - HttpRequest.newBuilder(URI.create("https://api.lbry.tv/api/v1/proxy?m=get")) - .POST(BodyPublishers.ofString( - String.valueOf(new JSONObject().put("id", System.currentTimeMillis()) - .put("jsonrpc", "2.0").put("method", "get").put("params", - new JSONObject().put("uri", "lbry://" + lbryId) - .put("save_file", true))))) - .build(), - BodyHandlers.ofString()).body()).getJSONObject("result").getString("streaming_url"); - - return null; - + return String.format("https://img.shields.io/badge/Registered%%20Users-%s-blue", registered); } public static void handleNewVideo(String url, long time, me.kavin.piped.utils.obj.db.Channel channel, Session s) { @@ -1018,8 +942,7 @@ public class ResponseHelper { } } - private static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.utils.obj.db.Channel channel, - Session s) { + private static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.utils.obj.db.Channel channel, Session s) { if (channel == null) channel = DatabaseHelper.getChannelFromId(s, @@ -1031,7 +954,7 @@ public class ResponseHelper { Video video = null; if (channel != null && (video = DatabaseHelper.getVideoFromId(s, info.getId())) == null - && (System.currentTimeMillis() - infoTime) < TimeUnit.DAYS.toMillis(10)) { + && (System.currentTimeMillis() - infoTime) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) { video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(), Math.max(infoTime, time), info.getThumbnailUrl(), channel); @@ -1084,29 +1007,22 @@ public class ResponseHelper { String callback = Constants.PUBLIC_URL + "/webhooks/pubsub"; String topic = "https://www.youtube.com/xml/feeds/videos.xml?channel_id=" + channelId; - Builder builder = HttpRequest.newBuilder(URI.create("https://pubsubhubbub.appspot.com/subscribe")); + var builder = new Request.Builder() + .url("https://pubsubhubbub.appspot.com/subscribe"); - Map formParams = new Object2ObjectOpenHashMap<>(); - StringBuilder formBody = new StringBuilder(); + var formBuilder = new FormBody.Builder(); - builder.header("content-type", "application/x-www-form-urlencoded"); + formBuilder.add("hub.callback", callback); + formBuilder.add("hub.topic", topic); + formBuilder.add("hub.verify", "async"); + formBuilder.add("hub.mode", "subscribe"); + formBuilder.add("hub.lease_seconds", "432000"); - formParams.put("hub.callback", callback); - formParams.put("hub.topic", topic); - formParams.put("hub.verify", "async"); - formParams.put("hub.mode", "subscribe"); - formParams.put("hub.lease_seconds", "432000"); + var resp = Constants.h2client + .newCall(builder.post(formBuilder.build()) + .build()).execute(); - formParams.forEach((name, value) -> { - formBody.append(name + "=" + URLUtils.silentEncode(value) + "&"); - }); - - builder.method("POST", - BodyPublishers.ofString(String.valueOf(formBody.substring(0, formBody.length() - 1)))); - - HttpResponse resp = Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()); - - if (resp.statusCode() == 202) { + if (resp.code() == 202) { if (pubsub == null) pubsub = new PubSub(channelId, System.currentTimeMillis()); else @@ -1118,16 +1034,11 @@ public class ResponseHelper { s.getTransaction().begin(); s.getTransaction().commit(); } else - System.out.println( - "Failed to subscribe: " + resp.statusCode() + "\n" + IOUtils.toString(resp.body(), "UTF-8")); + System.out.println("Failed to subscribe: " + resp.code() + "\n" + resp.body().string()); } } - private static final String substringYouTube(String s) { - return StringUtils.isEmpty(s) ? null : StringUtils.substringAfter(s, "youtube.com"); - } - private static StreamItem collectRelatedStream(Object o) { StreamInfoItem item = (StreamInfoItem) o; @@ -1137,35 +1048,4 @@ public class ResponseHelper { rewriteURL(item.getUploaderAvatarUrl()), item.getTextualUploadDate(), item.getDuration(), item.getViewCount(), item.isUploaderVerified()); } - - private static String rewriteURL(final String old) { - - if (old == null || old.isEmpty()) - return null; - - URL url = null; - - try { - url = new URL(old); - } catch (MalformedURLException e) { - ExceptionHandler.handle(e); - } - - final String host = url.getHost(); - - String query = url.getQuery(); - - boolean hasQuery = query != null; - - String path = url.getPath(); - - if (path.contains("=")) { - path = StringUtils.substringBefore(path, "=") + "=" - + StringUtils.substringAfter(path, "=").replace("-rj", "-rw"); - } - - return Constants.PROXY_PART + path + (hasQuery ? "?" + query + "&host=" : "?host=") - + URLUtils.silentEncode(host); - - } } diff --git a/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java b/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java index b971266..cfe003a 100644 --- a/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java +++ b/src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java @@ -1,46 +1,36 @@ package me.kavin.piped.utils; +import com.grack.nanojson.*; +import me.kavin.piped.consts.Constants; +import me.kavin.piped.utils.resp.InvalidRequestResponse; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; -import java.net.URI; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse.BodyHandlers; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import org.apache.commons.lang3.StringUtils; - -import com.grack.nanojson.JsonArray; -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; -import com.grack.nanojson.JsonWriter; - -import me.kavin.piped.consts.Constants; -import me.kavin.piped.utils.resp.InvalidRequestResponse; - public class SponsorBlockUtils { - public static final String getSponsors(String id, String categories) - throws IOException, InterruptedException, NoSuchAlgorithmException, JsonParserException { + public static String getSponsors(String id, String categories) + throws IOException, NoSuchAlgorithmException, JsonParserException { if (StringUtils.isEmpty(categories)) return Constants.mapper.writeValueAsString(new InvalidRequestResponse()); String hash = toSha256(id); - URI uri = URI.create("https://sponsor.ajay.app/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4)) - + "?categories=" + URLUtils.silentEncode(categories)); - JsonArray jArray = JsonParser.array().from( - Constants.h2client.send(HttpRequest.newBuilder(uri).build(), BodyHandlers.ofInputStream()).body()); + RequestUtils.sendGet("https://sponsor.ajay.app/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4)) + + "?categories=" + URLUtils.silentEncode(categories)) + ); jArray.removeIf(jObject -> !((JsonObject) jObject).getString("videoID").equalsIgnoreCase(id)); return JsonWriter.string(jArray.getObject(0)); } - private static final String toSha256(final String videoId) throws NoSuchAlgorithmException { + private static String toSha256(final String videoId) throws NoSuchAlgorithmException { final MessageDigest digest = MessageDigest.getInstance("SHA-256"); final byte[] bytes = digest.digest(videoId.getBytes(StandardCharsets.UTF_8)); final StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index 0ef9717..c700d99 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -1,13 +1,19 @@ package me.kavin.piped.utils; +import me.kavin.piped.consts.Constants; +import org.apache.commons.lang3.StringUtils; + +import java.net.MalformedURLException; +import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; public class URLUtils { public static String silentEncode(String s) { try { - return URLEncoder.encode(s, "UTF-8"); + return URLEncoder.encode(s, StandardCharsets.UTF_8); } catch (Exception e) { // ignored } @@ -16,10 +22,42 @@ public class URLUtils { public static String silentDecode(String s) { try { - return URLDecoder.decode(s, "UTF-8"); + return URLDecoder.decode(s, StandardCharsets.UTF_8); } catch (Exception e) { // ignored } return s; } + + public static String substringYouTube(String s) { + return StringUtils.isEmpty(s) ? null : StringUtils.substringAfter(s, "youtube.com"); + } + + public static String rewriteURL(final String old) { + + if (StringUtils.isEmpty(old)) return null; + + URL url = null; + try { + url = new URL(old); + } catch (MalformedURLException e) { + ExceptionHandler.handle(e); + } + assert url != null; + + final String host = url.getHost(); + + String query = url.getQuery(); + + boolean hasQuery = query != null; + + String path = url.getPath(); + + if (path.contains("=")) { + path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw"); + } + + return Constants.PROXY_PART + path + (hasQuery ? "?" + query + "&host=" : "?host=") + silentEncode(host); + + } } diff --git a/src/main/java/me/kavin/piped/utils/obj/db/Channel.java b/src/main/java/me/kavin/piped/utils/obj/db/Channel.java index e204e0b..a28a74b 100644 --- a/src/main/java/me/kavin/piped/utils/obj/db/Channel.java +++ b/src/main/java/me/kavin/piped/utils/obj/db/Channel.java @@ -7,7 +7,7 @@ import javax.persistence.Index; import javax.persistence.Table; @Entity -@Table(name = "channels", indexes = { @Index(columnList = "uploader_id", name = "channels_uploader_id_idx") }) +@Table(name = "channels", indexes = {@Index(columnList = "uploader_id", name = "channels_uploader_id_idx")}) public class Channel { @Id diff --git a/src/main/java/me/kavin/piped/utils/obj/db/Video.java b/src/main/java/me/kavin/piped/utils/obj/db/Video.java index e6a9f9e..5518ebc 100644 --- a/src/main/java/me/kavin/piped/utils/obj/db/Video.java +++ b/src/main/java/me/kavin/piped/utils/obj/db/Video.java @@ -1,13 +1,6 @@ package me.kavin.piped.utils.obj.db; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.Id; -import javax.persistence.Index; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; -import javax.persistence.Table; +import javax.persistence.*; @Entity @Table(name = "videos", indexes = { @Index(columnList = "id", name = "videos_id_idx"),