Refactor Utility methods and migrate to OkHttp (#179)

build.gradle

@@ -38,6 +38,9 @@ dependencies {
     implementation 'com.zaxxer:HikariCP:5.0.1'
     implementation 'org.springframework.security:spring-security-crypto:5.6.1'
     implementation 'commons-logging:commons-logging:1.2'
+    implementation(platform("com.squareup.okhttp3:okhttp-bom:4.9.3"))
+    implementation("com.squareup.okhttp3:okhttp")
+    implementation("com.squareup.okhttp3:okhttp-brotli")
 }
 
 shadowJar {

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://downloads.gradle.org/distributions/gradle-7.2-bin.zip
+distributionUrl=https\://downloads.gradle.org/distributions/gradle-7.3.3-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/java/me/kavin/piped/ServerLauncher.java

@@ -1,28 +1,8 @@
 package me.kavin.piped;
 
-import static io.activej.config.converter.ConfigConverters.ofInetSocketAddress;
-import static io.activej.http.HttpHeaders.AUTHORIZATION;
-import static io.activej.http.HttpHeaders.CACHE_CONTROL;
-import static io.activej.http.HttpHeaders.CONTENT_TYPE;
-import static io.activej.http.HttpHeaders.LINK;
-import static io.activej.http.HttpHeaders.LOCATION;
-import static io.activej.http.HttpMethod.GET;
-import static io.activej.http.HttpMethod.POST;
-
-import java.io.ByteArrayInputStream;
-import java.net.InetSocketAddress;
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.Executor;
-
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.hibernate.Session;
-import org.jetbrains.annotations.NotNull;
-import org.xml.sax.InputSource;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.rometools.rome.feed.synd.SyndFeed;
 import com.rometools.rome.io.SyndFeedInput;
-
 import io.activej.config.Config;
 import io.activej.http.AsyncServlet;
 import io.activej.http.HttpMethod;
@@ -33,15 +13,25 @@ import io.activej.inject.module.AbstractModule;
 import io.activej.inject.module.Module;
 import io.activej.launchers.http.MultithreadedHttpServerLauncher;
 import me.kavin.piped.consts.Constants;
-import me.kavin.piped.utils.CustomServletDecorator;
+import me.kavin.piped.utils.*;
-import me.kavin.piped.utils.DatabaseSessionFactory;
-import me.kavin.piped.utils.ExceptionHandler;
-import me.kavin.piped.utils.Multithreading;
-import me.kavin.piped.utils.ResponseHelper;
-import me.kavin.piped.utils.SponsorBlockUtils;
 import me.kavin.piped.utils.resp.ErrorResponse;
 import me.kavin.piped.utils.resp.LoginRequest;
 import me.kavin.piped.utils.resp.SubscriptionUpdateRequest;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.hibernate.Session;
+import org.jetbrains.annotations.NotNull;
+import org.xml.sax.InputSource;
+
+import java.io.ByteArrayInputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+import java.util.concurrent.Executor;
+
+import static io.activej.config.converter.ConfigConverters.ofInetSocketAddress;
+import static io.activej.http.HttpHeaders.*;
+import static io.activej.http.HttpMethod.GET;
+import static io.activej.http.HttpMethod.POST;
 
 public class ServerLauncher extends MultithreadedHttpServerLauncher {
 
@@ -55,9 +45,8 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
 
         RoutingServlet router = RoutingServlet.create()
                 .map(HttpMethod.OPTIONS, "/*", request -> HttpResponse.ofCode(200))
-                .map(GET, "/webhooks/pubsub", request -> {
+                .map(GET, "/webhooks/pubsub", request -> HttpResponse.ok200().withPlainText(Objects.requireNonNull(request.getQueryParameter("hub.challenge"))))
-                    return HttpResponse.ok200().withPlainText(request.getQueryParameter("hub.challenge"));
+                .map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> {
-                }).map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> {
                     try {
 
                         SyndFeed feed = new SyndFeedInput().build(
@@ -320,7 +309,7 @@ public class ServerLauncher extends MultithreadedHttpServerLauncher {
     }
 
     private @NotNull HttpResponse getRawResponse(int code, byte[] body, String contentType, String cache,
-            boolean prefetchProxy) {
+                                                 boolean prefetchProxy) {
         HttpResponse response = HttpResponse.ofCode(code).withBody(body).withHeader(CONTENT_TYPE, contentType)
                 .withHeader(CACHE_CONTROL, cache);
         if (prefetchProxy)

src/main/java/me/kavin/piped/consts/Constants.java

@@ -3,6 +3,8 @@ package me.kavin.piped.consts;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
 import me.kavin.piped.utils.PageMixin;
+import okhttp3.OkHttpClient;
+import okhttp3.brotli.BrotliInterceptor;
 import org.apache.commons.lang3.StringUtils;
 import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.Page;
@@ -11,10 +13,6 @@ import org.schabi.newpipe.extractor.StreamingService;
 import java.io.FileReader;
 import java.net.InetSocketAddress;
 import java.net.ProxySelector;
-import java.net.http.HttpClient;
-import java.net.http.HttpClient.Builder;
-import java.net.http.HttpClient.Redirect;
-import java.net.http.HttpClient.Version;
 import java.util.Properties;
 
 public class Constants {
@@ -36,8 +34,8 @@ public class Constants {
 
     public static final String FRONTEND_URL;
 
-    public static final HttpClient h2client;
+    public static final OkHttpClient h2client;
-    public static final HttpClient h2_no_redir_client;
+    public static final OkHttpClient h2_no_redir_client;
 
     public static final boolean COMPROMISED_PASSWORD_CHECK;
 
@@ -76,8 +74,12 @@ public class Constants {
                 if (key.startsWith("hibernate"))
                     hibernateProperties.put(key, value);
             });
-            Builder builder = HttpClient.newBuilder().followRedirects(Redirect.NORMAL).version(Version.HTTP_1_1);
+            OkHttpClient.Builder builder = new OkHttpClient.Builder()
-            Builder builder_noredir = HttpClient.newBuilder().followRedirects(Redirect.NEVER).version(Version.HTTP_1_1);
+                    .followRedirects(true)
+                    .addInterceptor(BrotliInterceptor.INSTANCE);
+            OkHttpClient.Builder builder_noredir = new OkHttpClient.Builder()
+                    .followRedirects(false)
+                    .addInterceptor(BrotliInterceptor.INSTANCE);
             if (HTTP_PROXY != null && HTTP_PROXY.contains(":")) {
                 String host = StringUtils.substringBefore(HTTP_PROXY, ":");
                 String port = StringUtils.substringAfter(HTTP_PROXY, ":");

src/main/java/me/kavin/piped/utils/CaptchaSolver.java

@@ -1,21 +1,18 @@
 package me.kavin.piped.utils;
 
-import java.io.IOException;
-import java.net.URI;
-import java.net.http.HttpRequest;
-import java.net.http.HttpRequest.BodyPublishers;
-import java.net.http.HttpRequest.Builder;
-import java.net.http.HttpResponse.BodyHandlers;
-import java.util.Map;
-
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import com.grack.nanojson.JsonWriter;
-
 import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
 import me.kavin.piped.consts.Constants;
 import me.kavin.piped.utils.obj.SolvedCaptcha;
+import okhttp3.MediaType;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+
+import java.io.IOException;
+import java.util.Map;
 
 public class CaptchaSolver {
 
@@ -29,9 +26,8 @@ public class CaptchaSolver {
     }
 
     private static int createTask(String url, String sitekey, String data_s)
-            throws JsonParserException, IOException, InterruptedException {
+            throws JsonParserException, IOException {
 
-        Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "createTask"));
         JsonObject jObject = new JsonObject();
         jObject.put("clientKey", Constants.CAPTCHA_API_KEY);
         {
@@ -43,17 +39,16 @@ public class CaptchaSolver {
             jObject.put("task", task);
         }
 
-        builder.method("POST", BodyPublishers.ofString(JsonWriter.string(jObject)));
+        var builder = new Request.Builder().url(Constants.CAPTCHA_BASE_URL + "createTask")
-
+                .post(RequestBody.create(JsonWriter.string(jObject), MediaType.get("application/json")));
-        builder.header("Content-Type", "application/json");
 
         JsonObject taskObj = JsonParser.object()
-                .from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body());
+                .from(Constants.h2client.newCall(builder.build()).execute().body().byteStream());
 
         return taskObj.getInt("taskId");
     }
 
-    private static final SolvedCaptcha waitForSolve(int taskId)
+    private static SolvedCaptcha waitForSolve(int taskId)
             throws JsonParserException, IOException, InterruptedException {
 
         String body = JsonWriter.string(
@@ -61,15 +56,15 @@ public class CaptchaSolver {
 
         SolvedCaptcha solved = null;
 
-        outer: while (true) {
+        while (true) {
-            Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "getTaskResult"));
+            var builder = new Request.Builder()
-
+                    .url(Constants.CAPTCHA_BASE_URL + "getTaskResult")
-            builder.method("POST", BodyPublishers.ofString(body));
+                    .post(RequestBody.create(body, MediaType.get("application/json")));
 
             builder.header("Content-Type", "application/json");
 
             JsonObject captchaObj = JsonParser.object()
-                    .from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body());
+                    .from(Constants.h2client.newCall(builder.build()).execute().body().byteStream());
 
             if (captchaObj.getInt("errorId") != 0)
                 break;
@@ -87,7 +82,7 @@ public class CaptchaSolver {
                     });
 
                     solved = new SolvedCaptcha(cookies, captchaResp);
-                    break outer;
+                    break;
                 }
             }
 

src/main/java/me/kavin/piped/utils/CustomServletDecorator.java

@@ -1,18 +1,12 @@
 package me.kavin.piped.utils;
 
+import io.activej.http.*;
+import io.activej.promise.Promisable;
+import org.jetbrains.annotations.NotNull;
+
 import static io.activej.http.HttpHeaders.ACCESS_CONTROL_ALLOW_HEADERS;
 import static io.activej.http.HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN;
 
-import org.jetbrains.annotations.NotNull;
-
-import io.activej.http.AsyncServlet;
-import io.activej.http.HttpHeader;
-import io.activej.http.HttpHeaderValue;
-import io.activej.http.HttpHeaders;
-import io.activej.http.HttpRequest;
-import io.activej.http.HttpResponse;
-import io.activej.promise.Promisable;
-
 public class CustomServletDecorator implements AsyncServlet {
 
     private static final HttpHeader HEADER = HttpHeaders.of("Server-Timing");

src/main/java/me/kavin/piped/utils/DownloaderImpl.java

@@ -1,18 +1,13 @@
 package me.kavin.piped.utils;
 
-import java.io.IOException;
+import com.github.benmanes.caffeine.cache.Caffeine;
-import java.net.HttpCookie;
+import com.github.benmanes.caffeine.cache.LoadingCache;
-import java.net.URI;
-import java.net.http.HttpRequest;
-import java.net.http.HttpRequest.BodyPublisher;
-import java.net.http.HttpRequest.BodyPublishers;
-import java.net.http.HttpRequest.Builder;
-import java.net.http.HttpResponse;
-import java.net.http.HttpResponse.BodyHandlers;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
 import com.github.benmanes.caffeine.cache.Scheduler;
+import com.grack.nanojson.JsonParserException;
+import me.kavin.piped.consts.Constants;
+import me.kavin.piped.utils.obj.SolvedCaptcha;
+import okhttp3.FormBody;
+import okhttp3.RequestBody;
 import org.apache.commons.lang3.StringUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jsoup.Jsoup;
@@ -22,13 +17,9 @@ import org.schabi.newpipe.extractor.downloader.Request;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 
-import com.github.benmanes.caffeine.cache.Caffeine;
+import java.io.IOException;
-import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.net.HttpCookie;
-import com.grack.nanojson.JsonParserException;
+import java.util.concurrent.TimeUnit;
-
-import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
-import me.kavin.piped.consts.Constants;
-import me.kavin.piped.utils.obj.SolvedCaptcha;
 
 public class DownloaderImpl extends Downloader {
 
@@ -52,30 +43,24 @@ public class DownloaderImpl extends Downloader {
     public Response executeRequest(Request request) throws IOException, ReCaptchaException {
 
         // TODO: HTTP/3 aka QUIC
-        Builder builder = HttpRequest.newBuilder(URI.create(request.url()));
+        var bytes = request.dataToSend();
+        RequestBody body = null;
+        if (bytes != null)
+            body = RequestBody.create(bytes);
 
-        byte[] data = request.dataToSend();
+        var builder = new okhttp3.Request.Builder()
-        BodyPublisher publisher = data == null ? HttpRequest.BodyPublishers.noBody()
+                .url(request.url())
-                : HttpRequest.BodyPublishers.ofByteArray(data);
+                .method(request.httpMethod(), body)
-
+                .header("User-Agent", Constants.USER_AGENT);
-        builder.method(request.httpMethod(), publisher);
-
-        builder.setHeader("User-Agent", Constants.USER_AGENT);
 
         if (saved_cookie != null && !saved_cookie.hasExpired())
-            builder.setHeader("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue());
+            builder.header("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue());
 
         request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value)));
 
-        HttpResponse<String> response = null;
+        var response = Constants.h2client.newCall(builder.build()).execute();
 
-        try {
+        if (response.code() == 429) {
-            response = Constants.h2client.send(builder.build(), BodyHandlers.ofString());
-        } catch (InterruptedException e) {
-            // ignored
-        }
-
-        if (response.statusCode() == 429) {
 
             synchronized (cookie_lock) {
 
@@ -83,25 +68,25 @@ public class DownloaderImpl extends Downloader {
                         || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
                     saved_cookie = null;
 
-                String redir_url = String.valueOf(response.request().uri());
+                String redir_url = String.valueOf(response.request().url());
 
                 if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
 
-                    Map<String, String> formParams = new Object2ObjectOpenHashMap<>();
+                    var formBuilder = new FormBody.Builder();
                     String sitekey = null, data_s = null;
 
-                    for (Element el : Jsoup.parse(response.body()).selectFirst("form").children()) {
+                    for (Element el : Jsoup.parse(response.body().string()).selectFirst("form").children()) {
                         String name;
                         if (!(name = el.tagName()).equals("script")) {
                             if (name.equals("input"))
-                                formParams.put(el.attr("name"), el.attr("value"));
+                                formBuilder.add(el.attr("name"), el.attr("value"));
                             else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
                                 sitekey = el.attr("data-sitekey");
                                 data_s = el.attr("data-s");
                             }
                         }
                     }
-                    if (sitekey == null || data_s == null)
+                    if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
                         throw new ReCaptchaException("Could not get recaptcha", redir_url);
 
                     SolvedCaptcha solved = null;
@@ -112,35 +97,19 @@ public class DownloaderImpl extends Downloader {
                         e.printStackTrace();
                     }
 
-                    formParams.put("g-recaptcha-response", solved.getRecaptchaResponse());
+                    formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
 
-                    Builder formBuilder = HttpRequest.newBuilder(URI.create("https://www.google.com/sorry/index"));
+                    var formReqBuilder = new okhttp3.Request.Builder()
+                            .url("https://www.google.com/sorry/index")
+                            .header("User-Agent", Constants.USER_AGENT)
+                            .post(formBuilder.build());
 
-                    formBuilder.setHeader("User-Agent", Constants.USER_AGENT);
+                    var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
 
-                    StringBuilder formBody = new StringBuilder();
+                    saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
-
+                                    .substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
-                    formParams.forEach((name, value) -> {
+                            .get(0);
-                        formBody.append(name + "=" + URLUtils.silentEncode(value) + "&");
+                    cookie_received = System.currentTimeMillis();
-                    });
-
-                    formBuilder.header("content-type", "application/x-www-form-urlencoded");
-
-                    formBuilder.method("POST",
-                            BodyPublishers.ofString(String.valueOf(formBody.substring(0, formBody.length() - 1))));
-
-                    try {
-                        HttpResponse<String> formResponse = Constants.h2_no_redir_client.send(formBuilder.build(),
-                                BodyHandlers.ofString());
-
-                        saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
-                                        .substringAfter(formResponse.headers().firstValue("Location").get(), "google_abuse=")))
-                                .get(0);
-                        cookie_received = System.currentTimeMillis();
-
-                    } catch (InterruptedException e) {
-                        e.printStackTrace();
-                    }
                 }
 
                 if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired.
@@ -149,7 +118,7 @@ public class DownloaderImpl extends Downloader {
 
         }
 
-        return new Response(response.statusCode(), "UNDEFINED", response.headers().map(), response.body(),
+        return new Response(response.code(), response.message(), response.headers().toMultimap(), response.body().string(),
-                String.valueOf(response.uri()));
+                String.valueOf(response.request().url()));
     }
 }

src/main/java/me/kavin/piped/utils/ExceptionHandler.java

@@ -1,12 +1,10 @@
 package me.kavin.piped.utils;
 
+import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
+
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
 
-import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
-import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
-import org.schabi.newpipe.extractor.exceptions.GeographicRestrictionException;
-
 public class ExceptionHandler {
 
     public static Exception handle(Exception e) {
@@ -18,8 +16,7 @@ public class ExceptionHandler {
         if (e.getCause() != null && (e instanceof ExecutionException || e instanceof CompletionException))
             e = (Exception) e.getCause();
 
-        if (!(e instanceof AgeRestrictedContentException || e instanceof ContentNotAvailableException
+        if (!(e instanceof ContentNotAvailableException)) {
-                || e instanceof GeographicRestrictionException)) {
             if (path != null)
                 System.err.println("An error occoured in the path: " + path);
             e.printStackTrace();

src/main/java/me/kavin/piped/utils/LbryHelper.java

@@ -0,0 +1,39 @@
+package me.kavin.piped.utils;
+
+import me.kavin.piped.consts.Constants;
+import okhttp3.MediaType;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import org.json.JSONObject;
+
+import java.io.IOException;
+
+public class LbryHelper {
+
+    public static String getLBRYId(String videoId) throws IOException, InterruptedException {
+        return new JSONObject(
+                RequestUtils.sendGet("https://api.lbry.com/yt/resolve?video_ids=" + URLUtils.silentEncode(videoId))
+        ).getJSONObject("data").getJSONObject("videos").optString(videoId, null);
+    }
+
+    public static String getLBRYStreamURL(String lbryId)
+            throws IOException {
+        var request = new Request.Builder()
+                .url("https://api.lbry.tv/api/v1/proxy?m=get")
+                .post(RequestBody.create(String.valueOf(
+                        new JSONObject().put("id", System.currentTimeMillis())
+                                .put("jsonrpc", "2.0")
+                                .put("method", "get")
+                                .put("params",
+                                        new JSONObject()
+                                                .put("uri", "lbry://" + lbryId)
+                                                .put("save_file", true)))
+                        , MediaType.get("application/json")))
+                .build();
+
+        return new JSONObject(
+                Constants.h2client.newCall(request).execute().body().string()
+        ).getJSONObject("result").getString("streaming_url");
+
+    }
+}

src/main/java/me/kavin/piped/utils/RequestUtils.java

@@ -1,23 +1,20 @@
 package me.kavin.piped.utils;
 
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse.BodyHandlers;
-
 import me.kavin.piped.consts.Constants;
+import okhttp3.Request;
+
+import java.io.IOException;
 
 public class RequestUtils {
 
-    public static String sendGet(String url) throws IOException, InterruptedException, URISyntaxException {
+    public static String sendGet(String url) throws IOException {
         return sendGet(url, Constants.USER_AGENT);
     }
 
-    public static String sendGet(String url, String ua) throws IOException, InterruptedException, URISyntaxException {
+    public static String sendGet(String url, String ua) throws IOException {
 
-        HttpRequest request = HttpRequest.newBuilder(new URI(url)).GET().setHeader("User-Agent", ua).build();
+        var request = new Request.Builder().header("User-Agent", ua).url(url).build();
 
-        return Constants.h2client.send(request, BodyHandlers.ofString()).body();
+        return Constants.h2client.newCall(request).execute().body().string();
     }
 }

src/main/java/me/kavin/piped/utils/ResponseHelper.java

@@ -1,41 +1,28 @@
 package me.kavin.piped.utils;
 
-import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
+import com.fasterxml.jackson.core.JsonProcessingException;
-
+import com.github.benmanes.caffeine.cache.Caffeine;
-import java.io.IOException;
+import com.github.benmanes.caffeine.cache.LoadingCache;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.http.HttpRequest;
-import java.net.http.HttpRequest.BodyPublishers;
-import java.net.http.HttpRequest.Builder;
-import java.net.http.HttpResponse;
-import java.net.http.HttpResponse.BodyHandlers;
-import java.nio.charset.StandardCharsets;
-import java.security.NoSuchAlgorithmException;
-import java.security.spec.InvalidKeySpecException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import javax.persistence.criteria.CriteriaBuilder;
-import javax.persistence.criteria.CriteriaQuery;
-import javax.persistence.criteria.Root;
-
 import com.github.benmanes.caffeine.cache.Scheduler;
+import com.rometools.rome.feed.synd.*;
+import com.rometools.rome.io.FeedException;
+import com.rometools.rome.io.SyndFeedOutput;
+import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import me.kavin.piped.consts.Constants;
+import me.kavin.piped.ipfs.IPFS;
+import me.kavin.piped.utils.obj.*;
+import me.kavin.piped.utils.obj.db.PubSub;
+import me.kavin.piped.utils.obj.db.User;
+import me.kavin.piped.utils.obj.db.Video;
+import me.kavin.piped.utils.obj.search.SearchChannel;
+import me.kavin.piped.utils.obj.search.SearchPlaylist;
+import me.kavin.piped.utils.resp.*;
+import okhttp3.FormBody;
+import okhttp3.Request;
 import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.hibernate.Session;
-import org.json.JSONObject;
 import org.schabi.newpipe.extractor.InfoItem;
 import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
 import org.schabi.newpipe.extractor.Page;
@@ -44,7 +31,6 @@ import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
 import org.schabi.newpipe.extractor.comments.CommentsInfo;
 import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
-import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
 import org.schabi.newpipe.extractor.kiosk.KioskInfo;
 import org.schabi.newpipe.extractor.kiosk.KioskList;
@@ -58,49 +44,19 @@ import org.schabi.newpipe.extractor.stream.StreamType;
 import org.springframework.security.crypto.argon2.Argon2PasswordEncoder;
 import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
+import javax.persistence.criteria.CriteriaBuilder;
-import com.github.benmanes.caffeine.cache.Caffeine;
+import javax.persistence.criteria.CriteriaQuery;
-import com.github.benmanes.caffeine.cache.LoadingCache;
+import javax.persistence.criteria.Root;
-import com.rometools.rome.feed.synd.SyndEntry;
+import java.io.IOException;
-import com.rometools.rome.feed.synd.SyndEntryImpl;
+import java.net.URISyntaxException;
-import com.rometools.rome.feed.synd.SyndFeed;
+import java.nio.charset.StandardCharsets;
-import com.rometools.rome.feed.synd.SyndFeedImpl;
+import java.util.*;
-import com.rometools.rome.feed.synd.SyndPerson;
+import java.util.concurrent.CompletableFuture;
-import com.rometools.rome.feed.synd.SyndPersonImpl;
+import java.util.concurrent.TimeUnit;
-import com.rometools.rome.io.FeedException;
-import com.rometools.rome.io.SyndFeedOutput;
 
-import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
+import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE;
-import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import static me.kavin.piped.utils.URLUtils.rewriteURL;
-import me.kavin.piped.consts.Constants;
+import static me.kavin.piped.utils.URLUtils.substringYouTube;
-import me.kavin.piped.ipfs.IPFS;
-import me.kavin.piped.utils.obj.Channel;
-import me.kavin.piped.utils.obj.ChapterSegment;
-import me.kavin.piped.utils.obj.Comment;
-import me.kavin.piped.utils.obj.CommentsPage;
-import me.kavin.piped.utils.obj.FeedItem;
-import me.kavin.piped.utils.obj.PipedStream;
-import me.kavin.piped.utils.obj.Playlist;
-import me.kavin.piped.utils.obj.SearchResults;
-import me.kavin.piped.utils.obj.StreamItem;
-import me.kavin.piped.utils.obj.Streams;
-import me.kavin.piped.utils.obj.StreamsPage;
-import me.kavin.piped.utils.obj.SubscriptionChannel;
-import me.kavin.piped.utils.obj.Subtitle;
-import me.kavin.piped.utils.obj.db.PubSub;
-import me.kavin.piped.utils.obj.db.User;
-import me.kavin.piped.utils.obj.db.Video;
-import me.kavin.piped.utils.obj.search.SearchChannel;
-import me.kavin.piped.utils.obj.search.SearchPlaylist;
-import me.kavin.piped.utils.resp.AcceptedResponse;
-import me.kavin.piped.utils.resp.AlreadyRegisteredResponse;
-import me.kavin.piped.utils.resp.AuthenticationFailureResponse;
-import me.kavin.piped.utils.resp.CompromisedPasswordResponse;
-import me.kavin.piped.utils.resp.DisabledRegistrationResponse;
-import me.kavin.piped.utils.resp.IncorrectCredentialsResponse;
-import me.kavin.piped.utils.resp.InvalidRequestResponse;
-import me.kavin.piped.utils.resp.LoginResponse;
-import me.kavin.piped.utils.resp.SubscribeStatusResponse;
 
 public class ResponseHelper {
 
@@ -122,7 +78,7 @@ public class ResponseHelper {
 
         CompletableFuture<String> futureLbryId = CompletableFuture.supplyAsync(() -> {
             try {
-                return getLBRYId(videoId);
+                return LbryHelper.getLBRYId(videoId);
             } catch (Exception e) {
                 ExceptionHandler.handle(e);
             }
@@ -135,7 +91,7 @@ public class ResponseHelper {
 
                 lbryId = futureLbryId.completeOnTimeout(null, 2, TimeUnit.SECONDS).get();
 
-                return getLBRYStreamURL(lbryId);
+                return LbryHelper.getLBRYStreamURL(lbryId);
             } catch (Exception e) {
                 ExceptionHandler.handle(e);
             }
@@ -194,15 +150,10 @@ public class ResponseHelper {
 
         info.getRelatedItems().forEach(o -> relatedStreams.add(collectRelatedStream(o)));
 
-        List<ChapterSegment> segments = new ObjectArrayList<>();
-
-        info.getStreamSegments().forEach(segment -> segments
-                .add(new ChapterSegment(segment.getTitle(), segment.getPreviewUrl(), segment.getStartTimeSeconds())));
-
         long time = info.getUploadDate() != null ? info.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
                 : System.currentTimeMillis();
 
-        if (info.getUploadDate() != null && System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(10))
+        if (info.getUploadDate() != null && System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
             updateViews(info.getId(), info.getViewCount(), time, false);
 
         final Streams streams = new Streams(info.getName(), info.getDescription().getContent(),
@@ -243,7 +194,7 @@ public class ResponseHelper {
                     long time = item.getUploadDate() != null
                             ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
                             : System.currentTimeMillis();
-                    if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(10))
+                    if (System.currentTimeMillis() - time < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
                         updateViews(item.getUrl().substring("https://www.youtube.com/watch?v=".length()),
                                 item.getViewCount(), time, true);
                 }
@@ -269,7 +220,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] channelPageResponse(String channelId, String prevpageStr)
-            throws IOException, ExtractionException, InterruptedException {
+            throws IOException, ExtractionException {
 
         Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
 
@@ -293,7 +244,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] trendingResponse(String region)
-            throws ParsingException, ExtractionException, IOException {
+            throws ExtractionException, IOException {
 
         if (region == null)
             return Constants.mapper.writeValueAsBytes(new InvalidRequestResponse());
@@ -312,7 +263,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] playlistResponse(String playlistId)
-            throws IOException, ExtractionException, InterruptedException {
+            throws IOException, ExtractionException {
 
         final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId);
 
@@ -337,7 +288,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] playlistPageResponse(String playlistId, String prevpageStr)
-            throws IOException, ExtractionException, InterruptedException {
+            throws IOException, ExtractionException {
 
         Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
 
@@ -361,7 +312,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] playlistRSSResponse(String playlistId)
-            throws IOException, ExtractionException, InterruptedException, FeedException {
+            throws IOException, ExtractionException, FeedException {
 
         final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId);
 
@@ -392,22 +343,24 @@ public class ResponseHelper {
     }
 
     public static final byte[] suggestionsResponse(String query)
-            throws JsonProcessingException, IOException, ExtractionException {
+            throws IOException, ExtractionException {
 
         return Constants.mapper.writeValueAsBytes(YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query));
 
     }
 
     public static final byte[] opensearchSuggestionsResponse(String query)
-            throws JsonProcessingException, IOException, ExtractionException {
+            throws IOException, ExtractionException {
 
-        return Constants.mapper.writeValueAsBytes(
+        return Constants.mapper.writeValueAsBytes(Arrays.asList(
-                Arrays.asList(query, YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query)));
+                query,
+                YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query)
+        ));
 
     }
 
     public static final byte[] searchResponse(String q, String filter)
-            throws IOException, ExtractionException, InterruptedException {
+            throws IOException, ExtractionException {
 
         final SearchInfo info = SearchInfo.getInfo(YOUTUBE_SERVICE,
                 YOUTUBE_SERVICE.getSearchQHFactory().fromQuery(q, Collections.singletonList(filter), null));
@@ -443,7 +396,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] searchPageResponse(String q, String filter, String prevpageStr)
-            throws IOException, ExtractionException, InterruptedException {
+            throws IOException, ExtractionException {
 
         Page prevpage = Constants.mapper.readValue(prevpageStr, Page.class);
 
@@ -552,8 +505,8 @@ public class ResponseHelper {
 
     private static final Argon2PasswordEncoder argon2PasswordEncoder = new Argon2PasswordEncoder();
 
-    public static final byte[] registerResponse(String user, String pass) throws IOException, NoSuchAlgorithmException,
+    public static final byte[] registerResponse(String user, String pass) throws IOException,
-            InvalidKeySpecException, InterruptedException, URISyntaxException {
+            InterruptedException, URISyntaxException {
 
         if (Constants.DISABLE_REGISTRATION)
             return Constants.mapper.writeValueAsBytes(new DisabledRegistrationResponse());
@@ -602,7 +555,7 @@ public class ResponseHelper {
     private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder();
 
     public static final byte[] loginResponse(String user, String pass)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+            throws IOException {
 
         if (user == null || pass == null)
             return Constants.mapper.writeValueAsBytes(new InvalidRequestResponse());
@@ -637,7 +590,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] subscribeResponse(String session, String channelId)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+            throws IOException {
 
         Session s = DatabaseSessionFactory.createSession();
 
@@ -684,7 +637,7 @@ public class ResponseHelper {
                             long time = item.getUploadDate() != null
                                     ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
                                     : System.currentTimeMillis();
-                            if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(10))
+                            if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
                                 handleNewVideo(item.getUrl(), time, channel, sess);
                         }
                     }
@@ -703,7 +656,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] unsubscribeResponse(String session, String channelId)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+            throws IOException {
 
         Session s = DatabaseSessionFactory.createSession();
 
@@ -725,7 +678,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] isSubscribedResponse(String session, String channelId)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+            throws IOException {
 
         Session s = DatabaseSessionFactory.createSession();
 
@@ -747,7 +700,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] feedResponse(String session)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+            throws IOException {
 
         Session s = DatabaseSessionFactory.createSession();
 
@@ -774,7 +727,7 @@ public class ResponseHelper {
 
             });
 
-            Collections.sort(feedItems, (a, b) -> (int) (b.uploaded - a.uploaded));
+            feedItems.sort((a, b) -> (int) (b.uploaded - a.uploaded));
 
             s.close();
 
@@ -788,7 +741,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] feedResponseRSS(String session)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException, FeedException {
+            throws IOException, FeedException {
 
         Session s = DatabaseSessionFactory.createSession();
 
@@ -811,8 +764,7 @@ public class ResponseHelper {
                         .setParameter("user", user.getId()).addEntity("Video", Video.class)
                         .addEntity("Channel", me.kavin.piped.utils.obj.db.Channel.class).getResultList();
 
-                Collections.sort(queryResults,
+                queryResults.sort((a, b) -> (int) (((Video) b[0]).getUploaded() - ((Video) a[0]).getUploaded()));
-                        (a, b) -> (int) (((Video) b[0]).getUploaded() - ((Video) a[0]).getUploaded()));
 
                 final List<SyndEntry> entries = new ObjectArrayList<>();
 
@@ -850,7 +802,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] importResponse(String session, String[] channelIds, boolean override)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+            throws IOException {
 
         Session s = DatabaseSessionFactory.createSession();
 
@@ -884,7 +836,7 @@ public class ResponseHelper {
                         me.kavin.piped.utils.obj.db.Channel channel = DatabaseHelper.getChannelFromId(sess, channelId);
 
                         if (channel == null) {
-                            ChannelInfo info = null;
+                            ChannelInfo info;
 
                             try {
                                 info = ChannelInfo.getInfo("https://youtube.com/channel/" + channelId);
@@ -910,7 +862,7 @@ public class ResponseHelper {
                                 long time = item.getUploadDate() != null
                                         ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli()
                                         : System.currentTimeMillis();
-                                if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(10))
+                                if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION))
                                     handleNewVideo(item.getUrl(), time, channel, sess);
                             }
 
@@ -939,7 +891,7 @@ public class ResponseHelper {
     }
 
     public static final byte[] subscriptionsResponse(String session)
-            throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+            throws IOException {
 
         Session s = DatabaseSessionFactory.createSession();
 
@@ -954,12 +906,10 @@ public class ResponseHelper {
                 List<me.kavin.piped.utils.obj.db.Channel> channels = DatabaseHelper.getChannelFromIds(s,
                         user.getSubscribed());
 
-                channels.forEach(channel -> {
+                channels.forEach(channel -> subscriptionItems.add(new SubscriptionChannel("/channel/" + channel.getUploaderId(),
-                    subscriptionItems.add(new SubscriptionChannel("/channel/" + channel.getUploaderId(),
+                        channel.getUploader(), rewriteURL(channel.getUploaderAvatar()), channel.isVerified())));
-                            channel.getUploader(), rewriteURL(channel.getUploaderAvatar()), channel.isVerified()));
-                });
 
-                Collections.sort(subscriptionItems, (a, b) -> (a.name.compareTo(b.name)));
+                subscriptionItems.sort(Comparator.comparing(o -> o.name));
             }
 
             s.close();
@@ -977,37 +927,11 @@ public class ResponseHelper {
 
         Session s = DatabaseSessionFactory.createSession();
 
-        long registered = ((Long) s.createQuery("select count(*) from User").uniqueResult()).longValue();
+        long registered = (Long) s.createQuery("select count(*) from User").uniqueResult();
 
         s.close();
 
-        return String.format("https://img.shields.io/badge/Registered%%20Users-%s-blue", String.valueOf(registered));
+        return String.format("https://img.shields.io/badge/Registered%%20Users-%s-blue", registered);
-    }
-
-    private static final String getLBRYId(String videoId) throws IOException, InterruptedException {
-        return new JSONObject(Constants.h2client.send(HttpRequest
-                .newBuilder(URI.create("https://api.lbry.com/yt/resolve?video_ids=" + URLUtils.silentEncode(videoId)))
-                .setHeader("User-Agent", Constants.USER_AGENT).build(), BodyHandlers.ofString()).body())
-                .getJSONObject("data").getJSONObject("videos").optString(videoId);
-    }
-
-    private static final String getLBRYStreamURL(String lbryId)
-            throws IOException, InterruptedException, ExecutionException {
-
-        if (lbryId != null && !lbryId.isEmpty())
-            return new JSONObject(
-                    Constants.h2client.send(
-                            HttpRequest.newBuilder(URI.create("https://api.lbry.tv/api/v1/proxy?m=get"))
-                                    .POST(BodyPublishers.ofString(
-                                            String.valueOf(new JSONObject().put("id", System.currentTimeMillis())
-                                                    .put("jsonrpc", "2.0").put("method", "get").put("params",
-                                                            new JSONObject().put("uri", "lbry://" + lbryId)
-                                                                    .put("save_file", true)))))
-                                    .build(),
-                            BodyHandlers.ofString()).body()).getJSONObject("result").getString("streaming_url");
-
-        return null;
-
     }
 
     public static void handleNewVideo(String url, long time, me.kavin.piped.utils.obj.db.Channel channel, Session s) {
@@ -1018,8 +942,7 @@ public class ResponseHelper {
         }
     }
 
-    private static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.utils.obj.db.Channel channel,
+    private static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.utils.obj.db.Channel channel, Session s) {
-                                       Session s) {
 
         if (channel == null)
             channel = DatabaseHelper.getChannelFromId(s,
@@ -1031,7 +954,7 @@ public class ResponseHelper {
         Video video = null;
 
         if (channel != null && (video = DatabaseHelper.getVideoFromId(s, info.getId())) == null
-                && (System.currentTimeMillis() - infoTime) < TimeUnit.DAYS.toMillis(10)) {
+                && (System.currentTimeMillis() - infoTime) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) {
 
             video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(),
                     Math.max(infoTime, time), info.getThumbnailUrl(), channel);
@@ -1084,29 +1007,22 @@ public class ResponseHelper {
             String callback = Constants.PUBLIC_URL + "/webhooks/pubsub";
             String topic = "https://www.youtube.com/xml/feeds/videos.xml?channel_id=" + channelId;
 
-            Builder builder = HttpRequest.newBuilder(URI.create("https://pubsubhubbub.appspot.com/subscribe"));
+            var builder = new Request.Builder()
+                    .url("https://pubsubhubbub.appspot.com/subscribe");
 
-            Map<String, String> formParams = new Object2ObjectOpenHashMap<>();
+            var formBuilder = new FormBody.Builder();
-            StringBuilder formBody = new StringBuilder();
 
-            builder.header("content-type", "application/x-www-form-urlencoded");
+            formBuilder.add("hub.callback", callback);
+            formBuilder.add("hub.topic", topic);
+            formBuilder.add("hub.verify", "async");
+            formBuilder.add("hub.mode", "subscribe");
+            formBuilder.add("hub.lease_seconds", "432000");
 
-            formParams.put("hub.callback", callback);
+            var resp = Constants.h2client
-            formParams.put("hub.topic", topic);
+                    .newCall(builder.post(formBuilder.build())
-            formParams.put("hub.verify", "async");
+                            .build()).execute();
-            formParams.put("hub.mode", "subscribe");
-            formParams.put("hub.lease_seconds", "432000");
 
-            formParams.forEach((name, value) -> {
+            if (resp.code() == 202) {
-                formBody.append(name + "=" + URLUtils.silentEncode(value) + "&");
-            });
-
-            builder.method("POST",
-                    BodyPublishers.ofString(String.valueOf(formBody.substring(0, formBody.length() - 1))));
-
-            HttpResponse<InputStream> resp = Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream());
-
-            if (resp.statusCode() == 202) {
                 if (pubsub == null)
                     pubsub = new PubSub(channelId, System.currentTimeMillis());
                 else
@@ -1118,16 +1034,11 @@ public class ResponseHelper {
                     s.getTransaction().begin();
                 s.getTransaction().commit();
             } else
-                System.out.println(
+                System.out.println("Failed to subscribe: " + resp.code() + "\n" + resp.body().string());
-                        "Failed to subscribe: " + resp.statusCode() + "\n" + IOUtils.toString(resp.body(), "UTF-8"));
         }
 
     }
 
-    private static final String substringYouTube(String s) {
-        return StringUtils.isEmpty(s) ? null : StringUtils.substringAfter(s, "youtube.com");
-    }
-
     private static StreamItem collectRelatedStream(Object o) {
 
         StreamInfoItem item = (StreamInfoItem) o;
@@ -1137,35 +1048,4 @@ public class ResponseHelper {
                 rewriteURL(item.getUploaderAvatarUrl()), item.getTextualUploadDate(), item.getDuration(),
                 item.getViewCount(), item.isUploaderVerified());
     }
-
-    private static String rewriteURL(final String old) {
-
-        if (old == null || old.isEmpty())
-            return null;
-
-        URL url = null;
-
-        try {
-            url = new URL(old);
-        } catch (MalformedURLException e) {
-            ExceptionHandler.handle(e);
-        }
-
-        final String host = url.getHost();
-
-        String query = url.getQuery();
-
-        boolean hasQuery = query != null;
-
-        String path = url.getPath();
-
-        if (path.contains("=")) {
-            path = StringUtils.substringBefore(path, "=") + "="
-                    + StringUtils.substringAfter(path, "=").replace("-rj", "-rw");
-        }
-
-        return Constants.PROXY_PART + path + (hasQuery ? "?" + query + "&host=" : "?host=")
-                + URLUtils.silentEncode(host);
-
-    }
 }

src/main/java/me/kavin/piped/utils/SponsorBlockUtils.java

@@ -1,46 +1,36 @@
 package me.kavin.piped.utils;
 
+import com.grack.nanojson.*;
+import me.kavin.piped.consts.Constants;
+import me.kavin.piped.utils.resp.InvalidRequestResponse;
+import org.apache.commons.lang3.StringUtils;
+
 import java.io.IOException;
-import java.net.URI;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse.BodyHandlers;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 
-import org.apache.commons.lang3.StringUtils;
-
-import com.grack.nanojson.JsonArray;
-import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
-import com.grack.nanojson.JsonWriter;
-
-import me.kavin.piped.consts.Constants;
-import me.kavin.piped.utils.resp.InvalidRequestResponse;
-
 public class SponsorBlockUtils {
 
-    public static final String getSponsors(String id, String categories)
+    public static String getSponsors(String id, String categories)
-            throws IOException, InterruptedException, NoSuchAlgorithmException, JsonParserException {
+            throws IOException, NoSuchAlgorithmException, JsonParserException {
 
         if (StringUtils.isEmpty(categories))
             return Constants.mapper.writeValueAsString(new InvalidRequestResponse());
 
         String hash = toSha256(id);
 
-        URI uri = URI.create("https://sponsor.ajay.app/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
-                + "?categories=" + URLUtils.silentEncode(categories));
-
         JsonArray jArray = JsonParser.array().from(
-                Constants.h2client.send(HttpRequest.newBuilder(uri).build(), BodyHandlers.ofInputStream()).body());
+                RequestUtils.sendGet("https://sponsor.ajay.app/api/skipSegments/" + URLUtils.silentEncode(hash.substring(0, 4))
+                        + "?categories=" + URLUtils.silentEncode(categories))
+        );
 
         jArray.removeIf(jObject -> !((JsonObject) jObject).getString("videoID").equalsIgnoreCase(id));
 
         return JsonWriter.string(jArray.getObject(0));
     }
 
-    private static final String toSha256(final String videoId) throws NoSuchAlgorithmException {
+    private static String toSha256(final String videoId) throws NoSuchAlgorithmException {
         final MessageDigest digest = MessageDigest.getInstance("SHA-256");
         final byte[] bytes = digest.digest(videoId.getBytes(StandardCharsets.UTF_8));
         final StringBuilder sb = new StringBuilder();

src/main/java/me/kavin/piped/utils/URLUtils.java

@@ -1,13 +1,19 @@
 package me.kavin.piped.utils;
 
+import me.kavin.piped.consts.Constants;
+import org.apache.commons.lang3.StringUtils;
+
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.net.URLDecoder;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 
 public class URLUtils {
 
     public static String silentEncode(String s) {
         try {
-            return URLEncoder.encode(s, "UTF-8");
+            return URLEncoder.encode(s, StandardCharsets.UTF_8);
         } catch (Exception e) {
             // ignored
         }
@@ -16,10 +22,42 @@ public class URLUtils {
 
     public static String silentDecode(String s) {
         try {
-            return URLDecoder.decode(s, "UTF-8");
+            return URLDecoder.decode(s, StandardCharsets.UTF_8);
         } catch (Exception e) {
             // ignored
         }
         return s;
     }
+
+    public static String substringYouTube(String s) {
+        return StringUtils.isEmpty(s) ? null : StringUtils.substringAfter(s, "youtube.com");
+    }
+
+    public static String rewriteURL(final String old) {
+
+        if (StringUtils.isEmpty(old)) return null;
+
+        URL url = null;
+        try {
+            url = new URL(old);
+        } catch (MalformedURLException e) {
+            ExceptionHandler.handle(e);
+        }
+        assert url != null;
+
+        final String host = url.getHost();
+
+        String query = url.getQuery();
+
+        boolean hasQuery = query != null;
+
+        String path = url.getPath();
+
+        if (path.contains("=")) {
+            path = StringUtils.substringBefore(path, "=") + "=" + StringUtils.substringAfter(path, "=").replace("-rj", "-rw");
+        }
+
+        return Constants.PROXY_PART + path + (hasQuery ? "?" + query + "&host=" : "?host=") + silentEncode(host);
+
+    }
 }

src/main/java/me/kavin/piped/utils/obj/db/Channel.java

@@ -7,7 +7,7 @@ import javax.persistence.Index;
 import javax.persistence.Table;
 
 @Entity
-@Table(name = "channels", indexes = { @Index(columnList = "uploader_id", name = "channels_uploader_id_idx") })
+@Table(name = "channels", indexes = {@Index(columnList = "uploader_id", name = "channels_uploader_id_idx")})
 public class Channel {
 
     @Id

src/main/java/me/kavin/piped/utils/obj/db/Video.java

@@ -1,13 +1,6 @@
 package me.kavin.piped.utils.obj.db;
 
-import javax.persistence.Column;
+import javax.persistence.*;
-import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.Id;
-import javax.persistence.Index;
-import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
-import javax.persistence.Table;
 
 @Entity
 @Table(name = "videos", indexes = { @Index(columnList = "id", name = "videos_id_idx"),