Add in-memory anti-captcha implementation.

build.gradle

@@ -28,6 +28,7 @@ dependencies {
     implementation 'com.github.ben-manes.caffeine:caffeine:2.8.6'
     implementation 'com.rometools:rome:1.15.0'
     implementation 'com.github.ipfs:java-ipfs-http-client:v1.3.3'
+    implementation 'org.jsoup:jsoup:1.13.1'
     implementation 'net.java.dev.jna:jna-platform:5.7.0'
 }
 

config.properties

@@ -3,5 +3,8 @@
 PORT: 8080
 
 # Proxy
-PROXY_PART: https://pipedproxy.kavin.rocks
+PROXY_PART: https://pipedproxy-ams.kavin.rocks
 
+# Captcha Parameters
+CAPTCHA_BASE_URL: https://api.capmonster.cloud/
+CAPTCHA_API_KEY: INSERT_HERE

src/main/java/me/kavin/piped/consts/Constants.java

@@ -22,10 +22,14 @@ public class Constants {
 
     public static final String PROXY_PART;
 
+    public static final String CAPTCHA_BASE_URL, CAPTCHA_API_KEY;
+
     public static final StreamingService YOUTUBE_SERVICE;
 
     public static final HttpClient h2client = HttpClient.newBuilder().followRedirects(Redirect.NORMAL)
             .version(Version.HTTP_2).build();
+    public static final HttpClient h2_no_redir_client = HttpClient.newBuilder().followRedirects(Redirect.NEVER)
+            .version(Version.HTTP_2).build();
 //    public static final HttpClient h3client = Http3ClientBuilder.newBuilder().followRedirects(Redirect.NORMAL).build();
 
     public static final MongoClient mongoClient;
@@ -40,6 +44,8 @@ public class Constants {
 
             PORT = Integer.parseInt(prop.getProperty("PORT"));
             PROXY_PART = prop.getProperty("PROXY_PART");
+            CAPTCHA_BASE_URL = prop.getProperty("CAPTCHA_BASE_URL");
+            CAPTCHA_API_KEY = prop.getProperty("CAPTCHA_API_KEY");
             mongoClient = null/* MongoClients.create(prop.getProperty("MONGO_URI")) */;
         } catch (Exception e) {
             throw new RuntimeException(e);

src/main/java/me/kavin/piped/utils/CaptchaSolver.java

@@ -0,0 +1,99 @@
+package me.kavin.piped.utils;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpRequest;
+import java.net.http.HttpRequest.BodyPublishers;
+import java.net.http.HttpRequest.Builder;
+import java.net.http.HttpResponse.BodyHandlers;
+import java.util.Map;
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+import com.grack.nanojson.JsonWriter;
+
+import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
+import me.kavin.piped.consts.Constants;
+import me.kavin.piped.utils.obj.SolvedCaptcha;
+
+public class CaptchaSolver {
+
+    public static SolvedCaptcha solve(String url, String sitekey, String data_s)
+            throws JsonParserException, IOException, InterruptedException {
+
+        int taskId = createTask(url, sitekey, data_s);
+
+        return waitForSolve(taskId);
+
+    }
+
+    private static int createTask(String url, String sitekey, String data_s)
+            throws JsonParserException, IOException, InterruptedException {
+
+        Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "createTask"));
+        JsonObject jObject = new JsonObject();
+        jObject.put("clientKey", Constants.CAPTCHA_API_KEY);
+        {
+            JsonObject task = new JsonObject();
+            task.put("type", "NoCaptchaTaskProxyless");
+            task.put("websiteURL", url);
+            task.put("websiteKey", sitekey);
+            task.put("recaptchaDataSValue", data_s);
+            jObject.put("task", task);
+        }
+
+        builder.method("POST", BodyPublishers.ofString(JsonWriter.string(jObject)));
+
+        builder.header("Content-Type", "application/json");
+
+        JsonObject taskObj = JsonParser.object()
+                .from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body());
+
+        return taskObj.getInt("taskId");
+    }
+
+    private static final SolvedCaptcha waitForSolve(int taskId)
+            throws JsonParserException, IOException, InterruptedException {
+
+        String body = JsonWriter.string(
+                JsonObject.builder().value("clientKey", Constants.CAPTCHA_API_KEY).value("taskId", taskId).done());
+
+        SolvedCaptcha solved = null;
+
+        outer: while (true) {
+            Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "getTaskResult"));
+
+            builder.method("POST", BodyPublishers.ofString(body));
+
+            builder.header("Content-Type", "application/json");
+
+            JsonObject captchaObj = JsonParser.object()
+                    .from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body());
+
+            if (captchaObj.getInt("errorId") != 0)
+                break;
+
+            if (captchaObj.has("solution")) {
+                JsonObject solution = captchaObj.getObject("solution");
+                String captchaResp = solution.getString("gRecaptchaResponse");
+                JsonObject cookieObj = solution.getObject("cookies");
+                Map<String, String> cookies = new Object2ObjectOpenHashMap<>();
+
+                if (captchaResp != null) {
+
+                    cookieObj.keySet().forEach(cookie -> {
+                        cookies.put(cookie, cookieObj.getString(cookie));
+                    });
+
+                    solved = new SolvedCaptcha(cookies, captchaResp);
+                    break outer;
+                }
+            }
+
+            Thread.sleep(1000);
+        }
+
+        return solved;
+    }
+}

src/main/java/me/kavin/piped/utils/DownloaderImpl.java

@@ -1,22 +1,35 @@
 package me.kavin.piped.utils;
 
 import java.io.IOException;
+import java.net.HttpCookie;
 import java.net.URI;
 import java.net.http.HttpRequest;
 import java.net.http.HttpRequest.BodyPublisher;
+import java.net.http.HttpRequest.BodyPublishers;
 import java.net.http.HttpRequest.Builder;
 import java.net.http.HttpResponse;
 import java.net.http.HttpResponse.BodyHandlers;
+import java.util.Map;
 
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Request;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 
+import com.grack.nanojson.JsonParserException;
+
+import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
 import me.kavin.piped.consts.Constants;
+import me.kavin.piped.utils.obj.SolvedCaptcha;
 
 public class DownloaderImpl extends Downloader {
 
+    private static HttpCookie saved_cookie;
+    private static final Object cookie_lock = new Object();
+
     /**
      * Executes a request with HTTP/2.
      */
@@ -31,10 +44,14 @@ public class DownloaderImpl extends Downloader {
                 : HttpRequest.BodyPublishers.ofByteArray(data);
 
         builder.method(request.httpMethod(), publisher);
-        request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value)));
 
         builder.setHeader("User-Agent", Constants.USER_AGENT);
 
+        if (saved_cookie != null && !saved_cookie.hasExpired())
+            builder.setHeader("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue());
+
+        request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value)));
+
         HttpResponse<String> response = null;
 
         try {
@@ -43,9 +60,76 @@ public class DownloaderImpl extends Downloader {
             // ignored
         }
 
-        // TODO: Implement solver
         if (response.statusCode() == 429) {
-            throw new ReCaptchaException("reCaptcha Challenge requested", String.valueOf(response.uri()));
+
+            synchronized (cookie_lock) {
+
+                if (saved_cookie != null && saved_cookie.hasExpired())
+                    saved_cookie = null;
+
+                String redir_url = String.valueOf(response.request().uri());
+
+                if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
+
+                    Map<String, String> formParams = new Object2ObjectOpenHashMap<>();
+                    String sitekey = null, data_s = null;
+
+                    for (Element el : Jsoup.parse(response.body()).selectFirst("form").children()) {
+                        String name;
+                        if (!(name = el.tagName()).equals("script")) {
+                            if (name.equals("input"))
+                                formParams.put(el.attr("name"), el.attr("value"));
+                            else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
+                                sitekey = el.attr("data-sitekey");
+                                data_s = el.attr("data-s");
+                            }
+                        }
+                    }
+                    if (sitekey == null || data_s == null)
+                        throw new ReCaptchaException("Could not get recaptcha", redir_url);
+
+                    SolvedCaptcha solved = null;
+
+                    try {
+                        solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
+                    } catch (JsonParserException | InterruptedException e) {
+                        e.printStackTrace();
+                    }
+
+                    formParams.put("g-recaptcha-response", solved.getRecaptchaResponse());
+
+                    Builder formBuilder = HttpRequest.newBuilder(URI.create("https://www.google.com/sorry/index"));
+
+                    formBuilder.setHeader("User-Agent", Constants.USER_AGENT);
+
+                    StringBuilder formBody = new StringBuilder();
+
+                    formParams.forEach((name, value) -> {
+                        formBody.append(name + "=" + URLUtils.silentEncode(value) + "&");
+                    });
+
+                    formBuilder.header("content-type", "application/x-www-form-urlencoded");
+
+                    formBuilder.method("POST",
+                            BodyPublishers.ofString(String.valueOf(formBody.substring(0, formBody.length() - 1))));
+
+                    try {
+                        HttpResponse<String> formResponse = Constants.h2_no_redir_client.send(formBuilder.build(),
+                                BodyHandlers.ofString());
+
+                        saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
+                                .substringAfter(formResponse.headers().firstValue("Location").get(), "google_abuse=")))
+                                .get(0);
+
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+                }
+
+                if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired.
+                    execute(request);
+            }
+
         }
 
         return new Response(response.statusCode(), "UNDEFINED", response.headers().map(), response.body(),

src/main/java/me/kavin/piped/utils/URLUtils.java

@@ -1,5 +1,6 @@
 package me.kavin.piped.utils;
 
+import java.net.URLDecoder;
 import java.net.URLEncoder;
 
 public class URLUtils {
@@ -12,4 +13,13 @@ public class URLUtils {
         }
         return s;
     }
+
+    public static String silentDecode(String s) {
+        try {
+            return URLDecoder.decode(s, "UTF-8");
+        } catch (Exception e) {
+            // ignored
+        }
+        return s;
+    }
 }

src/main/java/me/kavin/piped/utils/obj/SolvedCaptcha.java

@@ -0,0 +1,22 @@
+package me.kavin.piped.utils.obj;
+
+import java.util.Map;
+
+public class SolvedCaptcha {
+
+    private Map<String, String> cookies;
+    private String gRecaptchaResponse;
+
+    public SolvedCaptcha(Map<String, String> cookies, String gRecaptchaResponse) {
+        this.cookies = cookies;
+        this.gRecaptchaResponse = gRecaptchaResponse;
+    }
+
+    public Map<String, String> getCookies() {
+        return cookies;
+    }
+
+    public String getRecaptchaResponse() {
+        return gRecaptchaResponse;
+    }
+}