From a0759b4722dbb0b059fbffbfe9c28f7097fc79dc Mon Sep 17 00:00:00 2001 From: FireMasterK <20838718+FireMasterK@users.noreply.github.com> Date: Thu, 4 Mar 2021 19:44:52 +0530 Subject: [PATCH] Add in-memory anti-captcha implementation. --- build.gradle | 1 + config.properties | 5 +- .../java/me/kavin/piped/consts/Constants.java | 6 ++ .../me/kavin/piped/utils/CaptchaSolver.java | 99 +++++++++++++++++++ .../me/kavin/piped/utils/DownloaderImpl.java | 90 ++++++++++++++++- .../java/me/kavin/piped/utils/URLUtils.java | 10 ++ .../kavin/piped/utils/obj/SolvedCaptcha.java | 22 +++++ 7 files changed, 229 insertions(+), 4 deletions(-) create mode 100644 src/main/java/me/kavin/piped/utils/CaptchaSolver.java create mode 100644 src/main/java/me/kavin/piped/utils/obj/SolvedCaptcha.java diff --git a/build.gradle b/build.gradle index 7f9dfb9..8595ce2 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,7 @@ dependencies { implementation 'com.github.ben-manes.caffeine:caffeine:2.8.6' implementation 'com.rometools:rome:1.15.0' implementation 'com.github.ipfs:java-ipfs-http-client:v1.3.3' + implementation 'org.jsoup:jsoup:1.13.1' implementation 'net.java.dev.jna:jna-platform:5.7.0' } diff --git a/config.properties b/config.properties index 100e187..0109681 100644 --- a/config.properties +++ b/config.properties @@ -3,5 +3,8 @@ PORT: 8080 # Proxy -PROXY_PART: https://pipedproxy.kavin.rocks +PROXY_PART: https://pipedproxy-ams.kavin.rocks +# Captcha Parameters +CAPTCHA_BASE_URL: https://api.capmonster.cloud/ +CAPTCHA_API_KEY: INSERT_HERE diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index e36431d..e422369 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -22,10 +22,14 @@ public class Constants { public static final String PROXY_PART; + public static final String CAPTCHA_BASE_URL, CAPTCHA_API_KEY; + public static final StreamingService YOUTUBE_SERVICE; public static final HttpClient h2client = HttpClient.newBuilder().followRedirects(Redirect.NORMAL) .version(Version.HTTP_2).build(); + public static final HttpClient h2_no_redir_client = HttpClient.newBuilder().followRedirects(Redirect.NEVER) + .version(Version.HTTP_2).build(); // public static final HttpClient h3client = Http3ClientBuilder.newBuilder().followRedirects(Redirect.NORMAL).build(); public static final MongoClient mongoClient; @@ -40,6 +44,8 @@ public class Constants { PORT = Integer.parseInt(prop.getProperty("PORT")); PROXY_PART = prop.getProperty("PROXY_PART"); + CAPTCHA_BASE_URL = prop.getProperty("CAPTCHA_BASE_URL"); + CAPTCHA_API_KEY = prop.getProperty("CAPTCHA_API_KEY"); mongoClient = null/* MongoClients.create(prop.getProperty("MONGO_URI")) */; } catch (Exception e) { throw new RuntimeException(e); diff --git a/src/main/java/me/kavin/piped/utils/CaptchaSolver.java b/src/main/java/me/kavin/piped/utils/CaptchaSolver.java new file mode 100644 index 0000000..ac867cc --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/CaptchaSolver.java @@ -0,0 +1,99 @@ +package me.kavin.piped.utils; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpRequest.Builder; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.Map; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; +import com.grack.nanojson.JsonWriter; + +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import me.kavin.piped.consts.Constants; +import me.kavin.piped.utils.obj.SolvedCaptcha; + +public class CaptchaSolver { + + public static SolvedCaptcha solve(String url, String sitekey, String data_s) + throws JsonParserException, IOException, InterruptedException { + + int taskId = createTask(url, sitekey, data_s); + + return waitForSolve(taskId); + + } + + private static int createTask(String url, String sitekey, String data_s) + throws JsonParserException, IOException, InterruptedException { + + Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "createTask")); + JsonObject jObject = new JsonObject(); + jObject.put("clientKey", Constants.CAPTCHA_API_KEY); + { + JsonObject task = new JsonObject(); + task.put("type", "NoCaptchaTaskProxyless"); + task.put("websiteURL", url); + task.put("websiteKey", sitekey); + task.put("recaptchaDataSValue", data_s); + jObject.put("task", task); + } + + builder.method("POST", BodyPublishers.ofString(JsonWriter.string(jObject))); + + builder.header("Content-Type", "application/json"); + + JsonObject taskObj = JsonParser.object() + .from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body()); + + return taskObj.getInt("taskId"); + } + + private static final SolvedCaptcha waitForSolve(int taskId) + throws JsonParserException, IOException, InterruptedException { + + String body = JsonWriter.string( + JsonObject.builder().value("clientKey", Constants.CAPTCHA_API_KEY).value("taskId", taskId).done()); + + SolvedCaptcha solved = null; + + outer: while (true) { + Builder builder = HttpRequest.newBuilder(URI.create(Constants.CAPTCHA_BASE_URL + "getTaskResult")); + + builder.method("POST", BodyPublishers.ofString(body)); + + builder.header("Content-Type", "application/json"); + + JsonObject captchaObj = JsonParser.object() + .from(Constants.h2client.send(builder.build(), BodyHandlers.ofInputStream()).body()); + + if (captchaObj.getInt("errorId") != 0) + break; + + if (captchaObj.has("solution")) { + JsonObject solution = captchaObj.getObject("solution"); + String captchaResp = solution.getString("gRecaptchaResponse"); + JsonObject cookieObj = solution.getObject("cookies"); + Map cookies = new Object2ObjectOpenHashMap<>(); + + if (captchaResp != null) { + + cookieObj.keySet().forEach(cookie -> { + cookies.put(cookie, cookieObj.getString(cookie)); + }); + + solved = new SolvedCaptcha(cookies, captchaResp); + break outer; + } + } + + Thread.sleep(1000); + } + + return solved; + } +} diff --git a/src/main/java/me/kavin/piped/utils/DownloaderImpl.java b/src/main/java/me/kavin/piped/utils/DownloaderImpl.java index e306bc0..e59329a 100644 --- a/src/main/java/me/kavin/piped/utils/DownloaderImpl.java +++ b/src/main/java/me/kavin/piped/utils/DownloaderImpl.java @@ -1,22 +1,35 @@ package me.kavin.piped.utils; import java.io.IOException; +import java.net.HttpCookie; import java.net.URI; import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublisher; +import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpRequest.Builder; import java.net.http.HttpResponse; import java.net.http.HttpResponse.BodyHandlers; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.downloader.Request; import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; +import com.grack.nanojson.JsonParserException; + +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import me.kavin.piped.consts.Constants; +import me.kavin.piped.utils.obj.SolvedCaptcha; public class DownloaderImpl extends Downloader { + private static HttpCookie saved_cookie; + private static final Object cookie_lock = new Object(); + /** * Executes a request with HTTP/2. */ @@ -31,10 +44,14 @@ public class DownloaderImpl extends Downloader { : HttpRequest.BodyPublishers.ofByteArray(data); builder.method(request.httpMethod(), publisher); - request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value))); builder.setHeader("User-Agent", Constants.USER_AGENT); + if (saved_cookie != null && !saved_cookie.hasExpired()) + builder.setHeader("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue()); + + request.headers().forEach((name, values) -> values.forEach(value -> builder.header(name, value))); + HttpResponse response = null; try { @@ -43,9 +60,76 @@ public class DownloaderImpl extends Downloader { // ignored } - // TODO: Implement solver if (response.statusCode() == 429) { - throw new ReCaptchaException("reCaptcha Challenge requested", String.valueOf(response.uri())); + + synchronized (cookie_lock) { + + if (saved_cookie != null && saved_cookie.hasExpired()) + saved_cookie = null; + + String redir_url = String.valueOf(response.request().uri()); + + if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) { + + Map formParams = new Object2ObjectOpenHashMap<>(); + String sitekey = null, data_s = null; + + for (Element el : Jsoup.parse(response.body()).selectFirst("form").children()) { + String name; + if (!(name = el.tagName()).equals("script")) { + if (name.equals("input")) + formParams.put(el.attr("name"), el.attr("value")); + else if (name.equals("div") && el.attr("id").equals("recaptcha")) { + sitekey = el.attr("data-sitekey"); + data_s = el.attr("data-s"); + } + } + } + if (sitekey == null || data_s == null) + throw new ReCaptchaException("Could not get recaptcha", redir_url); + + SolvedCaptcha solved = null; + + try { + solved = CaptchaSolver.solve(redir_url, sitekey, data_s); + } catch (JsonParserException | InterruptedException e) { + e.printStackTrace(); + } + + formParams.put("g-recaptcha-response", solved.getRecaptchaResponse()); + + Builder formBuilder = HttpRequest.newBuilder(URI.create("https://www.google.com/sorry/index")); + + formBuilder.setHeader("User-Agent", Constants.USER_AGENT); + + StringBuilder formBody = new StringBuilder(); + + formParams.forEach((name, value) -> { + formBody.append(name + "=" + URLUtils.silentEncode(value) + "&"); + }); + + formBuilder.header("content-type", "application/x-www-form-urlencoded"); + + formBuilder.method("POST", + BodyPublishers.ofString(String.valueOf(formBody.substring(0, formBody.length() - 1)))); + + try { + HttpResponse formResponse = Constants.h2_no_redir_client.send(formBuilder.build(), + BodyHandlers.ofString()); + + saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils + .substringAfter(formResponse.headers().firstValue("Location").get(), "google_abuse="))) + .get(0); + + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired. + execute(request); + } + } return new Response(response.statusCode(), "UNDEFINED", response.headers().map(), response.body(), diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index 4dc5527..0ef9717 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -1,5 +1,6 @@ package me.kavin.piped.utils; +import java.net.URLDecoder; import java.net.URLEncoder; public class URLUtils { @@ -12,4 +13,13 @@ public class URLUtils { } return s; } + + public static String silentDecode(String s) { + try { + return URLDecoder.decode(s, "UTF-8"); + } catch (Exception e) { + // ignored + } + return s; + } } diff --git a/src/main/java/me/kavin/piped/utils/obj/SolvedCaptcha.java b/src/main/java/me/kavin/piped/utils/obj/SolvedCaptcha.java new file mode 100644 index 0000000..611067f --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/obj/SolvedCaptcha.java @@ -0,0 +1,22 @@ +package me.kavin.piped.utils.obj; + +import java.util.Map; + +public class SolvedCaptcha { + + private Map cookies; + private String gRecaptchaResponse; + + public SolvedCaptcha(Map cookies, String gRecaptchaResponse) { + this.cookies = cookies; + this.gRecaptchaResponse = gRecaptchaResponse; + } + + public Map getCookies() { + return cookies; + } + + public String getRecaptchaResponse() { + return gRecaptchaResponse; + } +}