From 66a99a26241e94231c71a6b8feb4a81914dd3235 Mon Sep 17 00:00:00 2001 From: Blatzar <46196380+Blatzar@users.noreply.github.com> Date: Sun, 21 Aug 2022 20:14:26 +0200 Subject: [PATCH] Add cloudflare bypass --- .../cloudstream3/network/CloudflareKiller.kt | 94 +++++++++++++++++++ .../cloudstream3/network/DdosGuardKiller.kt | 5 +- .../cloudstream3/network/RequestsHelper.kt | 8 +- .../cloudstream3/network/WebViewResolver.kt | 83 ++++++++++------ .../cloudstream3/ui/WebviewFragment.kt | 8 +- .../lagradost/cloudstream3/utils/AppUtils.kt | 2 +- .../cloudstream3/utils/Coroutines.kt | 7 ++ 7 files changed, 165 insertions(+), 42 deletions(-) create mode 100644 app/src/main/java/com/lagradost/cloudstream3/network/CloudflareKiller.kt diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/CloudflareKiller.kt b/app/src/main/java/com/lagradost/cloudstream3/network/CloudflareKiller.kt new file mode 100644 index 00000000..7c2acf4e --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/network/CloudflareKiller.kt @@ -0,0 +1,94 @@ +package com.lagradost.cloudstream3.network + +import android.util.Log +import android.webkit.CookieManager +import androidx.annotation.AnyThread +import com.lagradost.cloudstream3.app +import com.lagradost.cloudstream3.mvvm.debugWarning +import com.lagradost.nicehttp.Requests.Companion.await +import com.lagradost.nicehttp.cookies +import kotlinx.coroutines.runBlocking +import okhttp3.* + + +@AnyThread +class CloudflareKiller : Interceptor { + companion object { + const val TAG = "CloudflareKiller" + fun parseCookieMap(cookie: String): Map { + return cookie.split(";").associate { + val split = it.split("=") + (split.getOrNull(0)?.trim() ?: "") to (split.getOrNull(1)?.trim() ?: "") + }.filter { it.key.isNotBlank() && it.value.isNotBlank() } + } + } + + val savedCookies: MutableMap = mutableMapOf() + + override fun intercept(chain: Interceptor.Chain): Response = runBlocking { + val request = chain.request() + if (savedCookies[request.url.host] == null) { + bypassCloudflare(request).also { + debugWarning({ it == null }) { "Failed cloudflare at: ${request.url}" } + }?.let { + return@runBlocking it + } + } + return@runBlocking chain.proceed(request) + } + + private fun getWebViewCookie(url: String): String? { + return CookieManager.getInstance()?.getCookie(url) + } + + /** + * Returns true if the cf cookies were successfully fetched from the CookieManager + * Also saves the cookies. + * */ + private fun trySolveWithSavedCookies(request: Request): Boolean { + // Not sure if this takes expiration into account + return getWebViewCookie(request.url.toString())?.let { cookie -> + cookie.contains("cf_clearance").also { solved -> + if (solved) savedCookies[request.url.host] = cookie + } + } ?: false + } + + private suspend fun bypassCloudflare(request: Request): Response? { + val url = request.url.toString() + + // If no cookies then try to get them + // Remove this if statement if cookies expire + if (!trySolveWithSavedCookies(request)) { + Log.d(TAG, "Loading webview to solve cloudflare for ${request.url}") + WebViewResolver( + // Never exit based on url + Regex(".^"), + // Cloudflare needs default user agent + userAgent = null, + // Cannot use okhttp (i think intercepting cookies fails which causes the issues) + useOkhttp = false, + // Match every url for the requestCallBack + additionalUrls = listOf(Regex(".")) + ).resolveUsingWebView( + url + ) { + trySolveWithSavedCookies(request) + } + } + + val cookies = savedCookies[request.url.host] ?: return null + + val mappedCookies = parseCookieMap(cookies) + val userAgentMap = WebViewResolver.getWebViewUserAgent()?.let { + mapOf("user-agent" to it) + } ?: emptyMap() + + val headers = getHeaders(request.headers.toMap() + userAgentMap, mappedCookies + request.cookies) + return app.baseClient.newCall( + request.newBuilder() + .headers(headers) + .build() + ).await() + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/DdosGuardKiller.kt b/app/src/main/java/com/lagradost/cloudstream3/network/DdosGuardKiller.kt index d5271e10..dca3ee00 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/network/DdosGuardKiller.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/network/DdosGuardKiller.kt @@ -1,13 +1,10 @@ package com.lagradost.cloudstream3.network import androidx.annotation.AnyThread -import com.lagradost.cloudstream3.USER_AGENT import com.lagradost.cloudstream3.app import com.lagradost.nicehttp.Requests.Companion.await -import com.lagradost.nicehttp.getCookies +import com.lagradost.nicehttp.cookies import kotlinx.coroutines.runBlocking -import okhttp3.Headers -import okhttp3.Headers.Companion.toHeaders import okhttp3.Interceptor import okhttp3.Request import okhttp3.Response diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/RequestsHelper.kt b/app/src/main/java/com/lagradost/cloudstream3/network/RequestsHelper.kt index c961a117..85e9d318 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/network/RequestsHelper.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/network/RequestsHelper.kt @@ -43,10 +43,10 @@ fun Requests.initClient(context: Context): OkHttpClient { return baseClient } -val Request.cookies: Map - get() { - return this.headers.getCookies("Cookie") - } +//val Request.cookies: Map +// get() { +// return this.headers.getCookies("Cookie") +// } private val DEFAULT_HEADERS = mapOf("user-agent" to USER_AGENT) diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt index 0157fd65..651f104b 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt @@ -4,10 +4,12 @@ import android.annotation.SuppressLint import android.net.http.SslError import android.webkit.* import com.lagradost.cloudstream3.AcraApplication +import com.lagradost.cloudstream3.AcraApplication.Companion.context import com.lagradost.cloudstream3.USER_AGENT import com.lagradost.cloudstream3.app import com.lagradost.cloudstream3.mvvm.logError import com.lagradost.cloudstream3.utils.Coroutines.main +import com.lagradost.cloudstream3.utils.Coroutines.mainWork import com.lagradost.nicehttp.requestCreator import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking @@ -21,14 +23,33 @@ import java.net.URI * @param interceptUrl will stop the WebView when reaching this url. * @param additionalUrls this will make resolveUsingWebView also return all other requests matching the list of Regex. * @param userAgent if null then will use the default user agent + * @param useOkhttp will try to use the okhttp client as much as possible, but this might cause some requests to fail. Disable for cloudflare. * */ class WebViewResolver( val interceptUrl: Regex, val additionalUrls: List = emptyList(), val userAgent: String? = USER_AGENT, + val useOkhttp: Boolean = true ) : Interceptor { + companion object { + var webViewUserAgent: String? = null + + @JvmName("getWebViewUserAgent1") + fun getWebViewUserAgent(): String? { + return webViewUserAgent ?: context?.let { ctx -> + runBlocking { + mainWork { + WebView(ctx).settings.userAgentString.also { userAgent -> + webViewUserAgent = userAgent + } + } + } + } + } + } + override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() return runBlocking { @@ -76,7 +97,7 @@ class WebViewResolver( main { // Useful for debugging -// WebView.setWebContentsDebuggingEnabled(true) + WebView.setWebContentsDebuggingEnabled(true) try { webView = WebView( AcraApplication.context @@ -86,12 +107,13 @@ class WebViewResolver( settings.javaScriptEnabled = true settings.domStorageEnabled = true + webViewUserAgent = settings.userAgentString // Don't set user agent, setting user agent will make cloudflare break. if (userAgent != null) { settings.userAgentString = userAgent } // Blocks unnecessary images, remove if captcha fucks. - settings.blockNetworkImage = true +// settings.blockNetworkImage = true } webView?.webViewClient = object : WebViewClient() { @@ -100,11 +122,11 @@ class WebViewResolver( request: WebResourceRequest ): WebResourceResponse? = runBlocking { val webViewUrl = request.url.toString() -// println("Loading WebView URL: $webViewUrl") + println("Loading WebView URL: $webViewUrl") if (interceptUrl.containsMatchIn(webViewUrl)) { fixedRequest = request.toRequest().also { - if (requestCallBack(it)) destroyWebView() + requestCallBack(it) } println("Web-view request finished: $webViewUrl") destroyWebView() @@ -166,22 +188,22 @@ class WebViewResolver( null, null ) - - webViewUrl.contains("recaptcha") -> super.shouldInterceptRequest( + webViewUrl.contains("recaptcha") || webViewUrl.contains("/cdn-cgi/") -> super.shouldInterceptRequest( view, request ) - request.method == "GET" -> app.get( + useOkhttp && request.method == "GET" -> app.get( webViewUrl, headers = request.requestHeaders ).okhttpResponse.toWebResourceResponse() - request.method == "POST" -> app.post( + useOkhttp && request.method == "POST" -> app.post( webViewUrl, headers = request.requestHeaders ).okhttpResponse.toWebResourceResponse() - else -> return@runBlocking super.shouldInterceptRequest( + + else -> super.shouldInterceptRequest( view, request ) @@ -223,27 +245,28 @@ class WebViewResolver( return null to extraRequestList } - fun WebResourceRequest.toRequest(): Request { - val webViewUrl = this.url.toString() +} - return requestCreator( - this.method, - webViewUrl, - this.requestHeaders, - ) - } +fun WebResourceRequest.toRequest(): Request { + val webViewUrl = this.url.toString() - fun Response.toWebResourceResponse(): WebResourceResponse { - val contentTypeValue = this.header("Content-Type") - // 1. contentType. 2. charset - val typeRegex = Regex("""(.*);(?:.*charset=(.*)(?:|;)|)""") - return if (contentTypeValue != null) { - val found = typeRegex.find(contentTypeValue) - val contentType = found?.groupValues?.getOrNull(1)?.ifBlank { null } ?: contentTypeValue - val charset = found?.groupValues?.getOrNull(2)?.ifBlank { null } - WebResourceResponse(contentType, charset, this.body.byteStream()) - } else { - WebResourceResponse("application/octet-stream", null, this.body.byteStream()) - } + return requestCreator( + this.method, + webViewUrl, + this.requestHeaders, + ) +} + +fun Response.toWebResourceResponse(): WebResourceResponse { + val contentTypeValue = this.header("Content-Type") + // 1. contentType. 2. charset + val typeRegex = Regex("""(.*);(?:.*charset=(.*)(?:|;)|)""") + return if (contentTypeValue != null) { + val found = typeRegex.find(contentTypeValue) + val contentType = found?.groupValues?.getOrNull(1)?.ifBlank { null } ?: contentTypeValue + val charset = found?.groupValues?.getOrNull(2)?.ifBlank { null } + WebResourceResponse(contentType, charset, this.body.byteStream()) + } else { + WebResourceResponse("application/octet-stream", null, this.body.byteStream()) } -} \ No newline at end of file +} diff --git a/app/src/main/java/com/lagradost/cloudstream3/ui/WebviewFragment.kt b/app/src/main/java/com/lagradost/cloudstream3/ui/WebviewFragment.kt index cfbd0950..d773b3a2 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/ui/WebviewFragment.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/ui/WebviewFragment.kt @@ -1,16 +1,16 @@ package com.lagradost.cloudstream3.ui import android.os.Bundle -import androidx.fragment.app.Fragment import android.view.LayoutInflater import android.view.View import android.view.ViewGroup import android.webkit.WebResourceRequest import android.webkit.WebView import android.webkit.WebViewClient +import androidx.fragment.app.Fragment import androidx.navigation.fragment.findNavController import com.lagradost.cloudstream3.R -import com.lagradost.cloudstream3.USER_AGENT +import com.lagradost.cloudstream3.network.WebViewResolver import com.lagradost.cloudstream3.syncproviders.AccountManager.Companion.appStringRepo import com.lagradost.cloudstream3.utils.AppUtils.loadRepository import kotlinx.android.synthetic.main.fragment_webview.* @@ -48,7 +48,9 @@ class WebviewFragment : Fragment() { } web_view.settings.javaScriptEnabled = true web_view.settings.domStorageEnabled = true - web_view.settings.userAgentString = USER_AGENT + + WebViewResolver.webViewUserAgent = web_view.settings.userAgentString +// web_view.settings.userAgentString = USER_AGENT web_view.loadUrl(url) } diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/AppUtils.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/AppUtils.kt index 9b75fe14..fa39e374 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/utils/AppUtils.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/AppUtils.kt @@ -288,7 +288,7 @@ object AppUtils { return this.packageManager.hasSystemFeature("android.software.webview") } - private fun openWebView(fragment: Fragment?, url: String) { + fun openWebView(fragment: Fragment?, url: String) { if (fragment?.context?.hasWebView() == true) normalSafeApiCall { fragment diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/Coroutines.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/Coroutines.kt index 43718e0e..1b4c36f7 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/utils/Coroutines.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/Coroutines.kt @@ -43,6 +43,13 @@ object Coroutines { } } + suspend fun V.mainWork(work: suspend (CoroutineScope.(V) -> T)): T { + val value = this + return withContext(Dispatchers.Main) { + work(value) + } + } + fun runOnMainThread(work: (() -> Unit)) { val mainHandler = Handler(Looper.getMainLooper()) mainHandler.post {