Add cloudflare bypass

app/src/main/java/com/lagradost/cloudstream3/network/CloudflareKiller.kt

@@ -0,0 +1,94 @@
+package com.lagradost.cloudstream3.network
+
+import android.util.Log
+import android.webkit.CookieManager
+import androidx.annotation.AnyThread
+import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.mvvm.debugWarning
+import com.lagradost.nicehttp.Requests.Companion.await
+import com.lagradost.nicehttp.cookies
+import kotlinx.coroutines.runBlocking
+import okhttp3.*
+
+
+@AnyThread
+class CloudflareKiller : Interceptor {
+    companion object {
+        const val TAG = "CloudflareKiller"
+        fun parseCookieMap(cookie: String): Map<String, String> {
+            return cookie.split(";").associate {
+                val split = it.split("=")
+                (split.getOrNull(0)?.trim() ?: "") to (split.getOrNull(1)?.trim() ?: "")
+            }.filter { it.key.isNotBlank() && it.value.isNotBlank() }
+        }
+    }
+
+    val savedCookies: MutableMap<String, String> = mutableMapOf()
+
+    override fun intercept(chain: Interceptor.Chain): Response = runBlocking {
+        val request = chain.request()
+        if (savedCookies[request.url.host] == null) {
+            bypassCloudflare(request).also {
+                debugWarning({ it == null }) { "Failed cloudflare at: ${request.url}" }
+            }?.let {
+                return@runBlocking it
+            }
+        }
+        return@runBlocking chain.proceed(request)
+    }
+
+    private fun getWebViewCookie(url: String): String? {
+        return CookieManager.getInstance()?.getCookie(url)
+    }
+
+    /**
+     * Returns true if the cf cookies were successfully fetched from the CookieManager
+     * Also saves the cookies.
+     * */
+    private fun trySolveWithSavedCookies(request: Request): Boolean {
+        // Not sure if this takes expiration into account
+        return getWebViewCookie(request.url.toString())?.let { cookie ->
+            cookie.contains("cf_clearance").also { solved ->
+                if (solved) savedCookies[request.url.host] = cookie
+            }
+        } ?: false
+    }
+
+    private suspend fun bypassCloudflare(request: Request): Response? {
+        val url = request.url.toString()
+
+        // If no cookies then try to get them
+        // Remove this if statement if cookies expire
+        if (!trySolveWithSavedCookies(request)) {
+            Log.d(TAG, "Loading webview to solve cloudflare for ${request.url}")
+            WebViewResolver(
+                // Never exit based on url
+                Regex(".^"),
+                // Cloudflare needs default user agent
+                userAgent = null,
+                // Cannot use okhttp (i think intercepting cookies fails which causes the issues)
+                useOkhttp = false,
+                // Match every url for the requestCallBack
+                additionalUrls = listOf(Regex("."))
+            ).resolveUsingWebView(
+                url
+            ) {
+                trySolveWithSavedCookies(request)
+            }
+        }
+
+        val cookies = savedCookies[request.url.host] ?: return null
+
+        val mappedCookies = parseCookieMap(cookies)
+        val userAgentMap = WebViewResolver.getWebViewUserAgent()?.let {
+            mapOf("user-agent" to it)
+        } ?: emptyMap()
+
+        val headers = getHeaders(request.headers.toMap() + userAgentMap, mappedCookies + request.cookies)
+        return app.baseClient.newCall(
+            request.newBuilder()
+                .headers(headers)
+                .build()
+        ).await()
+    }
+}

app/src/main/java/com/lagradost/cloudstream3/network/DdosGuardKiller.kt

@@ -1,13 +1,10 @@
 package com.lagradost.cloudstream3.network
 
 import androidx.annotation.AnyThread
-import com.lagradost.cloudstream3.USER_AGENT
 import com.lagradost.cloudstream3.app
 import com.lagradost.nicehttp.Requests.Companion.await
-import com.lagradost.nicehttp.getCookies
+import com.lagradost.nicehttp.cookies
 import kotlinx.coroutines.runBlocking
-import okhttp3.Headers
-import okhttp3.Headers.Companion.toHeaders
 import okhttp3.Interceptor
 import okhttp3.Request
 import okhttp3.Response

app/src/main/java/com/lagradost/cloudstream3/network/RequestsHelper.kt

@@ -43,10 +43,10 @@ fun Requests.initClient(context: Context): OkHttpClient {
     return baseClient
 }
 
-val Request.cookies: Map<String, String>
+//val Request.cookies: Map<String, String>
-    get() {
+//    get() {
-        return this.headers.getCookies("Cookie")
+//        return this.headers.getCookies("Cookie")
-    }
+//    }
 
 private val DEFAULT_HEADERS = mapOf("user-agent" to USER_AGENT)
 

app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt

@@ -4,10 +4,12 @@ import android.annotation.SuppressLint
 import android.net.http.SslError
 import android.webkit.*
 import com.lagradost.cloudstream3.AcraApplication
+import com.lagradost.cloudstream3.AcraApplication.Companion.context
 import com.lagradost.cloudstream3.USER_AGENT
 import com.lagradost.cloudstream3.app
 import com.lagradost.cloudstream3.mvvm.logError
 import com.lagradost.cloudstream3.utils.Coroutines.main
+import com.lagradost.cloudstream3.utils.Coroutines.mainWork
 import com.lagradost.nicehttp.requestCreator
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.runBlocking
@@ -21,14 +23,33 @@ import java.net.URI
  * @param interceptUrl will stop the WebView when reaching this url.
  * @param additionalUrls this will make resolveUsingWebView also return all other requests matching the list of Regex.
  * @param userAgent if null then will use the default user agent
+ * @param useOkhttp will try to use the okhttp client as much as possible, but this might cause some requests to fail. Disable for cloudflare.
  * */
 class WebViewResolver(
     val interceptUrl: Regex,
     val additionalUrls: List<Regex> = emptyList(),
     val userAgent: String? = USER_AGENT,
+    val useOkhttp: Boolean = true
 ) :
     Interceptor {
 
+    companion object {
+        var webViewUserAgent: String? = null
+
+        @JvmName("getWebViewUserAgent1")
+        fun getWebViewUserAgent(): String? {
+            return webViewUserAgent ?: context?.let { ctx ->
+                runBlocking {
+                    mainWork {
+                        WebView(ctx).settings.userAgentString.also { userAgent ->
+                            webViewUserAgent = userAgent
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     override fun intercept(chain: Interceptor.Chain): Response {
         val request = chain.request()
         return runBlocking {
@@ -76,7 +97,7 @@ class WebViewResolver(
 
         main {
             // Useful for debugging
-//            WebView.setWebContentsDebuggingEnabled(true)
+            WebView.setWebContentsDebuggingEnabled(true)
             try {
                 webView = WebView(
                     AcraApplication.context
@@ -86,12 +107,13 @@ class WebViewResolver(
                     settings.javaScriptEnabled = true
                     settings.domStorageEnabled = true
 
+                    webViewUserAgent = settings.userAgentString
                     // Don't set user agent, setting user agent will make cloudflare break.
                     if (userAgent != null) {
                         settings.userAgentString = userAgent
                     }
                     // Blocks unnecessary images, remove if captcha fucks.
-                    settings.blockNetworkImage = true
+//                    settings.blockNetworkImage = true
                 }
 
                 webView?.webViewClient = object : WebViewClient() {
@@ -100,11 +122,11 @@ class WebViewResolver(
                         request: WebResourceRequest
                     ): WebResourceResponse? = runBlocking {
                         val webViewUrl = request.url.toString()
-//                    println("Loading WebView URL: $webViewUrl")
+                        println("Loading WebView URL: $webViewUrl")
 
                         if (interceptUrl.containsMatchIn(webViewUrl)) {
                             fixedRequest = request.toRequest().also {
-                                if (requestCallBack(it)) destroyWebView()
+                                requestCallBack(it)
                             }
                             println("Web-view request finished: $webViewUrl")
                             destroyWebView()
@@ -166,22 +188,22 @@ class WebViewResolver(
                                     null,
                                     null
                                 )
-
+                                webViewUrl.contains("recaptcha") || webViewUrl.contains("/cdn-cgi/") -> super.shouldInterceptRequest(
-                                webViewUrl.contains("recaptcha") -> super.shouldInterceptRequest(
                                     view,
                                     request
                                 )
 
-                                request.method == "GET" -> app.get(
+                                useOkhttp && request.method == "GET" -> app.get(
                                     webViewUrl,
                                     headers = request.requestHeaders
                                 ).okhttpResponse.toWebResourceResponse()
 
-                                request.method == "POST" -> app.post(
+                                useOkhttp && request.method == "POST" -> app.post(
                                     webViewUrl,
                                     headers = request.requestHeaders
                                 ).okhttpResponse.toWebResourceResponse()
-                                else -> return@runBlocking super.shouldInterceptRequest(
+
+                                else -> super.shouldInterceptRequest(
                                     view,
                                     request
                                 )
@@ -223,27 +245,28 @@ class WebViewResolver(
         return null to extraRequestList
     }
 
-    fun WebResourceRequest.toRequest(): Request {
+}
-        val webViewUrl = this.url.toString()
 
-        return requestCreator(
+fun WebResourceRequest.toRequest(): Request {
-            this.method,
+    val webViewUrl = this.url.toString()
-            webViewUrl,
-            this.requestHeaders,
-        )
-    }
 
-    fun Response.toWebResourceResponse(): WebResourceResponse {
+    return requestCreator(
-        val contentTypeValue = this.header("Content-Type")
+        this.method,
-        // 1. contentType. 2. charset
+        webViewUrl,
-        val typeRegex = Regex("""(.*);(?:.*charset=(.*)(?:|;)|)""")
+        this.requestHeaders,
-        return if (contentTypeValue != null) {
+    )
-            val found = typeRegex.find(contentTypeValue)
+}
-            val contentType = found?.groupValues?.getOrNull(1)?.ifBlank { null } ?: contentTypeValue
+
-            val charset = found?.groupValues?.getOrNull(2)?.ifBlank { null }
+fun Response.toWebResourceResponse(): WebResourceResponse {
-            WebResourceResponse(contentType, charset, this.body.byteStream())
+    val contentTypeValue = this.header("Content-Type")
-        } else {
+    // 1. contentType. 2. charset
-            WebResourceResponse("application/octet-stream", null, this.body.byteStream())
+    val typeRegex = Regex("""(.*);(?:.*charset=(.*)(?:|;)|)""")
-        }
+    return if (contentTypeValue != null) {
+        val found = typeRegex.find(contentTypeValue)
+        val contentType = found?.groupValues?.getOrNull(1)?.ifBlank { null } ?: contentTypeValue
+        val charset = found?.groupValues?.getOrNull(2)?.ifBlank { null }
+        WebResourceResponse(contentType, charset, this.body.byteStream())
+    } else {
+        WebResourceResponse("application/octet-stream", null, this.body.byteStream())
     }
 }

app/src/main/java/com/lagradost/cloudstream3/ui/WebviewFragment.kt

@@ -1,16 +1,16 @@
 package com.lagradost.cloudstream3.ui
 
 import android.os.Bundle
-import androidx.fragment.app.Fragment
 import android.view.LayoutInflater
 import android.view.View
 import android.view.ViewGroup
 import android.webkit.WebResourceRequest
 import android.webkit.WebView
 import android.webkit.WebViewClient
+import androidx.fragment.app.Fragment
 import androidx.navigation.fragment.findNavController
 import com.lagradost.cloudstream3.R
-import com.lagradost.cloudstream3.USER_AGENT
+import com.lagradost.cloudstream3.network.WebViewResolver
 import com.lagradost.cloudstream3.syncproviders.AccountManager.Companion.appStringRepo
 import com.lagradost.cloudstream3.utils.AppUtils.loadRepository
 import kotlinx.android.synthetic.main.fragment_webview.*
@@ -48,7 +48,9 @@ class WebviewFragment : Fragment() {
         }
         web_view.settings.javaScriptEnabled = true
         web_view.settings.domStorageEnabled = true
-        web_view.settings.userAgentString = USER_AGENT
+
+        WebViewResolver.webViewUserAgent = web_view.settings.userAgentString
+//        web_view.settings.userAgentString = USER_AGENT
         web_view.loadUrl(url)
     }
 

app/src/main/java/com/lagradost/cloudstream3/utils/AppUtils.kt

@@ -288,7 +288,7 @@ object AppUtils {
         return this.packageManager.hasSystemFeature("android.software.webview")
     }
 
-    private fun openWebView(fragment: Fragment?, url: String) {
+    fun openWebView(fragment: Fragment?, url: String) {
         if (fragment?.context?.hasWebView() == true)
             normalSafeApiCall {
                 fragment

app/src/main/java/com/lagradost/cloudstream3/utils/Coroutines.kt

@@ -43,6 +43,13 @@ object Coroutines {
         }
     }
 
+    suspend fun <T, V> V.mainWork(work: suspend (CoroutineScope.(V) -> T)): T {
+        val value = this
+        return withContext(Dispatchers.Main) {
+            work(value)
+        }
+    }
+
     fun runOnMainThread(work: (() -> Unit)) {
         val mainHandler = Handler(Looper.getMainLooper())
         mainHandler.post {