From 921a1dab3721689619d1c3e5dd941bd40d896afa Mon Sep 17 00:00:00 2001 From: Arjix <53124886+ArjixWasTaken@users.noreply.github.com> Date: Sun, 21 Aug 2022 22:09:56 +0300 Subject: [PATCH] Feat: AdvancedWebView --- .../lagradost/cloudstream3/MainActivity.kt | 3 + .../cloudstream3/network/AdvancedWebview.kt | 483 ++++++++++++++++++ .../cloudstream3/network/WebViewResolver.kt | 35 +- 3 files changed, 494 insertions(+), 27 deletions(-) create mode 100644 app/src/main/java/com/lagradost/cloudstream3/network/AdvancedWebview.kt diff --git a/app/src/main/java/com/lagradost/cloudstream3/MainActivity.kt b/app/src/main/java/com/lagradost/cloudstream3/MainActivity.kt index 49864e65..236c7912 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/MainActivity.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/MainActivity.kt @@ -131,6 +131,8 @@ var app = Requests(responseParser = object : ResponseParser { class MainActivity : AppCompatActivity(), ColorPickerDialogListener { companion object { const val TAG = "MAINACT" + var context : MainActivity? = null + val afterPluginsLoadedEvent = Event() val mainPluginsLoadedEvent = Event() // homepage api, used to speed up time to load for homepage @@ -415,6 +417,7 @@ class MainActivity : AppCompatActivity(), ColorPickerDialogListener { } override fun onCreate(savedInstanceState: Bundle?) { + context = this app.initClient(this) val settingsManager = PreferenceManager.getDefaultSharedPreferences(this) diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/AdvancedWebview.kt b/app/src/main/java/com/lagradost/cloudstream3/network/AdvancedWebview.kt new file mode 100644 index 00000000..8b02625e --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/network/AdvancedWebview.kt @@ -0,0 +1,483 @@ +package com.lagradost.cloudstream3.network + + +import android.app.Dialog +import android.net.http.SslError +import android.util.Log +import android.view.View +import android.webkit.* +import android.widget.RelativeLayout +import com.lagradost.cloudstream3.* +import com.lagradost.cloudstream3.utils.Coroutines.main +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import okhttp3.Response +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import java.net.URI + + +enum class WebViewActions { + VISIT_ADDRESS, + WAIT_FOR_PAGE_LOAD, + WAIT_FOR_X_SECONDS, + WAIT_FOR_NETWORK_CALL, + WAIT_FOR_NETWORK_IDLE, + WAIT_FOR_ELEMENT, + WAIT_FOR_ELEMENT_GONE, + EXECUTE_JAVASCRIPT, + WAIT_FOR_ELEMENT_TO_BE_CLICKABLE, + CAPTURE_REQUESTS_THAT_MATCH_REGEX, +// SEND_KEYS_TO_ELEMENT, + RETURN +} + +data class WebViewAction(val actionType: WebViewActions, val parameter: Any = "", val callback: (AdvancedWebView) -> Unit = { }) + +class AdvancedWebView private constructor( + val url: String, + val actions: ArrayList, + val referer: String?, + val method: String, + val callback: (AdvancedWebView) -> Unit = { }, + val debug: Boolean = false +) { + companion object { + const val TAG = "AdvancedWebViewTag" + } + val headers = mapOf() + var webView: WebView? = null + val remainingActions: ArrayList = actions + var currentHTML: String = "" + + // Made this a getter, because `currentHTML` changes on the fly + val document: Document? + get() = try { Jsoup.parse(currentHTML) } catch (e: Exception) { null } + + private val Instance = this + + data class Builder( + var url: String = "", + var actions: ArrayList = arrayListOf(), + var referer: String? = null, + var method: String = "GET", + var debug: Boolean = false + ) { + fun visitAddress(url: String, cb: (AdvancedWebView) -> Unit = { }) = apply { + if (this.url != "") { + addAction(WebViewAction(WebViewActions.VISIT_ADDRESS, url, cb)) + } else this.url = url + } + fun setReferer(referer: String) = apply { this.referer = referer } + fun setMethod(method: String) = apply { this.method = method } + + private fun addAction(action: WebViewAction) = apply { this.actions.add(action) } + + fun waitForElement(selector: String, cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.WAIT_FOR_ELEMENT, selector, cb)) + } + fun waitForElementGone(selector: String, cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.WAIT_FOR_ELEMENT, selector, cb)) + } + fun waitForElementToBeClickable(selector: String, cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.WAIT_FOR_ELEMENT_TO_BE_CLICKABLE, selector, cb)) + } + fun waitForSeconds(seconds: Long, cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.WAIT_FOR_X_SECONDS, seconds, cb)) + } + fun waitForPageLoad(cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.WAIT_FOR_PAGE_LOAD, "", cb)) + } + fun waitForNetworkIdle(cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.WAIT_FOR_NETWORK_IDLE, "", cb)) + } + fun waitForNetworkCall(targetResource: String, cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.WAIT_FOR_NETWORK_CALL, targetResource, cb)) + } + fun executeJavaScript(code: String, cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.EXECUTE_JAVASCRIPT, code, cb)) + } + fun captureReqsThatMatchRegex(regex: Regex, cb: (AdvancedWebView) -> Unit = { }) = apply { + addAction(WebViewAction(WebViewActions.CAPTURE_REQUESTS_THAT_MATCH_REGEX, regex, cb)) + } +// fun sendKeysToElement(selector: String, text: String, delayInMsPerKeyPress: Long = 50, cb: (AdvancedWebView) -> Unit = { }) = apply { +// addAction(WebViewAction(WebViewActions.SEND_KEYS_TO_ELEMENT, "$selector(__++`__||__`++__)$text(__++`__||__`++__)$delayInMsPerKeyPress", cb)) +// } + fun debug() = apply { debug = true } + fun close() = apply { addAction(WebViewAction(WebViewActions.RETURN, "")) } + + fun build(callback: (AdvancedWebView) -> Unit = { }) = AdvancedWebView(this.url, this.actions, this.referer, this.method, callback, debug) + fun buildAndStart(callback: (AdvancedWebView) -> Unit = { }) = build(callback).apply { this.start() } + } + + private var actionExecutionsPaused = false + private var networkIdleTimestamp = -1; + private var pageHasLoaded = false; + private var isInSleep = false + private var isSendingKeys = false + private var actionStartTimestamp = -1; + + private fun onActionEnded() { + actionExecutionsPaused = false + isSendingKeys = false + actionStartTimestamp = -1 + } + + var Error = "" + + private suspend fun tryExecuteAction() { + if (actionExecutionsPaused || remainingActions.size == 0) return + actionExecutionsPaused = true + actionStartTimestamp = (System.currentTimeMillis() / 1000).toInt() + + main { + if (remainingActions.size > 0) { + val action = remainingActions[0] + when (action.actionType){ + WebViewActions.WAIT_FOR_ELEMENT -> { + webView?.evaluateJavascript("document.querySelector(\"${action.parameter}\")") { + Log.i(TAG, "WAIT_FOR_ELEMENT:: <$it>") + if (it == "{}") { + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + } + onActionEnded() + } + } + + WebViewActions.VISIT_ADDRESS -> { + webView?.loadUrl(action.parameter as String) + + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + onActionEnded() + } + +// WebViewActions.SEND_KEYS_TO_ELEMENT -> { +// isSendingKeys = true +// val (element, characters, timing) = (action.parameter as String).split("(__++`__||__`++__)") // discriminator +// val msPerKey: Long = timing.toLongOrNull() ?: return@main +// +// Log.i(TAG, "SEND_KEYS_TO_ELEMENT:: start") +// webView?.evaluateJavascript("document.querySelector(`$element`)?.click()") { +// main { +// delay(300) +// for (character in characters) { +// Log.i(TAG, "SEND_KEYS_TO_ELEMENT:: character :: $character") +// +// webView?.dispatchKeyEvent(KeyEvent(KeyEvent.ACTION_DOWN, character.code)) +// delay(70) +// webView?.dispatchKeyEvent(KeyEvent(KeyEvent.ACTION_UP, character.code)) +// delay(msPerKey) +// } +// +// updateCurrentHtmlAndRun(action.callback) +// remainingActions.remove(action) +// onActionEnded() +// } +// } +// } + + WebViewActions.WAIT_FOR_ELEMENT_TO_BE_CLICKABLE -> { + webView?.evaluateJavascript( + """ + ((selector) => { + const elem = document.querySelector(selector) + if (elem == undefined) return + const attribute = elem.getAttribute("disabled") + if (attribute === "true" || attribute === '') return + + return "" + (!elem.disabled || true) + })(`${action.parameter}`); + """.trimIndent()) { + if (it == "\"true\""){ + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + } + onActionEnded() + } + } + + WebViewActions.WAIT_FOR_ELEMENT_GONE -> { + webView?.evaluateJavascript("\"\"+ (document.querySelector(\"${action.parameter}\") == undefined)") { + if (it == "\"true\"") { + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + } + onActionEnded() + } + } + + WebViewActions.WAIT_FOR_NETWORK_IDLE -> { +// if (!pageHasLoaded || ((System.currentTimeMillis() / 1000L) - networkIdleTimestamp) < 10) return@main + // we need at least 10 seconds of no network calls being done in order to be in an "IDLE" state + + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + + onActionEnded() + } + + WebViewActions.WAIT_FOR_X_SECONDS -> { + Log.i(TAG, "Waiting for ${remainingActions[0].parameter} seconds...") + isInSleep = true + delay(action.parameter as Long * 1000) + isInSleep = false + Log.i(TAG, "Finished waiting!") + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + + onActionEnded() + } + + WebViewActions.EXECUTE_JAVASCRIPT -> { + Log.i(TAG, "Executing javascript from action...") + webView?.evaluateJavascript(action.parameter as String) { + Log.i(TAG, "JavaScript Execution done! Result: <$it>") + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + + onActionEnded() + } + } + + WebViewActions.RETURN -> { + updateCurrentHtmlAndRun() { /* Do nothing, we only want to update the html */ } + destroyWebView() + remainingActions.clear() + actionStartTimestamp = -1 + } + + else -> { + onActionEnded() + } + } + } + } + } + + private fun destroyWebView() { + main { + webView?.stopLoading() + webView?.destroy() + webView = null + Log.i(TAG, "Destroyed the WebView!") + } + } + + private fun updateCurrentHtmlAndRun(cb: (AdvancedWebView) -> Unit) { + if (webView == null) { + Instance.run(cb) + return + } + + main { + webView?.evaluateJavascript("document.documentElement.outerHTML") { + currentHTML = it + .replace("\\u003C", "<") + .replace("\\\"", "\"") + .replace("\\n", "\n") + .replace("\\t", "\t") + .trimStart('"').trimEnd('"') + + Instance.run(cb) + } + } + } + + var initialized = false + + suspend fun waitUntilDone() = apply { + while (!initialized) { + delay(100) + } + while (webView != null) { + delay(100) + } + } + + val capturedRequests = arrayListOf() + + private var dialog: Dialog? = null + + fun start() { + main { + try { + webView = WebView( + AcraApplication.context + ?: throw RuntimeException("No base context in WebViewResolver") + ).apply { + // Bare minimum to bypass captcha + settings.javaScriptEnabled = true + settings.domStorageEnabled = true + settings.userAgentString = USER_AGENT + settings.blockNetworkImage = true + } + } catch (e: Exception) { + Error = "Error: Failed to create an Advanced WebView, reason: <${e.message}>" + Log.e(TAG, Error) + Log.e(TAG, e.toString()) + destroyWebView() + callback(this) + } + + if (debug) { + webView!!.visibility = View.VISIBLE + val layout = RelativeLayout.LayoutParams( + RelativeLayout.LayoutParams.MATCH_PARENT, + RelativeLayout.LayoutParams.MATCH_PARENT + ) + dialog = Dialog(MainActivity.context!!, android.R.style.Theme_Black_NoTitleBar_Fullscreen) + dialog!!.addContentView(webView as View, layout) + dialog!!.show() + } + + try { + webView?.webViewClient = object : WebViewClient() { + override fun onPageFinished(view: WebView?, url: String?) { + super.onPageFinished(view, url) + pageHasLoaded = true + networkIdleTimestamp = (System.currentTimeMillis() / 1000).toInt(); + + if (remainingActions.size > 0 && remainingActions[0].actionType == WebViewActions.WAIT_FOR_PAGE_LOAD) { + Log.i(TAG, "PAGE FINISHED!") + val action = remainingActions[0] + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + } + } + + override fun onLoadResource(view: WebView?, url: String?) { + super.onLoadResource(view, url) + networkIdleTimestamp = (System.currentTimeMillis() / 1000L).toInt(); + if (remainingActions.size > 0) { + val action = remainingActions[0] + when (action.actionType) { + WebViewActions.WAIT_FOR_NETWORK_CALL -> { + if (URI(url) == URI(action.parameter as String)) { + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + } + } + else -> { /* nothing */ } + } + } + } + + override fun shouldInterceptRequest( + view: WebView, + request: WebResourceRequest + ): WebResourceResponse? = runBlocking { + networkIdleTimestamp = (System.currentTimeMillis() / 1000L).toInt(); + val webViewUrl = request.url.toString() + + val blacklistedFiles = listOf( + ".jpg", ".png", ".webp", ".mpg", + ".mpeg", ".jpeg", ".webm", ".mp4", + ".mp3", ".gifv", ".flv", ".asf", + ".mov", ".mng", ".mkv", ".ogg", + ".avi", ".wav", ".woff2", ".woff", + ".ttf", ".vtt", ".srt", + ".ts", ".gif", + // Warning, this might fuck some future sites, but it's used to make Sflix work. + "wss://" + ) + + val response = try { + when { + blacklistedFiles.any { URI(webViewUrl).path.contains(it) } || webViewUrl.endsWith( + "/favicon.ico" + ) -> WebResourceResponse( + "image/png", + null, + null + ) + + request.method == "GET" -> app.get( + webViewUrl, + headers = request.requestHeaders + ).okhttpResponse.toWebResourceResponse() + + request.method == "POST" -> app.post( + webViewUrl, + headers = request.requestHeaders + ).okhttpResponse.toWebResourceResponse() + + else -> return@runBlocking super.shouldInterceptRequest( + view, + request + ) + } + } catch (e: Exception) { + null + } + + if (remainingActions.size > 0){ + val action = remainingActions[0] + + when (action.actionType) { + WebViewActions.CAPTURE_REQUESTS_THAT_MATCH_REGEX -> { + if ((action.parameter as Regex).containsMatchIn(webViewUrl)) { + if (response != null) capturedRequests.add(response) + updateCurrentHtmlAndRun(action.callback) + remainingActions.remove(action) + } + } + else -> { /* nothing */ } + } + } + + return@runBlocking response + } + + override fun onReceivedSslError( + view: WebView?, + handler: SslErrorHandler?, + error: SslError? + ) { + handler?.proceed() // Ignore ssl issues + } + } + webView?.loadUrl(url, headers.toMap()) + } catch (e: Exception){ + Error = "Failed to create a WebView client!" + Log.e(TAG, Error) + destroyWebView() + Instance.run(callback) + return@main + } + initialized = true + + while (remainingActions.size > 0 && webView != null) { + if (!isInSleep && !isSendingKeys && actionStartTimestamp != -1 && ((System.currentTimeMillis()/1000) - actionStartTimestamp > 20)) { + Log.e(TAG, "AdvancedWebview:: Timeout, an action failed to end in under 20 seconds...") + Error = "ActionTimeout" + break + } + + delay(300) + if (!actionExecutionsPaused) tryExecuteAction() + } + try { + updateCurrentHtmlAndRun(callback) + } catch (e: Exception) { + Log.e(TAG, "Err: $e") + } + if (debug) dialog!!.hide() + destroyWebView() + } + } + fun Response.toWebResourceResponse(): WebResourceResponse { + val contentTypeValue = this.header("Content-Type") + // 1. contentType. 2. charset + val typeRegex = Regex("""(.*);(?:.*charset=(.*)(?:|;)|)""") + return if (contentTypeValue != null) { + val found = typeRegex.find(contentTypeValue) + val contentType = found?.groupValues?.getOrNull(1)?.ifBlank { null } ?: contentTypeValue + val charset = found?.groupValues?.getOrNull(2)?.ifBlank { null } + WebResourceResponse(contentType, charset, this.body?.byteStream()) + } else { + WebResourceResponse("application/octet-stream", null, this.body?.byteStream()) + } + } +} diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt index 538b96f8..5583d34b 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt @@ -145,33 +145,14 @@ class WebViewResolver( // Suppress image requests as we don't display them anywhere // Less data, low chance of causing issues. // blockNetworkImage also does this job but i will keep it for the future. - val blacklistedFiles = listOf( - ".jpg", - ".png", - ".webp", - ".mpg", - ".mpeg", - ".jpeg", - ".webm", - ".mp4", - ".mp3", - ".gifv", - ".flv", - ".asf", - ".mov", - ".mng", - ".mkv", - ".ogg", - ".avi", - ".wav", - ".woff2", - ".woff", - ".ttf", - ".css", - ".vtt", - ".srt", - ".ts", - ".gif", + val blacklistedFiles = listOf( + ".jpg", ".png", ".webp", ".mpg", + ".mpeg", ".jpeg", ".webm", ".mp4", + ".mp3", ".gifv", ".flv", ".asf", + ".mov", ".mng", ".mkv", ".ogg", + ".avi", ".wav", ".woff2", ".woff", + ".ttf", ".vtt", ".srt", + ".ts", ".gif", // Warning, this might fuck some future sites, but it's used to make Sflix work. "wss://" )