From 9e26139f82b9752fcfb246a14a09f758f01e3382 Mon Sep 17 00:00:00 2001 From: Blatzar <46196380+Blatzar@users.noreply.github.com> Date: Fri, 11 Feb 2022 10:17:04 +0100 Subject: [PATCH] Cracked Sflix anti-scraping --- .../com/lagradost/cloudstream3/MainAPI.kt | 27 +++- .../animeproviders/ZoroProvider.kt | 2 +- .../movieproviders/SflixProvider.kt | 136 ++++++++++++++++-- .../cloudstream3/network/Requests.kt | 29 ++-- .../cloudstream3/network/WebViewResolver.kt | 19 ++- .../cloudstream3/ui/player/GeneratorPlayer.kt | 18 +++ .../cloudstream3/utils/ExtractorApi.kt | 4 +- 7 files changed, 202 insertions(+), 33 deletions(-) diff --git a/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt b/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt index 881bbee5..537e041d 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt @@ -2,6 +2,7 @@ package com.lagradost.cloudstream3 import android.annotation.SuppressLint import android.content.Context +import androidx.annotation.WorkerThread import androidx.preference.PreferenceManager import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.json.JsonMapper @@ -245,23 +246,43 @@ abstract class MainAPI { open val vpnStatus = VPNStatus.None open val providerType = ProviderType.DirectProvider + @WorkerThread suspend open fun getMainPage(): HomePageResponse? { throw NotImplementedError() } - + @WorkerThread suspend open fun search(query: String): List? { throw NotImplementedError() } - + @WorkerThread suspend open fun quickSearch(query: String): List? { throw NotImplementedError() } - + @WorkerThread + /** + * Based on data from search() or getMainPage() it generates a LoadResponse, + * basically opening the info page from a link. + * */ suspend open fun load(url: String): LoadResponse? { throw NotImplementedError() } + /** + * Largely redundant feature for most providers. + * + * This job runs in the background when a link is playing in exoplayer. + * First implemented to do polling for sflix to keep the link from getting expired. + * + * This function might be updated to include exoplayer timestamps etc in the future + * if the need arises. + * */ + @WorkerThread + suspend open fun extractorVerifierJob(extractorData: String?) { + throw NotImplementedError() + } + /**Callback is fired once a link is found, will return true if method is executed successfully*/ + @WorkerThread suspend open fun loadLinks( data: String, isCasting: Boolean, diff --git a/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt b/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt index 53f980ef..3a764941 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt @@ -359,7 +359,7 @@ class ZoroProvider : MainAPI() { list.forEach { subList -> subList.first?.forEach { a -> - a?.toExtractorLink(this, subList.second + " - ${it.first}") + a?.toExtractorLink(this, subList.second + " - ${it.first}", null) ?.forEach(callback) } } diff --git a/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt b/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt index e13e2bbc..9b0cb2a0 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt @@ -1,20 +1,26 @@ package com.lagradost.cloudstream3.movieproviders +import android.util.Log import com.fasterxml.jackson.annotation.JsonProperty import com.lagradost.cloudstream3.* +import com.lagradost.cloudstream3.APIHolder.unixTimeMS import com.lagradost.cloudstream3.LoadResponse.Companion.addActors import com.lagradost.cloudstream3.LoadResponse.Companion.setDuration import com.lagradost.cloudstream3.mvvm.suspendSafeApiCall import com.lagradost.cloudstream3.network.WebViewResolver +import com.lagradost.cloudstream3.network.getRequestCreator +import com.lagradost.cloudstream3.network.text import com.lagradost.cloudstream3.utils.AppUtils.parseJson import com.lagradost.cloudstream3.utils.AppUtils.toJson import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson import com.lagradost.cloudstream3.utils.ExtractorLink import com.lagradost.cloudstream3.utils.M3u8Helper import com.lagradost.cloudstream3.utils.getQualityFromName +import kotlinx.coroutines.delay import org.jsoup.Jsoup import org.jsoup.nodes.Element import java.net.URI +import kotlin.system.measureTimeMillis class SflixProvider(providerUrl: String, providerName: String) : MainAPI() { override val mainUrl = providerUrl @@ -292,12 +298,19 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() { urls?.apmap { url -> suspendSafeApiCall { - val sources = app.get( - url, - interceptor = WebViewResolver( - Regex("""/getSources"""), - ) - ).text + val resolved = WebViewResolver( + Regex("""/getSources"""), + // This is unreliable, generating my own link instead +// additionalUrls = listOf(Regex("""^.*transport=polling(?!.*sid=).*$""")) + ).resolveUsingWebView(getRequestCreator(url)) +// val extractorData = resolved.second.getOrNull(0)?.url?.toString() + + // Some smarter ws11 or w10 selection might be required in the future. + val extractorData = + "https://ws10.rabbitstream.net/socket.io/?EIO=4&transport=polling" + + val sources = resolved.first?.let { app.baseClient.newCall(it).execute().text } + ?: return@suspendSafeApiCall val mapped = parseJson(sources) @@ -314,7 +327,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() { mapped.sourcesBackup to "source backup" ).forEach { (sources, sourceName) -> sources?.forEach { - it?.toExtractorLink(this, sourceName)?.forEach(callback) + it?.toExtractorLink(this, sourceName, extractorData)?.forEach(callback) } } } @@ -323,6 +336,105 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() { return !urls.isNullOrEmpty() } + data class PollingData( + @JsonProperty("sid") val sid: String? = null, + @JsonProperty("upgrades") val upgrades: ArrayList = arrayListOf(), + @JsonProperty("pingInterval") val pingInterval: Int? = null, + @JsonProperty("pingTimeout") val pingTimeout: Int? = null + ) + + /* + # python code to figure out the time offset based on code if necessary + chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_" + code = "Nxa_-bM" + total = 0 + for i, char in enumerate(code[::-1]): + index = chars.index(char) + value = index * 64**i + total += value + print(f"total {total}") + */ + private fun generateTimeStamp(): String { + val chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_" + var code = "" + var time = unixTimeMS + while (time > 0) { + code += chars[(time % (chars.length)).toInt()] + time /= chars.length + } + return code.reversed() + } + + override suspend fun extractorVerifierJob(extractorData: String?) { + if (extractorData == null) return + + val jsonText = + app.get("$extractorData&t=${generateTimeStamp()}").text.replaceBefore("{", "") + val data = parseJson(jsonText) + val headers = mapOf( + "User-Agent" to USER_AGENT, + "Referer" to "https://rabbitstream.net/" + ) + + // 40 is hardcoded, dunno how it's generated, but it seems to work everywhere. + // This request is obligatory + app.post( + "$extractorData&t=${generateTimeStamp()}&sid=${data.sid}", + data = 40, headers = headers + )//.also { println("First post ${it.text}") } + // This makes the second get request work, and re-connect work. + val reconnectSid = + parseJson( + app.get( + "$extractorData&t=${generateTimeStamp()}&sid=${data.sid}", + headers = headers + ) + //.also { println("First get ${it.text}") } + .text.replaceBefore("{", "") + ).sid + // This response is used in the post requests. Same contents in all it seems. + val authInt = + app.get( + "$extractorData&t=${generateTimeStamp()}&sid=${data.sid}", + timeout = 60, + headers = headers + ).text + //.also { println("Second get ${it}") } + // Dunno if it's actually generated like this, just guessing. + .toIntOrNull()?.plus(1) ?: 3 + + // Prevents them from fucking us over with doing a while(true){} loop + val interval = maxOf(data.pingInterval?.toLong()?.plus(2000) ?: return, 10000L) + var reconnect = false + // New SID can be negotiated as above, but not implemented yet as it seems rare. + while (true) { + val authData = if (reconnect) """ + 42["_reconnect", "$reconnectSid"] + """.trimIndent() else authInt + + val url = "${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}" + app.post(url, data = authData, headers = headers) + //.also { println("Sflix post job ${it.text}") } + Log.d(this.name, "Running Sflix job $url") + + val time = measureTimeMillis { + // This acts as a timeout + val getResponse = app.get( + "${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}", + timeout = 60, + headers = headers + ).text //.also { println("Sflix get job $it") } + if (getResponse.contains("sid")) { + reconnect = true +// println("Reconnecting") + } + } + // Always waits even if the get response is instant, to prevent a while true loop. + if (time < interval - 4000) + delay(interval) + } + } + private fun Element.toSearchResult(): SearchResponse { val img = this.select("img") val title = img.attr("title") @@ -363,7 +475,11 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() { } // For re-use in Zoro - fun Sources.toExtractorLink(caller: MainAPI, name: String): List? { + fun Sources.toExtractorLink( + caller: MainAPI, + name: String, + extractorData: String? = null + ): List? { return this.file?.let { file -> //println("FILE::: $file") val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals( @@ -382,7 +498,8 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() { stream.streamUrl, caller.mainUrl, getQualityFromName(stream.quality.toString()), - true + true, + extractorData = extractorData ) } } else { @@ -393,6 +510,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() { caller.mainUrl, getQualityFromName(this.type ?: ""), false, + extractorData = extractorData )) } } diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/Requests.kt b/app/src/main/java/com/lagradost/cloudstream3/network/Requests.kt index 70b9a494..41c2d5bb 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/network/Requests.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/network/Requests.kt @@ -5,13 +5,18 @@ import android.content.Context import android.util.Log import androidx.preference.PreferenceManager import com.fasterxml.jackson.module.kotlin.readValue -import com.lagradost.cloudstream3.* +import com.lagradost.cloudstream3.R +import com.lagradost.cloudstream3.USER_AGENT +import com.lagradost.cloudstream3.app +import com.lagradost.cloudstream3.mapper import kotlinx.coroutines.CancellableContinuation import kotlinx.coroutines.CompletionHandler import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.suspendCancellableCoroutine import okhttp3.* import okhttp3.Headers.Companion.toHeaders +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody.Companion.toRequestBody import org.jsoup.Jsoup import org.jsoup.nodes.Document import java.io.File @@ -107,14 +112,20 @@ class AppResponse( } } -private fun getData(data: Map): RequestBody { - val builder = FormBody.Builder() - data.forEach { - it.value?.let { value -> - builder.add(it.key, value) +private fun getData(data: Any?): RequestBody { + return when (data) { + null -> FormBody.Builder().build() + is Map<*, *> -> { + val builder = FormBody.Builder() + data.forEach { + if (it.key is String && it.value is String) + builder.add(it.key as String, it.value as String) + } + builder.build() } + else -> + data.toString().toRequestBody("text/plain;charset=UTF-8".toMediaTypeOrNull()) } - return builder.build() } // https://github.com, id=test -> https://github.com?id=test @@ -169,7 +180,7 @@ fun postRequestCreator( referer: String? = null, params: Map = emptyMap(), cookies: Map = emptyMap(), - data: Map = emptyMap(), + data: Any? = DEFAULT_DATA, cacheTime: Int = DEFAULT_TIME, cacheUnit: TimeUnit = DEFAULT_TIME_UNIT ): Request { @@ -340,7 +351,7 @@ open class Requests { referer: String? = null, params: Map = mapOf(), cookies: Map = mapOf(), - data: Map = DEFAULT_DATA, + data: Any? = DEFAULT_DATA, allowRedirects: Boolean = true, cacheTime: Int = DEFAULT_TIME, cacheUnit: TimeUnit = DEFAULT_TIME_UNIT, diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt index e0ede7c6..9aa44a08 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt @@ -32,13 +32,13 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List = } /** - * @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls. + * @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls. If true, destroy WebView. * @return the final request (by interceptUrl) and all the collected urls (by additionalUrls). * */ @SuppressLint("SetJavaScriptEnabled") suspend fun resolveUsingWebView( request: Request, - requestCallBack: (Request) -> Unit = {} + requestCallBack: (Request) -> Boolean = { false } ): Pair> { val url = request.url.toString() val headers = request.headers @@ -81,14 +81,18 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List = // println("Loading WebView URL: $webViewUrl") if (interceptUrl.containsMatchIn(webViewUrl)) { - fixedRequest = request.toRequest().also(requestCallBack) + fixedRequest = request.toRequest().also { + if (requestCallBack(it)) destroyWebView() + } println("Web-view request finished: $webViewUrl") destroyWebView() return@runBlocking null } if (additionalUrls.any { it.containsMatchIn(webViewUrl) }) { - extraRequestList.add(request.toRequest().also(requestCallBack)) + extraRequestList.add(request.toRequest().also { + if (requestCallBack(it)) destroyWebView() + }) } // Suppress image requests as we don't display them anywhere @@ -129,11 +133,6 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List = * e.g the recaptcha request. * **/ - /** NOTE! request.requestHeaders is not perfect! - * They don't contain all the headers the browser actually gives. - * Overriding with okhttp might fuck up otherwise working requests, - * e.g the recaptcha request. - * **/ return@runBlocking try { when { blacklistedFiles.any { URI(webViewUrl).path.contains(it) } || webViewUrl.endsWith( @@ -204,7 +203,7 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List = null, emptyMap(), emptyMap(), - emptyMap(), + emptyMap(), 10, TimeUnit.MINUTES ) diff --git a/app/src/main/java/com/lagradost/cloudstream3/ui/player/GeneratorPlayer.kt b/app/src/main/java/com/lagradost/cloudstream3/ui/player/GeneratorPlayer.kt index 97e17b89..4d3811bf 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/ui/player/GeneratorPlayer.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/ui/player/GeneratorPlayer.kt @@ -15,6 +15,7 @@ import androidx.lifecycle.ViewModelProvider import com.google.android.material.button.MaterialButton import com.hippo.unifile.UniFile import com.lagradost.cloudstream3.* +import com.lagradost.cloudstream3.APIHolder.getApiFromNameNull import com.lagradost.cloudstream3.CommonActivity.showToast import com.lagradost.cloudstream3.mvvm.Resource import com.lagradost.cloudstream3.mvvm.logError @@ -26,11 +27,13 @@ import com.lagradost.cloudstream3.ui.result.ResultFragment import com.lagradost.cloudstream3.ui.settings.SettingsFragment.Companion.isTvSettings import com.lagradost.cloudstream3.ui.subtitles.SubtitlesFragment import com.lagradost.cloudstream3.utils.* +import com.lagradost.cloudstream3.utils.Coroutines.ioSafe import com.lagradost.cloudstream3.utils.UIHelper.dismissSafe import com.lagradost.cloudstream3.utils.UIHelper.hideSystemUI import com.lagradost.cloudstream3.utils.UIHelper.popCurrentPage import kotlinx.android.synthetic.main.fragment_player.* import kotlinx.android.synthetic.main.player_custom_layout.* +import kotlinx.coroutines.Job class GeneratorPlayer : FullScreenPlayer() { companion object { @@ -80,6 +83,19 @@ class GeneratorPlayer : FullScreenPlayer() { return durPos.position } + var currentVerifyLink: Job? = null + + private fun loadExtractorJob(extractorLink: ExtractorLink?) { + currentVerifyLink?.cancel() + extractorLink?.let { + currentVerifyLink = ioSafe { + if (it.extractorData != null) { + getApiFromNameNull(it.source)?.extractorVerifierJob(it.extractorData) + } + } + } + } + private fun loadLink(link: Pair?, sameEpisode: Boolean) { if (link == null) return @@ -93,6 +109,7 @@ class GeneratorPlayer : FullScreenPlayer() { setPlayerDimen(null) setTitle() + loadExtractorJob(link.first) // load player context?.let { ctx -> val (url, uri) = link @@ -345,6 +362,7 @@ class GeneratorPlayer : FullScreenPlayer() { override fun onDestroy() { ResultFragment.updateUI() + currentVerifyLink?.cancel() super.onDestroy() } diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt index 89b93f8e..d6be7ba9 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt @@ -16,7 +16,9 @@ data class ExtractorLink( override val referer: String, val quality: Int, val isM3u8: Boolean = false, - override val headers: Map = mapOf() + override val headers: Map = mapOf(), + /** Used for getExtractorVerifierJob() */ + val extractorData: String? = null ) : VideoDownloadManager.IDownloadableMinimum data class ExtractorUri(