Cracked Sflix anti-scraping

This commit is contained in:
Blatzar 2022-02-11 10:17:04 +01:00
parent e04e6e686e
commit 9e26139f82
7 changed files with 202 additions and 33 deletions

View file

@ -2,6 +2,7 @@ package com.lagradost.cloudstream3
import android.annotation.SuppressLint import android.annotation.SuppressLint
import android.content.Context import android.content.Context
import androidx.annotation.WorkerThread
import androidx.preference.PreferenceManager import androidx.preference.PreferenceManager
import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.json.JsonMapper import com.fasterxml.jackson.databind.json.JsonMapper
@ -245,23 +246,43 @@ abstract class MainAPI {
open val vpnStatus = VPNStatus.None open val vpnStatus = VPNStatus.None
open val providerType = ProviderType.DirectProvider open val providerType = ProviderType.DirectProvider
@WorkerThread
suspend open fun getMainPage(): HomePageResponse? { suspend open fun getMainPage(): HomePageResponse? {
throw NotImplementedError() throw NotImplementedError()
} }
@WorkerThread
suspend open fun search(query: String): List<SearchResponse>? { suspend open fun search(query: String): List<SearchResponse>? {
throw NotImplementedError() throw NotImplementedError()
} }
@WorkerThread
suspend open fun quickSearch(query: String): List<SearchResponse>? { suspend open fun quickSearch(query: String): List<SearchResponse>? {
throw NotImplementedError() throw NotImplementedError()
} }
@WorkerThread
/**
* Based on data from search() or getMainPage() it generates a LoadResponse,
* basically opening the info page from a link.
* */
suspend open fun load(url: String): LoadResponse? { suspend open fun load(url: String): LoadResponse? {
throw NotImplementedError() throw NotImplementedError()
} }
/**
* Largely redundant feature for most providers.
*
* This job runs in the background when a link is playing in exoplayer.
* First implemented to do polling for sflix to keep the link from getting expired.
*
* This function might be updated to include exoplayer timestamps etc in the future
* if the need arises.
* */
@WorkerThread
suspend open fun extractorVerifierJob(extractorData: String?) {
throw NotImplementedError()
}
/**Callback is fired once a link is found, will return true if method is executed successfully*/ /**Callback is fired once a link is found, will return true if method is executed successfully*/
@WorkerThread
suspend open fun loadLinks( suspend open fun loadLinks(
data: String, data: String,
isCasting: Boolean, isCasting: Boolean,

View file

@ -359,7 +359,7 @@ class ZoroProvider : MainAPI() {
list.forEach { subList -> list.forEach { subList ->
subList.first?.forEach { a -> subList.first?.forEach { a ->
a?.toExtractorLink(this, subList.second + " - ${it.first}") a?.toExtractorLink(this, subList.second + " - ${it.first}", null)
?.forEach(callback) ?.forEach(callback)
} }
} }

View file

@ -1,20 +1,26 @@
package com.lagradost.cloudstream3.movieproviders package com.lagradost.cloudstream3.movieproviders
import android.util.Log
import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonProperty
import com.lagradost.cloudstream3.* import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.APIHolder.unixTimeMS
import com.lagradost.cloudstream3.LoadResponse.Companion.addActors import com.lagradost.cloudstream3.LoadResponse.Companion.addActors
import com.lagradost.cloudstream3.LoadResponse.Companion.setDuration import com.lagradost.cloudstream3.LoadResponse.Companion.setDuration
import com.lagradost.cloudstream3.mvvm.suspendSafeApiCall import com.lagradost.cloudstream3.mvvm.suspendSafeApiCall
import com.lagradost.cloudstream3.network.WebViewResolver import com.lagradost.cloudstream3.network.WebViewResolver
import com.lagradost.cloudstream3.network.getRequestCreator
import com.lagradost.cloudstream3.network.text
import com.lagradost.cloudstream3.utils.AppUtils.parseJson import com.lagradost.cloudstream3.utils.AppUtils.parseJson
import com.lagradost.cloudstream3.utils.AppUtils.toJson import com.lagradost.cloudstream3.utils.AppUtils.toJson
import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
import com.lagradost.cloudstream3.utils.ExtractorLink import com.lagradost.cloudstream3.utils.ExtractorLink
import com.lagradost.cloudstream3.utils.M3u8Helper import com.lagradost.cloudstream3.utils.M3u8Helper
import com.lagradost.cloudstream3.utils.getQualityFromName import com.lagradost.cloudstream3.utils.getQualityFromName
import kotlinx.coroutines.delay
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.nodes.Element import org.jsoup.nodes.Element
import java.net.URI import java.net.URI
import kotlin.system.measureTimeMillis
class SflixProvider(providerUrl: String, providerName: String) : MainAPI() { class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
override val mainUrl = providerUrl override val mainUrl = providerUrl
@ -292,12 +298,19 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
urls?.apmap { url -> urls?.apmap { url ->
suspendSafeApiCall { suspendSafeApiCall {
val sources = app.get( val resolved = WebViewResolver(
url, Regex("""/getSources"""),
interceptor = WebViewResolver( // This is unreliable, generating my own link instead
Regex("""/getSources"""), // additionalUrls = listOf(Regex("""^.*transport=polling(?!.*sid=).*$"""))
) ).resolveUsingWebView(getRequestCreator(url))
).text // val extractorData = resolved.second.getOrNull(0)?.url?.toString()
// Some smarter ws11 or w10 selection might be required in the future.
val extractorData =
"https://ws10.rabbitstream.net/socket.io/?EIO=4&transport=polling"
val sources = resolved.first?.let { app.baseClient.newCall(it).execute().text }
?: return@suspendSafeApiCall
val mapped = parseJson<SourceObject>(sources) val mapped = parseJson<SourceObject>(sources)
@ -314,7 +327,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
mapped.sourcesBackup to "source backup" mapped.sourcesBackup to "source backup"
).forEach { (sources, sourceName) -> ).forEach { (sources, sourceName) ->
sources?.forEach { sources?.forEach {
it?.toExtractorLink(this, sourceName)?.forEach(callback) it?.toExtractorLink(this, sourceName, extractorData)?.forEach(callback)
} }
} }
} }
@ -323,6 +336,105 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
return !urls.isNullOrEmpty() return !urls.isNullOrEmpty()
} }
data class PollingData(
@JsonProperty("sid") val sid: String? = null,
@JsonProperty("upgrades") val upgrades: ArrayList<String> = arrayListOf(),
@JsonProperty("pingInterval") val pingInterval: Int? = null,
@JsonProperty("pingTimeout") val pingTimeout: Int? = null
)
/*
# python code to figure out the time offset based on code if necessary
chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"
code = "Nxa_-bM"
total = 0
for i, char in enumerate(code[::-1]):
index = chars.index(char)
value = index * 64**i
total += value
print(f"total {total}")
*/
private fun generateTimeStamp(): String {
val chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"
var code = ""
var time = unixTimeMS
while (time > 0) {
code += chars[(time % (chars.length)).toInt()]
time /= chars.length
}
return code.reversed()
}
override suspend fun extractorVerifierJob(extractorData: String?) {
if (extractorData == null) return
val jsonText =
app.get("$extractorData&t=${generateTimeStamp()}").text.replaceBefore("{", "")
val data = parseJson<PollingData>(jsonText)
val headers = mapOf(
"User-Agent" to USER_AGENT,
"Referer" to "https://rabbitstream.net/"
)
// 40 is hardcoded, dunno how it's generated, but it seems to work everywhere.
// This request is obligatory
app.post(
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
data = 40, headers = headers
)//.also { println("First post ${it.text}") }
// This makes the second get request work, and re-connect work.
val reconnectSid =
parseJson<PollingData>(
app.get(
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
headers = headers
)
//.also { println("First get ${it.text}") }
.text.replaceBefore("{", "")
).sid
// This response is used in the post requests. Same contents in all it seems.
val authInt =
app.get(
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
timeout = 60,
headers = headers
).text
//.also { println("Second get ${it}") }
// Dunno if it's actually generated like this, just guessing.
.toIntOrNull()?.plus(1) ?: 3
// Prevents them from fucking us over with doing a while(true){} loop
val interval = maxOf(data.pingInterval?.toLong()?.plus(2000) ?: return, 10000L)
var reconnect = false
// New SID can be negotiated as above, but not implemented yet as it seems rare.
while (true) {
val authData = if (reconnect) """
42["_reconnect", "$reconnectSid"]
""".trimIndent() else authInt
val url = "${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}"
app.post(url, data = authData, headers = headers)
//.also { println("Sflix post job ${it.text}") }
Log.d(this.name, "Running Sflix job $url")
val time = measureTimeMillis {
// This acts as a timeout
val getResponse = app.get(
"${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}",
timeout = 60,
headers = headers
).text //.also { println("Sflix get job $it") }
if (getResponse.contains("sid")) {
reconnect = true
// println("Reconnecting")
}
}
// Always waits even if the get response is instant, to prevent a while true loop.
if (time < interval - 4000)
delay(interval)
}
}
private fun Element.toSearchResult(): SearchResponse { private fun Element.toSearchResult(): SearchResponse {
val img = this.select("img") val img = this.select("img")
val title = img.attr("title") val title = img.attr("title")
@ -363,7 +475,11 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
} }
// For re-use in Zoro // For re-use in Zoro
fun Sources.toExtractorLink(caller: MainAPI, name: String): List<ExtractorLink>? { fun Sources.toExtractorLink(
caller: MainAPI,
name: String,
extractorData: String? = null
): List<ExtractorLink>? {
return this.file?.let { file -> return this.file?.let { file ->
//println("FILE::: $file") //println("FILE::: $file")
val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals( val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals(
@ -382,7 +498,8 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
stream.streamUrl, stream.streamUrl,
caller.mainUrl, caller.mainUrl,
getQualityFromName(stream.quality.toString()), getQualityFromName(stream.quality.toString()),
true true,
extractorData = extractorData
) )
} }
} else { } else {
@ -393,6 +510,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
caller.mainUrl, caller.mainUrl,
getQualityFromName(this.type ?: ""), getQualityFromName(this.type ?: ""),
false, false,
extractorData = extractorData
)) ))
} }
} }

View file

@ -5,13 +5,18 @@ import android.content.Context
import android.util.Log import android.util.Log
import androidx.preference.PreferenceManager import androidx.preference.PreferenceManager
import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.readValue
import com.lagradost.cloudstream3.* import com.lagradost.cloudstream3.R
import com.lagradost.cloudstream3.USER_AGENT
import com.lagradost.cloudstream3.app
import com.lagradost.cloudstream3.mapper
import kotlinx.coroutines.CancellableContinuation import kotlinx.coroutines.CancellableContinuation
import kotlinx.coroutines.CompletionHandler import kotlinx.coroutines.CompletionHandler
import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.* import okhttp3.*
import okhttp3.Headers.Companion.toHeaders import okhttp3.Headers.Companion.toHeaders
import okhttp3.MediaType.Companion.toMediaTypeOrNull
import okhttp3.RequestBody.Companion.toRequestBody
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import java.io.File import java.io.File
@ -107,14 +112,20 @@ class AppResponse(
} }
} }
private fun getData(data: Map<String, String?>): RequestBody { private fun getData(data: Any?): RequestBody {
val builder = FormBody.Builder() return when (data) {
data.forEach { null -> FormBody.Builder().build()
it.value?.let { value -> is Map<*, *> -> {
builder.add(it.key, value) val builder = FormBody.Builder()
data.forEach {
if (it.key is String && it.value is String)
builder.add(it.key as String, it.value as String)
}
builder.build()
} }
else ->
data.toString().toRequestBody("text/plain;charset=UTF-8".toMediaTypeOrNull())
} }
return builder.build()
} }
// https://github.com, id=test -> https://github.com?id=test // https://github.com, id=test -> https://github.com?id=test
@ -169,7 +180,7 @@ fun postRequestCreator(
referer: String? = null, referer: String? = null,
params: Map<String, String> = emptyMap(), params: Map<String, String> = emptyMap(),
cookies: Map<String, String> = emptyMap(), cookies: Map<String, String> = emptyMap(),
data: Map<String, String> = emptyMap(), data: Any? = DEFAULT_DATA,
cacheTime: Int = DEFAULT_TIME, cacheTime: Int = DEFAULT_TIME,
cacheUnit: TimeUnit = DEFAULT_TIME_UNIT cacheUnit: TimeUnit = DEFAULT_TIME_UNIT
): Request { ): Request {
@ -340,7 +351,7 @@ open class Requests {
referer: String? = null, referer: String? = null,
params: Map<String, String> = mapOf(), params: Map<String, String> = mapOf(),
cookies: Map<String, String> = mapOf(), cookies: Map<String, String> = mapOf(),
data: Map<String, String> = DEFAULT_DATA, data: Any? = DEFAULT_DATA,
allowRedirects: Boolean = true, allowRedirects: Boolean = true,
cacheTime: Int = DEFAULT_TIME, cacheTime: Int = DEFAULT_TIME,
cacheUnit: TimeUnit = DEFAULT_TIME_UNIT, cacheUnit: TimeUnit = DEFAULT_TIME_UNIT,

View file

@ -32,13 +32,13 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
} }
/** /**
* @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls. * @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls. If true, destroy WebView.
* @return the final request (by interceptUrl) and all the collected urls (by additionalUrls). * @return the final request (by interceptUrl) and all the collected urls (by additionalUrls).
* */ * */
@SuppressLint("SetJavaScriptEnabled") @SuppressLint("SetJavaScriptEnabled")
suspend fun resolveUsingWebView( suspend fun resolveUsingWebView(
request: Request, request: Request,
requestCallBack: (Request) -> Unit = {} requestCallBack: (Request) -> Boolean = { false }
): Pair<Request?, List<Request>> { ): Pair<Request?, List<Request>> {
val url = request.url.toString() val url = request.url.toString()
val headers = request.headers val headers = request.headers
@ -81,14 +81,18 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
// println("Loading WebView URL: $webViewUrl") // println("Loading WebView URL: $webViewUrl")
if (interceptUrl.containsMatchIn(webViewUrl)) { if (interceptUrl.containsMatchIn(webViewUrl)) {
fixedRequest = request.toRequest().also(requestCallBack) fixedRequest = request.toRequest().also {
if (requestCallBack(it)) destroyWebView()
}
println("Web-view request finished: $webViewUrl") println("Web-view request finished: $webViewUrl")
destroyWebView() destroyWebView()
return@runBlocking null return@runBlocking null
} }
if (additionalUrls.any { it.containsMatchIn(webViewUrl) }) { if (additionalUrls.any { it.containsMatchIn(webViewUrl) }) {
extraRequestList.add(request.toRequest().also(requestCallBack)) extraRequestList.add(request.toRequest().also {
if (requestCallBack(it)) destroyWebView()
})
} }
// Suppress image requests as we don't display them anywhere // Suppress image requests as we don't display them anywhere
@ -129,11 +133,6 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
* e.g the recaptcha request. * e.g the recaptcha request.
* **/ * **/
/** NOTE! request.requestHeaders is not perfect!
* They don't contain all the headers the browser actually gives.
* Overriding with okhttp might fuck up otherwise working requests,
* e.g the recaptcha request.
* **/
return@runBlocking try { return@runBlocking try {
when { when {
blacklistedFiles.any { URI(webViewUrl).path.contains(it) } || webViewUrl.endsWith( blacklistedFiles.any { URI(webViewUrl).path.contains(it) } || webViewUrl.endsWith(
@ -204,7 +203,7 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
null, null,
emptyMap(), emptyMap(),
emptyMap(), emptyMap(),
emptyMap(), emptyMap<String, String>(),
10, 10,
TimeUnit.MINUTES TimeUnit.MINUTES
) )

View file

@ -15,6 +15,7 @@ import androidx.lifecycle.ViewModelProvider
import com.google.android.material.button.MaterialButton import com.google.android.material.button.MaterialButton
import com.hippo.unifile.UniFile import com.hippo.unifile.UniFile
import com.lagradost.cloudstream3.* import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.APIHolder.getApiFromNameNull
import com.lagradost.cloudstream3.CommonActivity.showToast import com.lagradost.cloudstream3.CommonActivity.showToast
import com.lagradost.cloudstream3.mvvm.Resource import com.lagradost.cloudstream3.mvvm.Resource
import com.lagradost.cloudstream3.mvvm.logError import com.lagradost.cloudstream3.mvvm.logError
@ -26,11 +27,13 @@ import com.lagradost.cloudstream3.ui.result.ResultFragment
import com.lagradost.cloudstream3.ui.settings.SettingsFragment.Companion.isTvSettings import com.lagradost.cloudstream3.ui.settings.SettingsFragment.Companion.isTvSettings
import com.lagradost.cloudstream3.ui.subtitles.SubtitlesFragment import com.lagradost.cloudstream3.ui.subtitles.SubtitlesFragment
import com.lagradost.cloudstream3.utils.* import com.lagradost.cloudstream3.utils.*
import com.lagradost.cloudstream3.utils.Coroutines.ioSafe
import com.lagradost.cloudstream3.utils.UIHelper.dismissSafe import com.lagradost.cloudstream3.utils.UIHelper.dismissSafe
import com.lagradost.cloudstream3.utils.UIHelper.hideSystemUI import com.lagradost.cloudstream3.utils.UIHelper.hideSystemUI
import com.lagradost.cloudstream3.utils.UIHelper.popCurrentPage import com.lagradost.cloudstream3.utils.UIHelper.popCurrentPage
import kotlinx.android.synthetic.main.fragment_player.* import kotlinx.android.synthetic.main.fragment_player.*
import kotlinx.android.synthetic.main.player_custom_layout.* import kotlinx.android.synthetic.main.player_custom_layout.*
import kotlinx.coroutines.Job
class GeneratorPlayer : FullScreenPlayer() { class GeneratorPlayer : FullScreenPlayer() {
companion object { companion object {
@ -80,6 +83,19 @@ class GeneratorPlayer : FullScreenPlayer() {
return durPos.position return durPos.position
} }
var currentVerifyLink: Job? = null
private fun loadExtractorJob(extractorLink: ExtractorLink?) {
currentVerifyLink?.cancel()
extractorLink?.let {
currentVerifyLink = ioSafe {
if (it.extractorData != null) {
getApiFromNameNull(it.source)?.extractorVerifierJob(it.extractorData)
}
}
}
}
private fun loadLink(link: Pair<ExtractorLink?, ExtractorUri?>?, sameEpisode: Boolean) { private fun loadLink(link: Pair<ExtractorLink?, ExtractorUri?>?, sameEpisode: Boolean) {
if (link == null) return if (link == null) return
@ -93,6 +109,7 @@ class GeneratorPlayer : FullScreenPlayer() {
setPlayerDimen(null) setPlayerDimen(null)
setTitle() setTitle()
loadExtractorJob(link.first)
// load player // load player
context?.let { ctx -> context?.let { ctx ->
val (url, uri) = link val (url, uri) = link
@ -345,6 +362,7 @@ class GeneratorPlayer : FullScreenPlayer() {
override fun onDestroy() { override fun onDestroy() {
ResultFragment.updateUI() ResultFragment.updateUI()
currentVerifyLink?.cancel()
super.onDestroy() super.onDestroy()
} }

View file

@ -16,7 +16,9 @@ data class ExtractorLink(
override val referer: String, override val referer: String,
val quality: Int, val quality: Int,
val isM3u8: Boolean = false, val isM3u8: Boolean = false,
override val headers: Map<String, String> = mapOf() override val headers: Map<String, String> = mapOf(),
/** Used for getExtractorVerifierJob() */
val extractorData: String? = null
) : VideoDownloadManager.IDownloadableMinimum ) : VideoDownloadManager.IDownloadableMinimum
data class ExtractorUri( data class ExtractorUri(