recloudstream
/
cloudstream
mirror of https://github.com/recloudstream/cloudstream.git
3
3
Fork 1
Code Issues Releases Wiki Activity

Cracked Sflix anti-scraping

This commit is contained in:
Blatzar 2022-02-11 10:17:04 +01:00
parent e04e6e686e
commit 9e26139f82
7 changed files with 202 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt
View File

@ -2,6 +2,7 @@ package com.lagradost.cloudstream3
import android.annotation.SuppressLint
import android.content.Context
import androidx.annotation.WorkerThread
import androidx.preference.PreferenceManager
import com.fasterxml.jackson.databind.DeserializationFeature
import com.fasterxml.jackson.databind.json.JsonMapper
@ -245,23 +246,43 @@ abstract class MainAPI {
open val vpnStatus = VPNStatus.None
open val providerType = ProviderType.DirectProvider
@WorkerThread
suspend open fun getMainPage(): HomePageResponse? {
throw NotImplementedError()
}
@WorkerThread
suspend open fun search(query: String): List<SearchResponse>? {
throw NotImplementedError()
}
@WorkerThread
suspend open fun quickSearch(query: String): List<SearchResponse>? {
throw NotImplementedError()
}
@WorkerThread
/**
* Based on data from search() or getMainPage() it generates a LoadResponse,
* basically opening the info page from a link.
* */
suspend open fun load(url: String): LoadResponse? {
throw NotImplementedError()
}
/**
* Largely redundant feature for most providers.
*
* This job runs in the background when a link is playing in exoplayer.
* First implemented to do polling for sflix to keep the link from getting expired.
*
* This function might be updated to include exoplayer timestamps etc in the future
* if the need arises.
* */
@WorkerThread
suspend open fun extractorVerifierJob(extractorData: String?) {
throw NotImplementedError()
}
/**Callback is fired once a link is found, will return true if method is executed successfully*/
@WorkerThread
suspend open fun loadLinks(
data: String,
isCasting: Boolean,

2
app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt
View File

@ -359,7 +359,7 @@ class ZoroProvider : MainAPI() {
list.forEach { subList ->
subList.first?.forEach { a ->
a?.toExtractorLink(this, subList.second + " - ${it.first}")
a?.toExtractorLink(this, subList.second + " - ${it.first}", null)
?.forEach(callback)
}
}

136
app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt
View File

@ -1,20 +1,26 @@
package com.lagradost.cloudstream3.movieproviders
import android.util.Log
import com.fasterxml.jackson.annotation.JsonProperty
import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.APIHolder.unixTimeMS
import com.lagradost.cloudstream3.LoadResponse.Companion.addActors
import com.lagradost.cloudstream3.LoadResponse.Companion.setDuration
import com.lagradost.cloudstream3.mvvm.suspendSafeApiCall
import com.lagradost.cloudstream3.network.WebViewResolver
import com.lagradost.cloudstream3.network.getRequestCreator
import com.lagradost.cloudstream3.network.text
import com.lagradost.cloudstream3.utils.AppUtils.parseJson
import com.lagradost.cloudstream3.utils.AppUtils.toJson
import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
import com.lagradost.cloudstream3.utils.ExtractorLink
import com.lagradost.cloudstream3.utils.M3u8Helper
import com.lagradost.cloudstream3.utils.getQualityFromName
import kotlinx.coroutines.delay
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import java.net.URI
import kotlin.system.measureTimeMillis
class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
override val mainUrl = providerUrl
@ -292,12 +298,19 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
urls?.apmap { url ->
suspendSafeApiCall {
val sources = app.get(
url,
interceptor = WebViewResolver(
Regex("""/getSources"""),
)
).text
val resolved = WebViewResolver(
Regex("""/getSources"""),
// This is unreliable, generating my own link instead
// additionalUrls = listOf(Regex("""^.*transport=polling(?!.*sid=).*$"""))
).resolveUsingWebView(getRequestCreator(url))
// val extractorData = resolved.second.getOrNull(0)?.url?.toString()
// Some smarter ws11 or w10 selection might be required in the future.
val extractorData =
"https://ws10.rabbitstream.net/socket.io/?EIO=4&transport=polling"
val sources = resolved.first?.let { app.baseClient.newCall(it).execute().text }
?: return@suspendSafeApiCall
val mapped = parseJson<SourceObject>(sources)
@ -314,7 +327,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
mapped.sourcesBackup to "source backup"
).forEach { (sources, sourceName) ->
sources?.forEach {
it?.toExtractorLink(this, sourceName)?.forEach(callback)
it?.toExtractorLink(this, sourceName, extractorData)?.forEach(callback)
}
}
}
@ -323,6 +336,105 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
return !urls.isNullOrEmpty()
}
data class PollingData(
@JsonProperty("sid") val sid: String? = null,
@JsonProperty("upgrades") val upgrades: ArrayList<String> = arrayListOf(),
@JsonProperty("pingInterval") val pingInterval: Int? = null,
@JsonProperty("pingTimeout") val pingTimeout: Int? = null
)
/*
# python code to figure out the time offset based on code if necessary
chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"
code = "Nxa_-bM"
total = 0
for i, char in enumerate(code[::-1]):
index = chars.index(char)
value = index * 64**i
total += value
print(f"total {total}")
*/
private fun generateTimeStamp(): String {
val chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"
var code = ""
var time = unixTimeMS
while (time > 0) {
code += chars[(time % (chars.length)).toInt()]
time /= chars.length
}
return code.reversed()
}
override suspend fun extractorVerifierJob(extractorData: String?) {
if (extractorData == null) return
val jsonText =
app.get("$extractorData&t=${generateTimeStamp()}").text.replaceBefore("{", "")
val data = parseJson<PollingData>(jsonText)
val headers = mapOf(
"User-Agent" to USER_AGENT,
"Referer" to "https://rabbitstream.net/"
)
// 40 is hardcoded, dunno how it's generated, but it seems to work everywhere.
// This request is obligatory
app.post(
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
data = 40, headers = headers
)//.also { println("First post ${it.text}") }
// This makes the second get request work, and re-connect work.
val reconnectSid =
parseJson<PollingData>(
app.get(
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
headers = headers
)
//.also { println("First get ${it.text}") }
.text.replaceBefore("{", "")
).sid
// This response is used in the post requests. Same contents in all it seems.
val authInt =
app.get(
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
timeout = 60,
headers = headers
).text
//.also { println("Second get ${it}") }
// Dunno if it's actually generated like this, just guessing.
.toIntOrNull()?.plus(1) ?: 3
// Prevents them from fucking us over with doing a while(true){} loop
val interval = maxOf(data.pingInterval?.toLong()?.plus(2000) ?: return, 10000L)
var reconnect = false
// New SID can be negotiated as above, but not implemented yet as it seems rare.
while (true) {
val authData = if (reconnect) """
42["_reconnect", "$reconnectSid"]
""".trimIndent() else authInt
val url = "${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}"
app.post(url, data = authData, headers = headers)
//.also { println("Sflix post job ${it.text}") }
Log.d(this.name, "Running Sflix job $url")
val time = measureTimeMillis {
// This acts as a timeout
val getResponse = app.get(
"${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}",
timeout = 60,
headers = headers
).text //.also { println("Sflix get job $it") }
if (getResponse.contains("sid")) {
reconnect = true
// println("Reconnecting")
}
}
// Always waits even if the get response is instant, to prevent a while true loop.
if (time < interval - 4000)
delay(interval)
}
}
private fun Element.toSearchResult(): SearchResponse {
val img = this.select("img")
val title = img.attr("title")
@ -363,7 +475,11 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
}
// For re-use in Zoro
fun Sources.toExtractorLink(caller: MainAPI, name: String): List<ExtractorLink>? {
fun Sources.toExtractorLink(
caller: MainAPI,
name: String,
extractorData: String? = null
): List<ExtractorLink>? {
return this.file?.let { file ->
//println("FILE::: $file")
val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals(
@ -382,7 +498,8 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
stream.streamUrl,
caller.mainUrl,
getQualityFromName(stream.quality.toString()),
true
true,
extractorData = extractorData
)
}
} else {
@ -393,6 +510,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
caller.mainUrl,
getQualityFromName(this.type ?: ""),
false,
extractorData = extractorData
))
}
}

29
app/src/main/java/com/lagradost/cloudstream3/network/Requests.kt
View File

@ -5,13 +5,18 @@ import android.content.Context
import android.util.Log
import androidx.preference.PreferenceManager
import com.fasterxml.jackson.module.kotlin.readValue
import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.R
import com.lagradost.cloudstream3.USER_AGENT
import com.lagradost.cloudstream3.app
import com.lagradost.cloudstream3.mapper
import kotlinx.coroutines.CancellableContinuation
import kotlinx.coroutines.CompletionHandler
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.suspendCancellableCoroutine
import okhttp3.*
import okhttp3.Headers.Companion.toHeaders
import okhttp3.MediaType.Companion.toMediaTypeOrNull
import okhttp3.RequestBody.Companion.toRequestBody
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import java.io.File
@ -107,14 +112,20 @@ class AppResponse(
}
}
private fun getData(data: Map<String, String?>): RequestBody {
val builder = FormBody.Builder()
data.forEach {
it.value?.let { value ->
builder.add(it.key, value)
private fun getData(data: Any?): RequestBody {
return when (data) {
null -> FormBody.Builder().build()
is Map<*, *> -> {
val builder = FormBody.Builder()
data.forEach {
if (it.key is String && it.value is String)
builder.add(it.key as String, it.value as String)
}
builder.build()
}
else ->
data.toString().toRequestBody("text/plain;charset=UTF-8".toMediaTypeOrNull())
}
return builder.build()
}
// https://github.com, id=test -> https://github.com?id=test
@ -169,7 +180,7 @@ fun postRequestCreator(
referer: String? = null,
params: Map<String, String> = emptyMap(),
cookies: Map<String, String> = emptyMap(),
data: Map<String, String> = emptyMap(),
data: Any? = DEFAULT_DATA,
cacheTime: Int = DEFAULT_TIME,
cacheUnit: TimeUnit = DEFAULT_TIME_UNIT
): Request {
@ -340,7 +351,7 @@ open class Requests {
referer: String? = null,
params: Map<String, String> = mapOf(),
cookies: Map<String, String> = mapOf(),
data: Map<String, String> = DEFAULT_DATA,
data: Any? = DEFAULT_DATA,
allowRedirects: Boolean = true,
cacheTime: Int = DEFAULT_TIME,
cacheUnit: TimeUnit = DEFAULT_TIME_UNIT,

19
app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt
View File

@ -32,13 +32,13 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
}
/**
* @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls.
* @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls. If true, destroy WebView.
* @return the final request (by interceptUrl) and all the collected urls (by additionalUrls).
* */
@SuppressLint("SetJavaScriptEnabled")
suspend fun resolveUsingWebView(
request: Request,
requestCallBack: (Request) -> Unit = {}
requestCallBack: (Request) -> Boolean = { false }
): Pair<Request?, List<Request>> {
val url = request.url.toString()
val headers = request.headers
@ -81,14 +81,18 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
// println("Loading WebView URL: $webViewUrl")
if (interceptUrl.containsMatchIn(webViewUrl)) {
fixedRequest = request.toRequest().also(requestCallBack)
fixedRequest = request.toRequest().also {
if (requestCallBack(it)) destroyWebView()
}
println("Web-view request finished: $webViewUrl")
destroyWebView()
return@runBlocking null
}
if (additionalUrls.any { it.containsMatchIn(webViewUrl) }) {
extraRequestList.add(request.toRequest().also(requestCallBack))
extraRequestList.add(request.toRequest().also {
if (requestCallBack(it)) destroyWebView()
})
}
// Suppress image requests as we don't display them anywhere
@ -129,11 +133,6 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
* e.g the recaptcha request.
* **/
/** NOTE! request.requestHeaders is not perfect!
* They don't contain all the headers the browser actually gives.
* Overriding with okhttp might fuck up otherwise working requests,
* e.g the recaptcha request.
* **/
return@runBlocking try {
when {
blacklistedFiles.any { URI(webViewUrl).path.contains(it) } || webViewUrl.endsWith(
@ -204,7 +203,7 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
null,
emptyMap(),
emptyMap(),
emptyMap(),
emptyMap<String, String>(),
10,
TimeUnit.MINUTES
)

18
app/src/main/java/com/lagradost/cloudstream3/ui/player/GeneratorPlayer.kt
View File

@ -15,6 +15,7 @@ import androidx.lifecycle.ViewModelProvider
import com.google.android.material.button.MaterialButton
import com.hippo.unifile.UniFile
import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.APIHolder.getApiFromNameNull
import com.lagradost.cloudstream3.CommonActivity.showToast
import com.lagradost.cloudstream3.mvvm.Resource
import com.lagradost.cloudstream3.mvvm.logError
@ -26,11 +27,13 @@ import com.lagradost.cloudstream3.ui.result.ResultFragment
import com.lagradost.cloudstream3.ui.settings.SettingsFragment.Companion.isTvSettings
import com.lagradost.cloudstream3.ui.subtitles.SubtitlesFragment
import com.lagradost.cloudstream3.utils.*
import com.lagradost.cloudstream3.utils.Coroutines.ioSafe
import com.lagradost.cloudstream3.utils.UIHelper.dismissSafe
import com.lagradost.cloudstream3.utils.UIHelper.hideSystemUI
import com.lagradost.cloudstream3.utils.UIHelper.popCurrentPage
import kotlinx.android.synthetic.main.fragment_player.*
import kotlinx.android.synthetic.main.player_custom_layout.*
import kotlinx.coroutines.Job
class GeneratorPlayer : FullScreenPlayer() {
companion object {
@ -80,6 +83,19 @@ class GeneratorPlayer : FullScreenPlayer() {
return durPos.position
}
var currentVerifyLink: Job? = null
private fun loadExtractorJob(extractorLink: ExtractorLink?) {
currentVerifyLink?.cancel()
extractorLink?.let {
currentVerifyLink = ioSafe {
if (it.extractorData != null) {
getApiFromNameNull(it.source)?.extractorVerifierJob(it.extractorData)
}
}
}
}
private fun loadLink(link: Pair<ExtractorLink?, ExtractorUri?>?, sameEpisode: Boolean) {
if (link == null) return
@ -93,6 +109,7 @@ class GeneratorPlayer : FullScreenPlayer() {
setPlayerDimen(null)
setTitle()
loadExtractorJob(link.first)
// load player
context?.let { ctx ->
val (url, uri) = link
@ -345,6 +362,7 @@ class GeneratorPlayer : FullScreenPlayer() {
override fun onDestroy() {
ResultFragment.updateUI()
currentVerifyLink?.cancel()
super.onDestroy()
}

4
app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt
View File

@ -16,7 +16,9 @@ data class ExtractorLink(
override val referer: String,
val quality: Int,
val isM3u8: Boolean = false,
override val headers: Map<String, String> = mapOf()
override val headers: Map<String, String> = mapOf(),
/** Used for getExtractorVerifierJob() */
val extractorData: String? = null
) : VideoDownloadManager.IDownloadableMinimum
data class ExtractorUri(