mirror of
https://github.com/recloudstream/cloudstream.git
synced 2024-08-15 01:53:11 +00:00
Cracked Sflix anti-scraping
This commit is contained in:
Blatzar
parent
e04e6e686e
commit
9e26139f82
7 changed files with 202 additions and 33 deletions
|
|
@ -2,6 +2,7 @@ package com.lagradost.cloudstream3
|
|||
|
||||
import android.annotation.SuppressLint
|
||||
import android.content.Context
|
||||
import androidx.annotation.WorkerThread
|
||||
import androidx.preference.PreferenceManager
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature
|
||||
import com.fasterxml.jackson.databind.json.JsonMapper
|
||||
|
|
@ -245,23 +246,43 @@ abstract class MainAPI {
|
|||
open val vpnStatus = VPNStatus.None
|
||||
open val providerType = ProviderType.DirectProvider
|
||||
|
||||
@WorkerThread
|
||||
suspend open fun getMainPage(): HomePageResponse? {
|
||||
throw NotImplementedError()
|
||||
}
|
||||
|
||||
@WorkerThread
|
||||
suspend open fun search(query: String): List<SearchResponse>? {
|
||||
throw NotImplementedError()
|
||||
}
|
||||
|
||||
@WorkerThread
|
||||
suspend open fun quickSearch(query: String): List<SearchResponse>? {
|
||||
throw NotImplementedError()
|
||||
}
|
||||
|
||||
@WorkerThread
|
||||
/**
|
||||
* Based on data from search() or getMainPage() it generates a LoadResponse,
|
||||
* basically opening the info page from a link.
|
||||
* */
|
||||
suspend open fun load(url: String): LoadResponse? {
|
||||
throw NotImplementedError()
|
||||
}
|
||||
|
||||
/**
|
||||
* Largely redundant feature for most providers.
|
||||
*
|
||||
* This job runs in the background when a link is playing in exoplayer.
|
||||
* First implemented to do polling for sflix to keep the link from getting expired.
|
||||
*
|
||||
* This function might be updated to include exoplayer timestamps etc in the future
|
||||
* if the need arises.
|
||||
* */
|
||||
@WorkerThread
|
||||
suspend open fun extractorVerifierJob(extractorData: String?) {
|
||||
throw NotImplementedError()
|
||||
}
|
||||
|
||||
/**Callback is fired once a link is found, will return true if method is executed successfully*/
|
||||
@WorkerThread
|
||||
suspend open fun loadLinks(
|
||||
data: String,
|
||||
isCasting: Boolean,
|
||||
|
|
|
|||
|
|
@ -359,7 +359,7 @@ class ZoroProvider : MainAPI() {
|
|||
|
||||
list.forEach { subList ->
|
||||
subList.first?.forEach { a ->
|
||||
a?.toExtractorLink(this, subList.second + " - ${it.first}")
|
||||
a?.toExtractorLink(this, subList.second + " - ${it.first}", null)
|
||||
?.forEach(callback)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +1,26 @@
|
|||
package com.lagradost.cloudstream3.movieproviders
|
||||
|
||||
import android.util.Log
|
||||
import com.fasterxml.jackson.annotation.JsonProperty
|
||||
import com.lagradost.cloudstream3.*
|
||||
import com.lagradost.cloudstream3.APIHolder.unixTimeMS
|
||||
import com.lagradost.cloudstream3.LoadResponse.Companion.addActors
|
||||
import com.lagradost.cloudstream3.LoadResponse.Companion.setDuration
|
||||
import com.lagradost.cloudstream3.mvvm.suspendSafeApiCall
|
||||
import com.lagradost.cloudstream3.network.WebViewResolver
|
||||
import com.lagradost.cloudstream3.network.getRequestCreator
|
||||
import com.lagradost.cloudstream3.network.text
|
||||
import com.lagradost.cloudstream3.utils.AppUtils.parseJson
|
||||
import com.lagradost.cloudstream3.utils.AppUtils.toJson
|
||||
import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
|
||||
import com.lagradost.cloudstream3.utils.ExtractorLink
|
||||
import com.lagradost.cloudstream3.utils.M3u8Helper
|
||||
import com.lagradost.cloudstream3.utils.getQualityFromName
|
||||
import kotlinx.coroutines.delay
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Element
|
||||
import java.net.URI
|
||||
import kotlin.system.measureTimeMillis
|
||||
|
||||
class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
|
||||
override val mainUrl = providerUrl
|
||||
|
|
@ -292,12 +298,19 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
|
|||
|
||||
urls?.apmap { url ->
|
||||
suspendSafeApiCall {
|
||||
val sources = app.get(
|
||||
url,
|
||||
interceptor = WebViewResolver(
|
||||
val resolved = WebViewResolver(
|
||||
Regex("""/getSources"""),
|
||||
)
|
||||
).text
|
||||
// This is unreliable, generating my own link instead
|
||||
// additionalUrls = listOf(Regex("""^.*transport=polling(?!.*sid=).*$"""))
|
||||
).resolveUsingWebView(getRequestCreator(url))
|
||||
// val extractorData = resolved.second.getOrNull(0)?.url?.toString()
|
||||
|
||||
// Some smarter ws11 or w10 selection might be required in the future.
|
||||
val extractorData =
|
||||
"https://ws10.rabbitstream.net/socket.io/?EIO=4&transport=polling"
|
||||
|
||||
val sources = resolved.first?.let { app.baseClient.newCall(it).execute().text }
|
||||
?: return@suspendSafeApiCall
|
||||
|
||||
val mapped = parseJson<SourceObject>(sources)
|
||||
|
||||
|
|
@ -314,7 +327,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
|
|||
mapped.sourcesBackup to "source backup"
|
||||
).forEach { (sources, sourceName) ->
|
||||
sources?.forEach {
|
||||
it?.toExtractorLink(this, sourceName)?.forEach(callback)
|
||||
it?.toExtractorLink(this, sourceName, extractorData)?.forEach(callback)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -323,6 +336,105 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
|
|||
return !urls.isNullOrEmpty()
|
||||
}
|
||||
|
||||
data class PollingData(
|
||||
@JsonProperty("sid") val sid: String? = null,
|
||||
@JsonProperty("upgrades") val upgrades: ArrayList<String> = arrayListOf(),
|
||||
@JsonProperty("pingInterval") val pingInterval: Int? = null,
|
||||
@JsonProperty("pingTimeout") val pingTimeout: Int? = null
|
||||
)
|
||||
|
||||
/*
|
||||
# python code to figure out the time offset based on code if necessary
|
||||
chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"
|
||||
code = "Nxa_-bM"
|
||||
total = 0
|
||||
for i, char in enumerate(code[::-1]):
|
||||
index = chars.index(char)
|
||||
value = index * 64**i
|
||||
total += value
|
||||
print(f"total {total}")
|
||||
*/
|
||||
private fun generateTimeStamp(): String {
|
||||
val chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"
|
||||
var code = ""
|
||||
var time = unixTimeMS
|
||||
while (time > 0) {
|
||||
code += chars[(time % (chars.length)).toInt()]
|
||||
time /= chars.length
|
||||
}
|
||||
return code.reversed()
|
||||
}
|
||||
|
||||
override suspend fun extractorVerifierJob(extractorData: String?) {
|
||||
if (extractorData == null) return
|
||||
|
||||
val jsonText =
|
||||
app.get("$extractorData&t=${generateTimeStamp()}").text.replaceBefore("{", "")
|
||||
val data = parseJson<PollingData>(jsonText)
|
||||
val headers = mapOf(
|
||||
"User-Agent" to USER_AGENT,
|
||||
"Referer" to "https://rabbitstream.net/"
|
||||
)
|
||||
|
||||
// 40 is hardcoded, dunno how it's generated, but it seems to work everywhere.
|
||||
// This request is obligatory
|
||||
app.post(
|
||||
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
|
||||
data = 40, headers = headers
|
||||
)//.also { println("First post ${it.text}") }
|
||||
// This makes the second get request work, and re-connect work.
|
||||
val reconnectSid =
|
||||
parseJson<PollingData>(
|
||||
app.get(
|
||||
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
|
||||
headers = headers
|
||||
)
|
||||
//.also { println("First get ${it.text}") }
|
||||
.text.replaceBefore("{", "")
|
||||
).sid
|
||||
// This response is used in the post requests. Same contents in all it seems.
|
||||
val authInt =
|
||||
app.get(
|
||||
"$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
|
||||
timeout = 60,
|
||||
headers = headers
|
||||
).text
|
||||
//.also { println("Second get ${it}") }
|
||||
// Dunno if it's actually generated like this, just guessing.
|
||||
.toIntOrNull()?.plus(1) ?: 3
|
||||
|
||||
// Prevents them from fucking us over with doing a while(true){} loop
|
||||
val interval = maxOf(data.pingInterval?.toLong()?.plus(2000) ?: return, 10000L)
|
||||
var reconnect = false
|
||||
// New SID can be negotiated as above, but not implemented yet as it seems rare.
|
||||
while (true) {
|
||||
val authData = if (reconnect) """
|
||||
42["_reconnect", "$reconnectSid"]
|
||||
""".trimIndent() else authInt
|
||||
|
||||
val url = "${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}"
|
||||
app.post(url, data = authData, headers = headers)
|
||||
//.also { println("Sflix post job ${it.text}") }
|
||||
Log.d(this.name, "Running Sflix job $url")
|
||||
|
||||
val time = measureTimeMillis {
|
||||
// This acts as a timeout
|
||||
val getResponse = app.get(
|
||||
"${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}",
|
||||
timeout = 60,
|
||||
headers = headers
|
||||
).text //.also { println("Sflix get job $it") }
|
||||
if (getResponse.contains("sid")) {
|
||||
reconnect = true
|
||||
// println("Reconnecting")
|
||||
}
|
||||
}
|
||||
// Always waits even if the get response is instant, to prevent a while true loop.
|
||||
if (time < interval - 4000)
|
||||
delay(interval)
|
||||
}
|
||||
}
|
||||
|
||||
private fun Element.toSearchResult(): SearchResponse {
|
||||
val img = this.select("img")
|
||||
val title = img.attr("title")
|
||||
|
|
@ -363,7 +475,11 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
|
|||
}
|
||||
|
||||
// For re-use in Zoro
|
||||
fun Sources.toExtractorLink(caller: MainAPI, name: String): List<ExtractorLink>? {
|
||||
fun Sources.toExtractorLink(
|
||||
caller: MainAPI,
|
||||
name: String,
|
||||
extractorData: String? = null
|
||||
): List<ExtractorLink>? {
|
||||
return this.file?.let { file ->
|
||||
//println("FILE::: $file")
|
||||
val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals(
|
||||
|
|
@ -382,7 +498,8 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
|
|||
stream.streamUrl,
|
||||
caller.mainUrl,
|
||||
getQualityFromName(stream.quality.toString()),
|
||||
true
|
||||
true,
|
||||
extractorData = extractorData
|
||||
)
|
||||
}
|
||||
} else {
|
||||
|
|
@ -393,6 +510,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
|
|||
caller.mainUrl,
|
||||
getQualityFromName(this.type ?: ""),
|
||||
false,
|
||||
extractorData = extractorData
|
||||
))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,13 +5,18 @@ import android.content.Context
|
|||
import android.util.Log
|
||||
import androidx.preference.PreferenceManager
|
||||
import com.fasterxml.jackson.module.kotlin.readValue
|
||||
import com.lagradost.cloudstream3.*
|
||||
import com.lagradost.cloudstream3.R
|
||||
import com.lagradost.cloudstream3.USER_AGENT
|
||||
import com.lagradost.cloudstream3.app
|
||||
import com.lagradost.cloudstream3.mapper
|
||||
import kotlinx.coroutines.CancellableContinuation
|
||||
import kotlinx.coroutines.CompletionHandler
|
||||
import kotlinx.coroutines.ExperimentalCoroutinesApi
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import okhttp3.*
|
||||
import okhttp3.Headers.Companion.toHeaders
|
||||
import okhttp3.MediaType.Companion.toMediaTypeOrNull
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import java.io.File
|
||||
|
|
@ -107,14 +112,20 @@ class AppResponse(
|
|||
}
|
||||
}
|
||||
|
||||
private fun getData(data: Map<String, String?>): RequestBody {
|
||||
private fun getData(data: Any?): RequestBody {
|
||||
return when (data) {
|
||||
null -> FormBody.Builder().build()
|
||||
is Map<*, *> -> {
|
||||
val builder = FormBody.Builder()
|
||||
data.forEach {
|
||||
it.value?.let { value ->
|
||||
builder.add(it.key, value)
|
||||
if (it.key is String && it.value is String)
|
||||
builder.add(it.key as String, it.value as String)
|
||||
}
|
||||
builder.build()
|
||||
}
|
||||
else ->
|
||||
data.toString().toRequestBody("text/plain;charset=UTF-8".toMediaTypeOrNull())
|
||||
}
|
||||
return builder.build()
|
||||
}
|
||||
|
||||
// https://github.com, id=test -> https://github.com?id=test
|
||||
|
|
@ -169,7 +180,7 @@ fun postRequestCreator(
|
|||
referer: String? = null,
|
||||
params: Map<String, String> = emptyMap(),
|
||||
cookies: Map<String, String> = emptyMap(),
|
||||
data: Map<String, String> = emptyMap(),
|
||||
data: Any? = DEFAULT_DATA,
|
||||
cacheTime: Int = DEFAULT_TIME,
|
||||
cacheUnit: TimeUnit = DEFAULT_TIME_UNIT
|
||||
): Request {
|
||||
|
|
@ -340,7 +351,7 @@ open class Requests {
|
|||
referer: String? = null,
|
||||
params: Map<String, String> = mapOf(),
|
||||
cookies: Map<String, String> = mapOf(),
|
||||
data: Map<String, String> = DEFAULT_DATA,
|
||||
data: Any? = DEFAULT_DATA,
|
||||
allowRedirects: Boolean = true,
|
||||
cacheTime: Int = DEFAULT_TIME,
|
||||
cacheUnit: TimeUnit = DEFAULT_TIME_UNIT,
|
||||
|
|
|
|||
|
|
@ -32,13 +32,13 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
|
|||
}
|
||||
|
||||
/**
|
||||
* @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls.
|
||||
* @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls. If true, destroy WebView.
|
||||
* @return the final request (by interceptUrl) and all the collected urls (by additionalUrls).
|
||||
* */
|
||||
@SuppressLint("SetJavaScriptEnabled")
|
||||
suspend fun resolveUsingWebView(
|
||||
request: Request,
|
||||
requestCallBack: (Request) -> Unit = {}
|
||||
requestCallBack: (Request) -> Boolean = { false }
|
||||
): Pair<Request?, List<Request>> {
|
||||
val url = request.url.toString()
|
||||
val headers = request.headers
|
||||
|
|
@ -81,14 +81,18 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
|
|||
// println("Loading WebView URL: $webViewUrl")
|
||||
|
||||
if (interceptUrl.containsMatchIn(webViewUrl)) {
|
||||
fixedRequest = request.toRequest().also(requestCallBack)
|
||||
fixedRequest = request.toRequest().also {
|
||||
if (requestCallBack(it)) destroyWebView()
|
||||
}
|
||||
println("Web-view request finished: $webViewUrl")
|
||||
destroyWebView()
|
||||
return@runBlocking null
|
||||
}
|
||||
|
||||
if (additionalUrls.any { it.containsMatchIn(webViewUrl) }) {
|
||||
extraRequestList.add(request.toRequest().also(requestCallBack))
|
||||
extraRequestList.add(request.toRequest().also {
|
||||
if (requestCallBack(it)) destroyWebView()
|
||||
})
|
||||
}
|
||||
|
||||
// Suppress image requests as we don't display them anywhere
|
||||
|
|
@ -129,11 +133,6 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
|
|||
* e.g the recaptcha request.
|
||||
* **/
|
||||
|
||||
/** NOTE! request.requestHeaders is not perfect!
|
||||
* They don't contain all the headers the browser actually gives.
|
||||
* Overriding with okhttp might fuck up otherwise working requests,
|
||||
* e.g the recaptcha request.
|
||||
* **/
|
||||
return@runBlocking try {
|
||||
when {
|
||||
blacklistedFiles.any { URI(webViewUrl).path.contains(it) } || webViewUrl.endsWith(
|
||||
|
|
@ -204,7 +203,7 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
|
|||
null,
|
||||
emptyMap(),
|
||||
emptyMap(),
|
||||
emptyMap(),
|
||||
emptyMap<String, String>(),
|
||||
10,
|
||||
TimeUnit.MINUTES
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import androidx.lifecycle.ViewModelProvider
|
|||
import com.google.android.material.button.MaterialButton
|
||||
import com.hippo.unifile.UniFile
|
||||
import com.lagradost.cloudstream3.*
|
||||
import com.lagradost.cloudstream3.APIHolder.getApiFromNameNull
|
||||
import com.lagradost.cloudstream3.CommonActivity.showToast
|
||||
import com.lagradost.cloudstream3.mvvm.Resource
|
||||
import com.lagradost.cloudstream3.mvvm.logError
|
||||
|
|
@ -26,11 +27,13 @@ import com.lagradost.cloudstream3.ui.result.ResultFragment
|
|||
import com.lagradost.cloudstream3.ui.settings.SettingsFragment.Companion.isTvSettings
|
||||
import com.lagradost.cloudstream3.ui.subtitles.SubtitlesFragment
|
||||
import com.lagradost.cloudstream3.utils.*
|
||||
import com.lagradost.cloudstream3.utils.Coroutines.ioSafe
|
||||
import com.lagradost.cloudstream3.utils.UIHelper.dismissSafe
|
||||
import com.lagradost.cloudstream3.utils.UIHelper.hideSystemUI
|
||||
import com.lagradost.cloudstream3.utils.UIHelper.popCurrentPage
|
||||
import kotlinx.android.synthetic.main.fragment_player.*
|
||||
import kotlinx.android.synthetic.main.player_custom_layout.*
|
||||
import kotlinx.coroutines.Job
|
||||
|
||||
class GeneratorPlayer : FullScreenPlayer() {
|
||||
companion object {
|
||||
|
|
@ -80,6 +83,19 @@ class GeneratorPlayer : FullScreenPlayer() {
|
|||
return durPos.position
|
||||
}
|
||||
|
||||
var currentVerifyLink: Job? = null
|
||||
|
||||
private fun loadExtractorJob(extractorLink: ExtractorLink?) {
|
||||
currentVerifyLink?.cancel()
|
||||
extractorLink?.let {
|
||||
currentVerifyLink = ioSafe {
|
||||
if (it.extractorData != null) {
|
||||
getApiFromNameNull(it.source)?.extractorVerifierJob(it.extractorData)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun loadLink(link: Pair<ExtractorLink?, ExtractorUri?>?, sameEpisode: Boolean) {
|
||||
if (link == null) return
|
||||
|
||||
|
|
@ -93,6 +109,7 @@ class GeneratorPlayer : FullScreenPlayer() {
|
|||
setPlayerDimen(null)
|
||||
setTitle()
|
||||
|
||||
loadExtractorJob(link.first)
|
||||
// load player
|
||||
context?.let { ctx ->
|
||||
val (url, uri) = link
|
||||
|
|
@ -345,6 +362,7 @@ class GeneratorPlayer : FullScreenPlayer() {
|
|||
|
||||
override fun onDestroy() {
|
||||
ResultFragment.updateUI()
|
||||
currentVerifyLink?.cancel()
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ data class ExtractorLink(
|
|||
override val referer: String,
|
||||
val quality: Int,
|
||||
val isM3u8: Boolean = false,
|
||||
override val headers: Map<String, String> = mapOf()
|
||||
override val headers: Map<String, String> = mapOf(),
|
||||
/** Used for getExtractorVerifierJob() */
|
||||
val extractorData: String? = null
|
||||
) : VideoDownloadManager.IDownloadableMinimum
|
||||
|
||||
data class ExtractorUri(
|
||||
|
|
|
|||
Loading…
Reference in a new issue