Merge remote-tracking branch 'origin/master'

app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt

@@ -2,6 +2,7 @@ package com.lagradost.cloudstream3
 
 import android.annotation.SuppressLint
 import android.content.Context
+import androidx.annotation.WorkerThread
 import androidx.preference.PreferenceManager
 import com.fasterxml.jackson.databind.DeserializationFeature
 import com.fasterxml.jackson.databind.json.JsonMapper
@@ -246,23 +247,43 @@ abstract class MainAPI {
     open val vpnStatus = VPNStatus.None
     open val providerType = ProviderType.DirectProvider
 
+    @WorkerThread
     suspend open fun getMainPage(): HomePageResponse? {
         throw NotImplementedError()
     }
-
+    @WorkerThread
     suspend open fun search(query: String): List<SearchResponse>? {
         throw NotImplementedError()
     }
-
+    @WorkerThread
     suspend open fun quickSearch(query: String): List<SearchResponse>? {
         throw NotImplementedError()
     }
-
+    @WorkerThread
+    /**
+     * Based on data from search() or getMainPage() it generates a LoadResponse,
+     * basically opening the info page from a link.
+     * */
     suspend open fun load(url: String): LoadResponse? {
         throw NotImplementedError()
     }
 
+    /**
+     * Largely redundant feature for most providers.
+     *
+     * This job runs in the background when a link is playing in exoplayer.
+     * First implemented to do polling for sflix to keep the link from getting expired.
+     *
+     * This function might be updated to include exoplayer timestamps etc in the future
+     * if the need arises.
+     * */
+    @WorkerThread
+    suspend open fun extractorVerifierJob(extractorData: String?) {
+        throw NotImplementedError()
+    }
+
     /**Callback is fired once a link is found, will return true if method is executed successfully*/
+    @WorkerThread
     suspend open fun loadLinks(
         data: String,
         isCasting: Boolean,

app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt

@@ -359,7 +359,7 @@ class ZoroProvider : MainAPI() {
 
                 list.forEach { subList ->
                     subList.first?.forEach { a ->
-                        a?.toExtractorLink(this, subList.second + " - ${it.first}")
+                        a?.toExtractorLink(this, subList.second + " - ${it.first}", null)
                             ?.forEach(callback)
                     }
                 }

app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt

@@ -1,20 +1,26 @@
 package com.lagradost.cloudstream3.movieproviders
 
+import android.util.Log
 import com.fasterxml.jackson.annotation.JsonProperty
 import com.lagradost.cloudstream3.*
+import com.lagradost.cloudstream3.APIHolder.unixTimeMS
 import com.lagradost.cloudstream3.LoadResponse.Companion.addActors
 import com.lagradost.cloudstream3.LoadResponse.Companion.setDuration
 import com.lagradost.cloudstream3.mvvm.suspendSafeApiCall
 import com.lagradost.cloudstream3.network.WebViewResolver
+import com.lagradost.cloudstream3.network.getRequestCreator
+import com.lagradost.cloudstream3.network.text
 import com.lagradost.cloudstream3.utils.AppUtils.parseJson
 import com.lagradost.cloudstream3.utils.AppUtils.toJson
 import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
 import com.lagradost.cloudstream3.utils.ExtractorLink
 import com.lagradost.cloudstream3.utils.M3u8Helper
 import com.lagradost.cloudstream3.utils.getQualityFromName
+import kotlinx.coroutines.delay
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Element
 import java.net.URI
+import kotlin.system.measureTimeMillis
 
 class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
     override val mainUrl = providerUrl
@@ -292,12 +298,19 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
 
         urls?.apmap { url ->
             suspendSafeApiCall {
-                val sources = app.get(
+                val resolved = WebViewResolver(
-                    url,
+                    Regex("""/getSources"""),
-                    interceptor = WebViewResolver(
+                    // This is unreliable, generating my own link instead
-                        Regex("""/getSources"""),
+//                  additionalUrls = listOf(Regex("""^.*transport=polling(?!.*sid=).*$"""))
-                    )
+                ).resolveUsingWebView(getRequestCreator(url))
-                ).text
+//              val extractorData = resolved.second.getOrNull(0)?.url?.toString()
+
+                // Some smarter ws11 or w10 selection might be required in the future.
+                val extractorData =
+                    "https://ws10.rabbitstream.net/socket.io/?EIO=4&transport=polling"
+
+                val sources = resolved.first?.let { app.baseClient.newCall(it).execute().text }
+                    ?: return@suspendSafeApiCall
 
                 val mapped = parseJson<SourceObject>(sources)
 
@@ -314,7 +327,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
                     mapped.sourcesBackup to "source backup"
                 ).forEach { (sources, sourceName) ->
                     sources?.forEach {
-                        it?.toExtractorLink(this, sourceName)?.forEach(callback)
+                        it?.toExtractorLink(this, sourceName, extractorData)?.forEach(callback)
                     }
                 }
             }
@@ -323,6 +336,105 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
         return !urls.isNullOrEmpty()
     }
 
+    data class PollingData(
+        @JsonProperty("sid") val sid: String? = null,
+        @JsonProperty("upgrades") val upgrades: ArrayList<String> = arrayListOf(),
+        @JsonProperty("pingInterval") val pingInterval: Int? = null,
+        @JsonProperty("pingTimeout") val pingTimeout: Int? = null
+    )
+
+    /*
+    # python code to figure out the time offset based on code if necessary
+    chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"
+    code = "Nxa_-bM"
+    total = 0
+    for i, char in enumerate(code[::-1]):
+        index = chars.index(char)
+        value = index * 64**i
+        total += value
+    print(f"total {total}")
+    */
+    private fun generateTimeStamp(): String {
+        val chars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"
+        var code = ""
+        var time = unixTimeMS
+        while (time > 0) {
+            code += chars[(time % (chars.length)).toInt()]
+            time /= chars.length
+        }
+        return code.reversed()
+    }
+
+    override suspend fun extractorVerifierJob(extractorData: String?) {
+        if (extractorData == null) return
+
+        val jsonText =
+            app.get("$extractorData&t=${generateTimeStamp()}").text.replaceBefore("{", "")
+        val data = parseJson<PollingData>(jsonText)
+        val headers = mapOf(
+            "User-Agent" to USER_AGENT,
+            "Referer" to "https://rabbitstream.net/"
+        )
+
+        // 40 is hardcoded, dunno how it's generated, but it seems to work everywhere.
+        // This request is obligatory
+        app.post(
+            "$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
+            data = 40, headers = headers
+        )//.also { println("First post ${it.text}") }
+        // This makes the second get request work, and re-connect work.
+        val reconnectSid =
+            parseJson<PollingData>(
+                app.get(
+                    "$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
+                    headers = headers
+                )
+                    //.also { println("First get ${it.text}") }
+                    .text.replaceBefore("{", "")
+            ).sid
+        // This response is used in the post requests. Same contents in all it seems.
+        val authInt =
+            app.get(
+                "$extractorData&t=${generateTimeStamp()}&sid=${data.sid}",
+                timeout = 60,
+                headers = headers
+            ).text
+                //.also { println("Second get ${it}") }
+                // Dunno if it's actually generated like this, just guessing.
+                .toIntOrNull()?.plus(1) ?: 3
+
+        // Prevents them from fucking us over with doing a while(true){} loop
+        val interval = maxOf(data.pingInterval?.toLong()?.plus(2000) ?: return, 10000L)
+        var reconnect = false
+        // New SID can be negotiated as above, but not implemented yet as it seems rare.
+        while (true) {
+            val authData = if (reconnect) """
+                42["_reconnect", "$reconnectSid"]
+            """.trimIndent() else authInt
+
+            val url = "${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}"
+            app.post(url, data = authData, headers = headers)
+            //.also { println("Sflix post job ${it.text}") }
+            Log.d(this.name, "Running Sflix job $url")
+
+            val time = measureTimeMillis {
+                // This acts as a timeout
+                val getResponse = app.get(
+                    "${extractorData}&t=${generateTimeStamp()}&sid=${data.sid}",
+                    timeout = 60,
+                    headers = headers
+                ).text //.also { println("Sflix get job $it") }
+                if (getResponse.contains("sid")) {
+                    reconnect = true
+//                println("Reconnecting")
+                }
+            }
+            // Always waits even if the get response is instant, to prevent a while true loop.
+            if (time < interval - 4000)
+                delay(interval)
+        }
+    }
+
     private fun Element.toSearchResult(): SearchResponse {
         val img = this.select("img")
         val title = img.attr("title")
@@ -363,7 +475,11 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
         }
 
         // For re-use in Zoro
-        fun Sources.toExtractorLink(caller: MainAPI, name: String): List<ExtractorLink>? {
+        fun Sources.toExtractorLink(
+            caller: MainAPI,
+            name: String,
+            extractorData: String? = null
+        ): List<ExtractorLink>? {
             return this.file?.let { file ->
                 //println("FILE::: $file")
                 val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals(
@@ -382,7 +498,8 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
                                 stream.streamUrl,
                                 caller.mainUrl,
                                 getQualityFromName(stream.quality.toString()),
-                                true
+                                true,
+                                extractorData = extractorData
                             )
                         }
                 } else {
@@ -393,6 +510,7 @@ class SflixProvider(providerUrl: String, providerName: String) : MainAPI() {
                         caller.mainUrl,
                         getQualityFromName(this.type ?: ""),
                         false,
+                        extractorData = extractorData
                     ))
                 }
             }

app/src/main/java/com/lagradost/cloudstream3/network/Requests.kt

@@ -5,13 +5,18 @@ import android.content.Context
 import android.util.Log
 import androidx.preference.PreferenceManager
 import com.fasterxml.jackson.module.kotlin.readValue
-import com.lagradost.cloudstream3.*
+import com.lagradost.cloudstream3.R
+import com.lagradost.cloudstream3.USER_AGENT
+import com.lagradost.cloudstream3.app
+import com.lagradost.cloudstream3.mapper
 import kotlinx.coroutines.CancellableContinuation
 import kotlinx.coroutines.CompletionHandler
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.suspendCancellableCoroutine
 import okhttp3.*
 import okhttp3.Headers.Companion.toHeaders
+import okhttp3.MediaType.Companion.toMediaTypeOrNull
+import okhttp3.RequestBody.Companion.toRequestBody
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import java.io.File
@@ -107,14 +112,20 @@ class AppResponse(
     }
 }
 
-private fun getData(data: Map<String, String?>): RequestBody {
+private fun getData(data: Any?): RequestBody {
-    val builder = FormBody.Builder()
+    return when (data) {
-    data.forEach {
+        null -> FormBody.Builder().build()
-        it.value?.let { value ->
+        is Map<*, *> -> {
-            builder.add(it.key, value)
+            val builder = FormBody.Builder()
+            data.forEach {
+                if (it.key is String && it.value is String)
+                    builder.add(it.key as String, it.value as String)
+            }
+            builder.build()
         }
+        else ->
+            data.toString().toRequestBody("text/plain;charset=UTF-8".toMediaTypeOrNull())
     }
-    return builder.build()
 }
 
 // https://github.com, id=test -> https://github.com?id=test
@@ -169,7 +180,7 @@ fun postRequestCreator(
     referer: String? = null,
     params: Map<String, String> = emptyMap(),
     cookies: Map<String, String> = emptyMap(),
-    data: Map<String, String> = emptyMap(),
+    data: Any? = DEFAULT_DATA,
     cacheTime: Int = DEFAULT_TIME,
     cacheUnit: TimeUnit = DEFAULT_TIME_UNIT
 ): Request {
@@ -340,7 +351,7 @@ open class Requests {
         referer: String? = null,
         params: Map<String, String> = mapOf(),
         cookies: Map<String, String> = mapOf(),
-        data: Map<String, String> = DEFAULT_DATA,
+        data: Any? = DEFAULT_DATA,
         allowRedirects: Boolean = true,
         cacheTime: Int = DEFAULT_TIME,
         cacheUnit: TimeUnit = DEFAULT_TIME_UNIT,

app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt

@@ -32,13 +32,13 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
     }
 
     /**
-     * @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls.
+     * @param requestCallBack asynchronously return matched requests by either interceptUrl or additionalUrls. If true, destroy WebView.
      * @return the final request (by interceptUrl) and all the collected urls (by additionalUrls).
      * */
     @SuppressLint("SetJavaScriptEnabled")
     suspend fun resolveUsingWebView(
         request: Request,
-        requestCallBack: (Request) -> Unit = {}
+        requestCallBack: (Request) -> Boolean = { false }
     ): Pair<Request?, List<Request>> {
         val url = request.url.toString()
         val headers = request.headers
@@ -81,14 +81,18 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
 //                    println("Loading WebView URL: $webViewUrl")
 
                     if (interceptUrl.containsMatchIn(webViewUrl)) {
-                        fixedRequest = request.toRequest().also(requestCallBack)
+                        fixedRequest = request.toRequest().also {
+                            if (requestCallBack(it)) destroyWebView()
+                        }
                         println("Web-view request finished: $webViewUrl")
                         destroyWebView()
                         return@runBlocking null
                     }
 
                     if (additionalUrls.any { it.containsMatchIn(webViewUrl) }) {
-                        extraRequestList.add(request.toRequest().also(requestCallBack))
+                        extraRequestList.add(request.toRequest().also {
+                            if (requestCallBack(it)) destroyWebView()
+                        })
                     }
 
                     // Suppress image requests as we don't display them anywhere
@@ -129,11 +133,6 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
                      *  e.g the recaptcha request.
                      * **/
 
-                    /** NOTE!  request.requestHeaders is not perfect!
-                     *  They don't contain all the headers the browser actually gives.
-                     *  Overriding with okhttp might fuck up otherwise working requests,
-                     *  e.g the recaptcha request.
-                     * **/
                     return@runBlocking try {
                         when {
                             blacklistedFiles.any { URI(webViewUrl).path.contains(it) } || webViewUrl.endsWith(
@@ -204,7 +203,7 @@ class WebViewResolver(val interceptUrl: Regex, val additionalUrls: List<Regex> =
                 null,
                 emptyMap(),
                 emptyMap(),
-                emptyMap(),
+                emptyMap<String, String>(),
                 10,
                 TimeUnit.MINUTES
             )

app/src/main/java/com/lagradost/cloudstream3/ui/player/GeneratorPlayer.kt

@@ -15,6 +15,7 @@ import androidx.lifecycle.ViewModelProvider
 import com.google.android.material.button.MaterialButton
 import com.hippo.unifile.UniFile
 import com.lagradost.cloudstream3.*
+import com.lagradost.cloudstream3.APIHolder.getApiFromNameNull
 import com.lagradost.cloudstream3.CommonActivity.showToast
 import com.lagradost.cloudstream3.mvvm.Resource
 import com.lagradost.cloudstream3.mvvm.logError
@@ -26,11 +27,13 @@ import com.lagradost.cloudstream3.ui.result.ResultFragment
 import com.lagradost.cloudstream3.ui.settings.SettingsFragment.Companion.isTvSettings
 import com.lagradost.cloudstream3.ui.subtitles.SubtitlesFragment
 import com.lagradost.cloudstream3.utils.*
+import com.lagradost.cloudstream3.utils.Coroutines.ioSafe
 import com.lagradost.cloudstream3.utils.UIHelper.dismissSafe
 import com.lagradost.cloudstream3.utils.UIHelper.hideSystemUI
 import com.lagradost.cloudstream3.utils.UIHelper.popCurrentPage
 import kotlinx.android.synthetic.main.fragment_player.*
 import kotlinx.android.synthetic.main.player_custom_layout.*
+import kotlinx.coroutines.Job
 
 class GeneratorPlayer : FullScreenPlayer() {
     companion object {
@@ -80,6 +83,19 @@ class GeneratorPlayer : FullScreenPlayer() {
         return durPos.position
     }
 
+    var currentVerifyLink: Job? = null
+
+    private fun loadExtractorJob(extractorLink: ExtractorLink?) {
+        currentVerifyLink?.cancel()
+        extractorLink?.let {
+            currentVerifyLink = ioSafe {
+                if (it.extractorData != null) {
+                    getApiFromNameNull(it.source)?.extractorVerifierJob(it.extractorData)
+                }
+            }
+        }
+    }
+
     private fun loadLink(link: Pair<ExtractorLink?, ExtractorUri?>?, sameEpisode: Boolean) {
         if (link == null) return
 
@@ -93,6 +109,7 @@ class GeneratorPlayer : FullScreenPlayer() {
         setPlayerDimen(null)
         setTitle()
 
+        loadExtractorJob(link.first)
         // load player
         context?.let { ctx ->
             val (url, uri) = link
@@ -344,6 +361,7 @@ class GeneratorPlayer : FullScreenPlayer() {
 
     override fun onDestroy() {
         ResultFragment.updateUI()
+        currentVerifyLink?.cancel()
         super.onDestroy()
     }
 

app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt

@@ -16,7 +16,9 @@ data class ExtractorLink(
     override val referer: String,
     val quality: Int,
     val isM3u8: Boolean = false,
-    override val headers: Map<String, String> = mapOf()
+    override val headers: Map<String, String> = mapOf(),
+    /** Used for getExtractorVerifierJob() */
+    val extractorData: String? = null
 ) : VideoDownloadManager.IDownloadableMinimum
 
 data class ExtractorUri(