Added Zoro with Arjix (#131)

* Added Zoro with Arjix Heavily improved search speed Upped webview timeout to 1 minute Co-authored-by: ArjixWasTaken <arjixg53@gmail.com>
2021-10-09 14:19:26 +02:00 · 2021-10-09 14:19:26 +02:00 · 44e0c1c606
commit 44e0c1c606
parent 8b88e42ec3
12 changed files with 384 additions and 78 deletions
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
  <component name="ExternalStorageConfigurationManager" enabled="true" />
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_11" project-jdk-name="11" project-jdk-type="JavaSDK" />
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="11" project-jdk-type="JavaSDK" />
 </project>
--- a/README.md
+++ b/README.md
@ -59,6 +59,7 @@ It merely scrapes 3rd-party websites that are publicly accessable via any regula
 - [vf-film.org](https://vf-film.org) 
 - [asianload.cc](https://asianload.cc) 
 - [sflix.to](https://sflix.to) 
+- [zoro.to](https://zoro.to) 
 - [trailers.to](https://trailers.to) 
 - [thenos.org](https://www.thenos.org) 
 - [asiaflix.app](https://asiaflix.app) 
--- a/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt
@ -1,5 +1,6 @@
 package com.lagradost.cloudstream3

+import android.annotation.SuppressLint
 import android.content.Context
 import androidx.preference.PreferenceManager
 import com.fasterxml.jackson.databind.DeserializationFeature
@ -47,6 +48,7 @@ object APIHolder {
        AsianLoadProvider(),

        SflixProvider(),
+        ZoroProvider()
    )

    val restrictedApis = arrayListOf(
@ -203,6 +205,7 @@ abstract class MainAPI {
 }

 /** Might need a different implementation for desktop*/
+@SuppressLint("NewApi")
 fun base64Decode(string: String): String {
    return try {
        String(android.util.Base64.decode(string, android.util.Base64.DEFAULT), Charsets.ISO_8859_1)
--- a/app/src/main/java/com/lagradost/cloudstream3/animeproviders/TenshiProvider.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/animeproviders/TenshiProvider.kt
@ -78,7 +78,7 @@ class TenshiProvider : MainAPI() {
                    val title = section.selectFirst("h2").text()
                    val anime = section.select("li > a").map {
                        AnimeSearchResponse(
-                            it.selectFirst(".thumb-title").text(),
+                            it.selectFirst(".thumb-title")?.text() ?: "",
                            fixUrl(it.attr("href")),
                            this.name,
                            TvType.Anime,
--- a/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt
@ -0,0 +1,269 @@
+package com.lagradost.cloudstream3.animeproviders
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.module.kotlin.readValue
+import com.lagradost.cloudstream3.*
+import com.lagradost.cloudstream3.movieproviders.SflixProvider
+import com.lagradost.cloudstream3.movieproviders.SflixProvider.Companion.toExtractorLink
+import com.lagradost.cloudstream3.movieproviders.SflixProvider.Companion.toSubtitleFile
+import com.lagradost.cloudstream3.network.WebViewResolver
+import com.lagradost.cloudstream3.network.get
+import com.lagradost.cloudstream3.network.text
+import com.lagradost.cloudstream3.utils.ExtractorLink
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+import java.net.URI
+import java.util.*
+
+class ZoroProvider : MainAPI() {
+    override val mainUrl: String
+        get() = "https://zoro.to"
+    override val name: String
+        get() = "Zoro"
+
+    override val hasQuickSearch: Boolean
+        get() = false
+
+    override val hasMainPage: Boolean
+        get() = true
+
+    override val hasChromecastSupport: Boolean
+        get() = true
+
+    override val hasDownloadSupport: Boolean
+        get() = true
+
+    override val supportedTypes: Set<TvType>
+        get() = setOf(
+            TvType.Anime,
+            TvType.AnimeMovie,
+            TvType.ONA
+        )
+
+    companion object {
+        fun getType(t: String): TvType {
+            return if (t.contains("OVA") || t.contains("Special")) TvType.ONA
+            else if (t.contains("Movie")) TvType.AnimeMovie
+            else TvType.Anime
+        }
+
+        fun getStatus(t: String): ShowStatus {
+            return when (t) {
+                "Finished Airing" -> ShowStatus.Completed
+                "Currently Airing" -> ShowStatus.Ongoing
+                else -> ShowStatus.Completed
+            }
+        }
+    }
+
+    fun Element.toSearchResult(): SearchResponse? {
+        val href = fixUrl(this.select("a").attr("href"))
+        val title = this.select("h3.film-name").text()
+        if (href.contains("/news/") || title.trim().equals("News", ignoreCase = true)) return null
+        val posterUrl = fixUrl(this.select("img").attr("data-src"))
+        val type = getType(this.select("div.fd-infor > span.fdi-item").text())
+
+        return AnimeSearchResponse(
+            title,
+            href,
+            this@ZoroProvider.name,
+            type,
+            posterUrl,
+            null,
+            null,
+            EnumSet.of(DubStatus.Subbed),
+            null,
+            null
+        )
+    }
+
+
+    override fun getMainPage(): HomePageResponse {
+        val html = get("$mainUrl/home").text
+        val document = Jsoup.parse(html)
+
+        val homePageList = ArrayList<HomePageList>()
+
+        document.select("div.anif-block").forEach { block ->
+            val header = block.select("div.anif-block-header").text().trim()
+            val animes = block.select("li").mapNotNull {
+                it.toSearchResult()
+            }
+            if (animes.isNotEmpty()) homePageList.add(HomePageList(header, animes))
+        }
+
+        document.select("section.block_area.block_area_home").forEach { block ->
+            val header = block.select("h2.cat-heading").text().trim()
+            val animes = block.select("div.flw-item").mapNotNull {
+                it.toSearchResult()
+            }
+            if (animes.isNotEmpty()) homePageList.add(HomePageList(header, animes))
+        }
+
+        return HomePageResponse(homePageList)
+    }
+
+    private data class Response(
+        @JsonProperty("status") val status: Boolean,
+        @JsonProperty("html") val html: String
+    )
+
+//    override fun quickSearch(query: String): List<SearchResponse> {
+//        val url = "$mainUrl/ajax/search/suggest?keyword=${query}"
+//        val html = mapper.readValue<Response>(khttp.get(url).text).html
+//        val document = Jsoup.parse(html)
+//
+//        return document.select("a.nav-item").map {
+//            val title = it.selectFirst(".film-name")?.text().toString()
+//            val href = fixUrl(it.attr("href"))
+//            val year = it.selectFirst(".film-infor > span")?.text()?.split(",")?.get(1)?.trim()?.toIntOrNull()
+//            val image = it.select("img").attr("data-src")
+//
+//            AnimeSearchResponse(
+//                title,
+//                href,
+//                this.name,
+//                TvType.TvSeries,
+//                image,
+//                year,
+//                null,
+//                EnumSet.of(DubStatus.Subbed),
+//                null,
+//                null
+//            )
+//
+//        }
+//    }
+
+    override fun search(query: String): List<SearchResponse> {
+        val link = "$mainUrl/search?keyword=$query"
+        val html = get(link).text
+        val document = Jsoup.parse(html)
+
+        return document.select(".flw-item").map {
+            val title = it.selectFirst(".film-detail > .film-name > a")?.attr("title").toString()
+            val poster = it.selectFirst(".film-poster > img")?.attr("data-src")
+
+            val tvType = getType(it.selectFirst(".film-detail > .fd-infor > .fdi-item")?.text().toString())
+            val href = fixUrl(it.selectFirst(".film-name a").attr("href"))
+
+            AnimeSearchResponse(
+                title,
+                href,
+                name,
+                tvType,
+                poster,
+                null,
+                null,
+                EnumSet.of(DubStatus.Subbed),
+                null,
+                null
+            )
+        }
+    }
+
+    override fun load(url: String): LoadResponse? {
+        val html = get(url).text
+        val document = Jsoup.parse(html)
+
+        val title = document.selectFirst(".anisc-detail > .film-name")?.text().toString()
+        val poster = document.selectFirst(".anisc-poster img")?.attr("src")
+        val tags = document.select(".anisc-info a[href*=\"/genre/\"]").map { it.text() }
+
+        var year: Int? = null
+        var japaneseTitle: String? = null
+        var status: ShowStatus? = null
+
+
+        for (info in document.select(".anisc-info > .item.item-title")) {
+            val text = info?.text().toString()
+            when {
+                (year != null && japaneseTitle != null && status != null) -> break
+                text.contains("Premiered") && year == null ->
+                    year = info.selectFirst(".name")?.text().toString().split(" ").last().toIntOrNull()
+
+                text.contains("Japanese") && japaneseTitle == null ->
+                    japaneseTitle = info.selectFirst(".name")?.text().toString()
+
+                text.contains("Status") && status == null ->
+                    status = getStatus(info.selectFirst(".name")?.text().toString())
+            }
+        }
+
+        val description = document.selectFirst(".film-description.m-hide > .text")?.text()
+        val animeId = URI(url).path.split("-").last()
+
+        val episodes = Jsoup.parse(
+            mapper.readValue<Response>(
+                get(
+                    "$mainUrl/ajax/v2/episode/list/$animeId"
+                ).text
+            ).html
+        ).select(".ss-list > a[href].ssl-item.ep-item").map {
+            val name = it?.attr("title")
+            AnimeEpisode(
+                fixUrl(it.attr("href")),
+                name,
+                null,
+                null,
+                null,
+                null,
+                it.selectFirst(".ssli-order")?.text()?.toIntOrNull()
+            )
+        }
+        return AnimeLoadResponse(
+            title,
+            japaneseTitle,
+            title,
+            url,
+            this.name,
+            TvType.Anime,
+            poster,
+            year,
+            null,
+            episodes,
+            status,
+            description,
+            tags,
+        )
+    }
+
+    override fun loadLinks(
+        data: String,
+        isCasting: Boolean,
+        subtitleCallback: (SubtitleFile) -> Unit,
+        callback: (ExtractorLink) -> Unit
+    ): Boolean {
+
+        // Copy pasted from Sflix :)
+
+        val sources = get(
+            data,
+            interceptor = WebViewResolver(
+                Regex("""/getSources""")
+            )
+        ).text
+
+        val mapped = mapper.readValue<SflixProvider.SourceObject>(sources)
+
+        val list = listOf(
+            mapped.sources to "source 1",
+            mapped.sources1 to "source 2",
+            mapped.sources2 to "source 3",
+            mapped.sourcesBackup to "source backup"
+        )
+
+        list.forEach { subList ->
+            subList.first?.forEach {
+                it?.toExtractorLink(this, subList.second)?.forEach(callback)
+            }
+        }
+
+        mapped.tracks?.forEach {
+            it?.toSubtitleFile()?.let { subtitleFile ->
+                subtitleCallback.invoke(subtitleFile)
+            }
+        }
+        return true
+    }
+}
--- a/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt
@ -1,6 +1,5 @@
 package com.lagradost.cloudstream3.movieproviders

-import android.net.Uri
 import com.fasterxml.jackson.annotation.JsonProperty
 import com.fasterxml.jackson.module.kotlin.readValue
 import com.lagradost.cloudstream3.*
@ -251,59 +250,20 @@ class SflixProvider : MainAPI() {
        @JsonProperty("kind") val kind: String?
    )

-    data class Sources1(
+    data class Sources(
        @JsonProperty("file") val file: String?,
        @JsonProperty("type") val type: String?,
        @JsonProperty("label") val label: String?
    )

    data class SourceObject(
-        @JsonProperty("sources") val sources: List<Sources1?>?,
-        @JsonProperty("sources_1") val sources1: List<Sources1?>?,
-        @JsonProperty("sources_2") val sources2: List<Sources1?>?,
-        @JsonProperty("sourcesBackup") val sourcesBackup: List<Sources1?>?,
+        @JsonProperty("sources") val sources: List<Sources?>?,
+        @JsonProperty("sources_1") val sources1: List<Sources?>?,
+        @JsonProperty("sources_2") val sources2: List<Sources?>?,
+        @JsonProperty("sourcesBackup") val sourcesBackup: List<Sources?>?,
        @JsonProperty("tracks") val tracks: List<Tracks?>?
    )

-    private fun Sources1.toExtractorLink(name: String): List<ExtractorLink>? {
-        return this.file?.let {
-            val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals("hls", ignoreCase = true)
-            if (isM3u8) {
-                M3u8Helper().m3u8Generation(M3u8Helper.M3u8Stream(this.file, null), true).map { stream ->
-                    val qualityString = if ((stream.quality ?: 0) == 0) label ?: "" else "${stream.quality}p"
-                    ExtractorLink(
-                        this@SflixProvider.name,
-                        "${this@SflixProvider.name} $qualityString $name",
-                        stream.streamUrl,
-                        mainUrl,
-                        getQualityFromName(stream.quality.toString()),
-                        true
-                    )
-                }
-            } else {
-                listOf(ExtractorLink(
-                    this@SflixProvider.name,
-                    this.label?.let { "${this@SflixProvider.name} - $it" } ?: this@SflixProvider.name,
-                    it,
-                    this@SflixProvider.mainUrl,
-                    getQualityFromName(this.type ?: ""),
-                    false,
-                ))
-            }
-
-        }
-    }
-
-    private fun Tracks.toSubtitleFile(): SubtitleFile? {
-        return this.file?.let {
-            SubtitleFile(
-                this.label ?: "Unknown",
-                it
-            )
-        }
-
-    }
-
    override fun loadLinks(
        data: String,
        isCasting: Boolean,
@ -339,14 +299,14 @@ class SflixProvider : MainAPI() {
        val mapped = mapper.readValue<SourceObject>(sources)

        val list = listOf(
-            mapped.sources1 to "source 1",
-            mapped.sources2 to "source 2",
-            mapped.sources to "source 0",
-            mapped.sourcesBackup to "source 3"
+            mapped.sources to "source 1",
+            mapped.sources1 to "source 2",
+            mapped.sources2 to "source 3",
+            mapped.sourcesBackup to "source backup"
        )
        list.forEach { subList ->
            subList.first?.forEach {
-                it?.toExtractorLink(subList.second)?.forEach(callback)
+                it?.toExtractorLink(this, subList.second)?.forEach(callback)
            }
        }
        mapped.tracks?.forEach {
@ -357,4 +317,47 @@ class SflixProvider : MainAPI() {
        return true
    }

+    companion object {
+        // For re-use in Zoro
+
+        fun Sources.toExtractorLink(caller: MainAPI, name: String): List<ExtractorLink>? {
+            return this.file?.let {
+                val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals("hls", ignoreCase = true)
+                if (isM3u8) {
+                    M3u8Helper().m3u8Generation(M3u8Helper.M3u8Stream(this.file, null), true).map { stream ->
+                        val qualityString = if ((stream.quality ?: 0) == 0) label ?: "" else "${stream.quality}p"
+                        ExtractorLink(
+                            caller.name,
+                            "${caller.name} $qualityString $name",
+                            stream.streamUrl,
+                            caller.mainUrl,
+                            getQualityFromName(stream.quality.toString()),
+                            true
+                        )
                    }
+                } else {
+                    listOf(ExtractorLink(
+                        caller.name,
+                        this.label?.let { "${caller.name} - $it" } ?: caller.name,
+                        it,
+                        caller.mainUrl,
+                        getQualityFromName(this.type ?: ""),
+                        false,
+                    ))
+                }
+
+            }
+        }
+
+        fun Tracks.toSubtitleFile(): SubtitleFile? {
+            return this.file?.let {
+                SubtitleFile(
+                    this.label ?: "Unknown",
+                    it
+                )
+            }
+        }
+
+    }
+}
+
--- a/app/src/main/java/com/lagradost/cloudstream3/movieproviders/VMoveeProvider.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/movieproviders/VMoveeProvider.kt
@ -111,7 +111,7 @@ class VMoveeProvider : MainAPI() {
            }
        }

-        return super.loadLinks(data, isCasting, subtitleCallback, callback)
+        return true
    }

    override fun load(url: String): LoadResponse {
--- a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt
@ -39,11 +39,14 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor {
        var fixedRequest: Request? = null

        main {
+            // Useful for debugging
+//            WebView.setWebContentsDebuggingEnabled(true)
            webView = WebView(
                AcraApplication.context ?: throw RuntimeException("No base context in WebViewResolver")
            ).apply {
-                settings.cacheMode
+                // Bare minimum to bypass captcha
                settings.javaScriptEnabled = true
+                settings.domStorageEnabled = true
            }

            webView?.webViewClient = object : WebViewClient() {
@ -52,6 +55,7 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor {
                    request: WebResourceRequest
                ): WebResourceResponse? {
                    val webViewUrl = request.url.toString()
+//                    println("Override url $webViewUrl")
                    if (interceptUrl.containsMatchIn(webViewUrl)) {
                        fixedRequest = getRequestCreator(
                            webViewUrl,
@ -62,6 +66,7 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor {
                            10,
                            TimeUnit.MINUTES
                        )
+
                        println("Web-view request finished: $webViewUrl")
                        destroyWebView()
                    }
@ -77,8 +82,9 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor {
        }

        var loop = 0
-        // Timeouts after this amount, 20s
-        val totalTime = 20000L
+        // Timeouts after this amount, 60s
+        val totalTime = 60000L
+
        val delayTime = 100L

        // A bit sloppy, but couldn't find a better way
--- a/app/src/main/java/com/lagradost/cloudstream3/ui/APIRepository.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/ui/APIRepository.kt
@ -37,7 +37,9 @@ class APIRepository(val api: MainAPI) {
    suspend fun search(query: String): Resource<List<SearchResponse>> {
        return safeApiCall {
            return@safeApiCall (api.search(query)
-                ?: throw ErrorLoadingException()).filter { typesActive.contains(it.type) }.toList()
+                ?: throw ErrorLoadingException())
+//                .filter { typesActive.contains(it.type) }
+                .toList()
        }
    }

--- a/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchFragment.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchFragment.kt
@ -21,6 +21,7 @@ import com.lagradost.cloudstream3.APIHolder.getApiProviderLangSettings
 import com.lagradost.cloudstream3.APIHolder.getApiSettings
 import com.lagradost.cloudstream3.APIHolder.getApiTypeSettings
 import com.lagradost.cloudstream3.mvvm.Resource
+import com.lagradost.cloudstream3.mvvm.logError
 import com.lagradost.cloudstream3.mvvm.observe
 import com.lagradost.cloudstream3.ui.APIRepository
 import com.lagradost.cloudstream3.ui.APIRepository.Companion.providersActive
@ -35,6 +36,8 @@ import com.lagradost.cloudstream3.utils.UIHelper.fixPaddingStatusbar
 import com.lagradost.cloudstream3.utils.UIHelper.getGridIsCompact
 import com.lagradost.cloudstream3.utils.UIHelper.hideKeyboard
 import kotlinx.android.synthetic.main.fragment_search.*
+import java.lang.Exception
+import java.util.concurrent.locks.ReentrantLock

 class SearchFragment : Fragment() {
    companion object {
@ -331,7 +334,11 @@ class SearchFragment : Fragment() {
            }
        }

+        val listLock = ReentrantLock()
        observe(searchViewModel.currentSearch) { list ->
+            try {
+                // https://stackoverflow.com/questions/6866238/concurrent-modification-exception-adding-to-an-arraylist
+                listLock.lock()
                (search_master_recycler?.adapter as ParentItemAdapter?)?.apply {
                    items = list.map { ongoing ->
                        val ongoingList = HomePageList(
@ -342,6 +349,11 @@ class SearchFragment : Fragment() {
                    }
                    notifyDataSetChanged()
                }
+            } catch (e: Exception) {
+                logError(e)
+            } finally {
+                listLock.unlock()
+            }
        }

        activity?.let {
--- a/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchViewModel.kt
+++ b/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchViewModel.kt
@ -6,11 +6,18 @@ import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import com.lagradost.cloudstream3.APIHolder.apis
 import com.lagradost.cloudstream3.SearchResponse
+import com.lagradost.cloudstream3.apmap
 import com.lagradost.cloudstream3.mvvm.Resource
+import com.lagradost.cloudstream3.pmap
 import com.lagradost.cloudstream3.ui.APIRepository
 import com.lagradost.cloudstream3.ui.APIRepository.Companion.providersActive
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import okhttp3.internal.notify
+import java.util.concurrent.locks.ReentrantLock
+import kotlin.concurrent.thread

 data class OnGoingSearch(
    val apiName: String,
@ -48,12 +55,15 @@ class SearchViewModel : ViewModel() {

        _currentSearch.postValue(ArrayList())

+        withContext(Dispatchers.IO) { // This interrupts UI otherwise
            repos.filter { a ->
                (providersActive.size == 0 || providersActive.contains(a.name))
-        }.map { a ->
-            currentList.add(OnGoingSearch(a.name, a.search(query)))
+            }.apmap { a -> // Parallel
+                val search = a.search(query)
+                currentList.add(OnGoingSearch(a.name,search ))
                _currentSearch.postValue(currentList)
            }
+        }
        _currentSearch.postValue(currentList)


--- a/app/src/test/java/com/lagradost/cloudstream3/ProviderTests.kt
+++ b/app/src/test/java/com/lagradost/cloudstream3/ProviderTests.kt
@ -144,7 +144,7 @@ class ProviderTests {

    @Test
    fun providerCorrectHomepage() {
-        for (api in getAllProviders()) {
+        getAllProviders().pmap { api ->
            if (api.hasMainPage) {
                try {
                    val homepage = api.getMainPage()
@ -177,13 +177,13 @@ class ProviderTests {
    @Test
    fun providerCorrect() {
        val providers = getAllProviders()
-        for ((index, api) in providers.withIndex()) {
+        providers.pmap { api ->
            try {
-                println("Trying $api (${index + 1}/${providers.size})")
+                println("Trying $api")
                if (testSingleProviderApi(api)) {
-                    println("Success $api (${index + 1}/${providers.size})")
+                    println("Success $api")
                } else {
-                    System.err.println("Error $api (${index + 1}/${providers.size})")
+                    System.err.println("Error $api")
                }
            } catch (e: Exception) {
                logError(e)