diff --git a/.idea/misc.xml b/.idea/misc.xml index 25d34a47..4bc4fc6e 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,5 +1,5 @@ - + \ No newline at end of file diff --git a/README.md b/README.md index b887afd2..cb1237d6 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ It merely scrapes 3rd-party websites that are publicly accessable via any regula - [vf-film.org](https://vf-film.org) - [asianload.cc](https://asianload.cc) - [sflix.to](https://sflix.to) +- [zoro.to](https://zoro.to) - [trailers.to](https://trailers.to) - [thenos.org](https://www.thenos.org) - [asiaflix.app](https://asiaflix.app) diff --git a/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt b/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt index 64a63cee..48459479 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt @@ -1,5 +1,6 @@ package com.lagradost.cloudstream3 +import android.annotation.SuppressLint import android.content.Context import androidx.preference.PreferenceManager import com.fasterxml.jackson.databind.DeserializationFeature @@ -47,6 +48,7 @@ object APIHolder { AsianLoadProvider(), SflixProvider(), + ZoroProvider() ) val restrictedApis = arrayListOf( @@ -203,6 +205,7 @@ abstract class MainAPI { } /** Might need a different implementation for desktop*/ +@SuppressLint("NewApi") fun base64Decode(string: String): String { return try { String(android.util.Base64.decode(string, android.util.Base64.DEFAULT), Charsets.ISO_8859_1) diff --git a/app/src/main/java/com/lagradost/cloudstream3/animeproviders/TenshiProvider.kt b/app/src/main/java/com/lagradost/cloudstream3/animeproviders/TenshiProvider.kt index 0fad07b8..383b0b26 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/animeproviders/TenshiProvider.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/animeproviders/TenshiProvider.kt @@ -78,7 +78,7 @@ class TenshiProvider : MainAPI() { val title = section.selectFirst("h2").text() val anime = section.select("li > a").map { AnimeSearchResponse( - it.selectFirst(".thumb-title").text(), + it.selectFirst(".thumb-title")?.text() ?: "", fixUrl(it.attr("href")), this.name, TvType.Anime, diff --git a/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt b/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt new file mode 100644 index 00000000..38bd8447 --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/animeproviders/ZoroProvider.kt @@ -0,0 +1,269 @@ +package com.lagradost.cloudstream3.animeproviders + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.module.kotlin.readValue +import com.lagradost.cloudstream3.* +import com.lagradost.cloudstream3.movieproviders.SflixProvider +import com.lagradost.cloudstream3.movieproviders.SflixProvider.Companion.toExtractorLink +import com.lagradost.cloudstream3.movieproviders.SflixProvider.Companion.toSubtitleFile +import com.lagradost.cloudstream3.network.WebViewResolver +import com.lagradost.cloudstream3.network.get +import com.lagradost.cloudstream3.network.text +import com.lagradost.cloudstream3.utils.ExtractorLink +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import java.net.URI +import java.util.* + +class ZoroProvider : MainAPI() { + override val mainUrl: String + get() = "https://zoro.to" + override val name: String + get() = "Zoro" + + override val hasQuickSearch: Boolean + get() = false + + override val hasMainPage: Boolean + get() = true + + override val hasChromecastSupport: Boolean + get() = true + + override val hasDownloadSupport: Boolean + get() = true + + override val supportedTypes: Set + get() = setOf( + TvType.Anime, + TvType.AnimeMovie, + TvType.ONA + ) + + companion object { + fun getType(t: String): TvType { + return if (t.contains("OVA") || t.contains("Special")) TvType.ONA + else if (t.contains("Movie")) TvType.AnimeMovie + else TvType.Anime + } + + fun getStatus(t: String): ShowStatus { + return when (t) { + "Finished Airing" -> ShowStatus.Completed + "Currently Airing" -> ShowStatus.Ongoing + else -> ShowStatus.Completed + } + } + } + + fun Element.toSearchResult(): SearchResponse? { + val href = fixUrl(this.select("a").attr("href")) + val title = this.select("h3.film-name").text() + if (href.contains("/news/") || title.trim().equals("News", ignoreCase = true)) return null + val posterUrl = fixUrl(this.select("img").attr("data-src")) + val type = getType(this.select("div.fd-infor > span.fdi-item").text()) + + return AnimeSearchResponse( + title, + href, + this@ZoroProvider.name, + type, + posterUrl, + null, + null, + EnumSet.of(DubStatus.Subbed), + null, + null + ) + } + + + override fun getMainPage(): HomePageResponse { + val html = get("$mainUrl/home").text + val document = Jsoup.parse(html) + + val homePageList = ArrayList() + + document.select("div.anif-block").forEach { block -> + val header = block.select("div.anif-block-header").text().trim() + val animes = block.select("li").mapNotNull { + it.toSearchResult() + } + if (animes.isNotEmpty()) homePageList.add(HomePageList(header, animes)) + } + + document.select("section.block_area.block_area_home").forEach { block -> + val header = block.select("h2.cat-heading").text().trim() + val animes = block.select("div.flw-item").mapNotNull { + it.toSearchResult() + } + if (animes.isNotEmpty()) homePageList.add(HomePageList(header, animes)) + } + + return HomePageResponse(homePageList) + } + + private data class Response( + @JsonProperty("status") val status: Boolean, + @JsonProperty("html") val html: String + ) + +// override fun quickSearch(query: String): List { +// val url = "$mainUrl/ajax/search/suggest?keyword=${query}" +// val html = mapper.readValue(khttp.get(url).text).html +// val document = Jsoup.parse(html) +// +// return document.select("a.nav-item").map { +// val title = it.selectFirst(".film-name")?.text().toString() +// val href = fixUrl(it.attr("href")) +// val year = it.selectFirst(".film-infor > span")?.text()?.split(",")?.get(1)?.trim()?.toIntOrNull() +// val image = it.select("img").attr("data-src") +// +// AnimeSearchResponse( +// title, +// href, +// this.name, +// TvType.TvSeries, +// image, +// year, +// null, +// EnumSet.of(DubStatus.Subbed), +// null, +// null +// ) +// +// } +// } + + override fun search(query: String): List { + val link = "$mainUrl/search?keyword=$query" + val html = get(link).text + val document = Jsoup.parse(html) + + return document.select(".flw-item").map { + val title = it.selectFirst(".film-detail > .film-name > a")?.attr("title").toString() + val poster = it.selectFirst(".film-poster > img")?.attr("data-src") + + val tvType = getType(it.selectFirst(".film-detail > .fd-infor > .fdi-item")?.text().toString()) + val href = fixUrl(it.selectFirst(".film-name a").attr("href")) + + AnimeSearchResponse( + title, + href, + name, + tvType, + poster, + null, + null, + EnumSet.of(DubStatus.Subbed), + null, + null + ) + } + } + + override fun load(url: String): LoadResponse? { + val html = get(url).text + val document = Jsoup.parse(html) + + val title = document.selectFirst(".anisc-detail > .film-name")?.text().toString() + val poster = document.selectFirst(".anisc-poster img")?.attr("src") + val tags = document.select(".anisc-info a[href*=\"/genre/\"]").map { it.text() } + + var year: Int? = null + var japaneseTitle: String? = null + var status: ShowStatus? = null + + + for (info in document.select(".anisc-info > .item.item-title")) { + val text = info?.text().toString() + when { + (year != null && japaneseTitle != null && status != null) -> break + text.contains("Premiered") && year == null -> + year = info.selectFirst(".name")?.text().toString().split(" ").last().toIntOrNull() + + text.contains("Japanese") && japaneseTitle == null -> + japaneseTitle = info.selectFirst(".name")?.text().toString() + + text.contains("Status") && status == null -> + status = getStatus(info.selectFirst(".name")?.text().toString()) + } + } + + val description = document.selectFirst(".film-description.m-hide > .text")?.text() + val animeId = URI(url).path.split("-").last() + + val episodes = Jsoup.parse( + mapper.readValue( + get( + "$mainUrl/ajax/v2/episode/list/$animeId" + ).text + ).html + ).select(".ss-list > a[href].ssl-item.ep-item").map { + val name = it?.attr("title") + AnimeEpisode( + fixUrl(it.attr("href")), + name, + null, + null, + null, + null, + it.selectFirst(".ssli-order")?.text()?.toIntOrNull() + ) + } + return AnimeLoadResponse( + title, + japaneseTitle, + title, + url, + this.name, + TvType.Anime, + poster, + year, + null, + episodes, + status, + description, + tags, + ) + } + + override fun loadLinks( + data: String, + isCasting: Boolean, + subtitleCallback: (SubtitleFile) -> Unit, + callback: (ExtractorLink) -> Unit + ): Boolean { + + // Copy pasted from Sflix :) + + val sources = get( + data, + interceptor = WebViewResolver( + Regex("""/getSources""") + ) + ).text + + val mapped = mapper.readValue(sources) + + val list = listOf( + mapped.sources to "source 1", + mapped.sources1 to "source 2", + mapped.sources2 to "source 3", + mapped.sourcesBackup to "source backup" + ) + + list.forEach { subList -> + subList.first?.forEach { + it?.toExtractorLink(this, subList.second)?.forEach(callback) + } + } + + mapped.tracks?.forEach { + it?.toSubtitleFile()?.let { subtitleFile -> + subtitleCallback.invoke(subtitleFile) + } + } + return true + } +} diff --git a/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt b/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt index f36607e9..af28ab9e 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/movieproviders/SflixProvider.kt @@ -1,6 +1,5 @@ package com.lagradost.cloudstream3.movieproviders -import android.net.Uri import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.module.kotlin.readValue import com.lagradost.cloudstream3.* @@ -251,59 +250,20 @@ class SflixProvider : MainAPI() { @JsonProperty("kind") val kind: String? ) - data class Sources1( + data class Sources( @JsonProperty("file") val file: String?, @JsonProperty("type") val type: String?, @JsonProperty("label") val label: String? ) data class SourceObject( - @JsonProperty("sources") val sources: List?, - @JsonProperty("sources_1") val sources1: List?, - @JsonProperty("sources_2") val sources2: List?, - @JsonProperty("sourcesBackup") val sourcesBackup: List?, + @JsonProperty("sources") val sources: List?, + @JsonProperty("sources_1") val sources1: List?, + @JsonProperty("sources_2") val sources2: List?, + @JsonProperty("sourcesBackup") val sourcesBackup: List?, @JsonProperty("tracks") val tracks: List? ) - private fun Sources1.toExtractorLink(name: String): List? { - return this.file?.let { - val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals("hls", ignoreCase = true) - if (isM3u8) { - M3u8Helper().m3u8Generation(M3u8Helper.M3u8Stream(this.file, null), true).map { stream -> - val qualityString = if ((stream.quality ?: 0) == 0) label ?: "" else "${stream.quality}p" - ExtractorLink( - this@SflixProvider.name, - "${this@SflixProvider.name} $qualityString $name", - stream.streamUrl, - mainUrl, - getQualityFromName(stream.quality.toString()), - true - ) - } - } else { - listOf(ExtractorLink( - this@SflixProvider.name, - this.label?.let { "${this@SflixProvider.name} - $it" } ?: this@SflixProvider.name, - it, - this@SflixProvider.mainUrl, - getQualityFromName(this.type ?: ""), - false, - )) - } - - } - } - - private fun Tracks.toSubtitleFile(): SubtitleFile? { - return this.file?.let { - SubtitleFile( - this.label ?: "Unknown", - it - ) - } - - } - override fun loadLinks( data: String, isCasting: Boolean, @@ -339,14 +299,14 @@ class SflixProvider : MainAPI() { val mapped = mapper.readValue(sources) val list = listOf( - mapped.sources1 to "source 1", - mapped.sources2 to "source 2", - mapped.sources to "source 0", - mapped.sourcesBackup to "source 3" + mapped.sources to "source 1", + mapped.sources1 to "source 2", + mapped.sources2 to "source 3", + mapped.sourcesBackup to "source backup" ) list.forEach { subList -> subList.first?.forEach { - it?.toExtractorLink(subList.second)?.forEach(callback) + it?.toExtractorLink(this, subList.second)?.forEach(callback) } } mapped.tracks?.forEach { @@ -357,4 +317,47 @@ class SflixProvider : MainAPI() { return true } + companion object { + // For re-use in Zoro + + fun Sources.toExtractorLink(caller: MainAPI, name: String): List? { + return this.file?.let { + val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals("hls", ignoreCase = true) + if (isM3u8) { + M3u8Helper().m3u8Generation(M3u8Helper.M3u8Stream(this.file, null), true).map { stream -> + val qualityString = if ((stream.quality ?: 0) == 0) label ?: "" else "${stream.quality}p" + ExtractorLink( + caller.name, + "${caller.name} $qualityString $name", + stream.streamUrl, + caller.mainUrl, + getQualityFromName(stream.quality.toString()), + true + ) + } + } else { + listOf(ExtractorLink( + caller.name, + this.label?.let { "${caller.name} - $it" } ?: caller.name, + it, + caller.mainUrl, + getQualityFromName(this.type ?: ""), + false, + )) + } + + } + } + + fun Tracks.toSubtitleFile(): SubtitleFile? { + return this.file?.let { + SubtitleFile( + this.label ?: "Unknown", + it + ) + } + } + + } } + diff --git a/app/src/main/java/com/lagradost/cloudstream3/movieproviders/VMoveeProvider.kt b/app/src/main/java/com/lagradost/cloudstream3/movieproviders/VMoveeProvider.kt index ce3a92e4..f780a490 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/movieproviders/VMoveeProvider.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/movieproviders/VMoveeProvider.kt @@ -111,7 +111,7 @@ class VMoveeProvider : MainAPI() { } } - return super.loadLinks(data, isCasting, subtitleCallback, callback) + return true } override fun load(url: String): LoadResponse { diff --git a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt index 952d1d59..7d297920 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/network/WebViewResolver.kt @@ -39,11 +39,14 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor { var fixedRequest: Request? = null main { + // Useful for debugging +// WebView.setWebContentsDebuggingEnabled(true) webView = WebView( AcraApplication.context ?: throw RuntimeException("No base context in WebViewResolver") ).apply { - settings.cacheMode + // Bare minimum to bypass captcha settings.javaScriptEnabled = true + settings.domStorageEnabled = true } webView?.webViewClient = object : WebViewClient() { @@ -52,6 +55,7 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor { request: WebResourceRequest ): WebResourceResponse? { val webViewUrl = request.url.toString() +// println("Override url $webViewUrl") if (interceptUrl.containsMatchIn(webViewUrl)) { fixedRequest = getRequestCreator( webViewUrl, @@ -62,6 +66,7 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor { 10, TimeUnit.MINUTES ) + println("Web-view request finished: $webViewUrl") destroyWebView() } @@ -77,8 +82,9 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor { } var loop = 0 - // Timeouts after this amount, 20s - val totalTime = 20000L + // Timeouts after this amount, 60s + val totalTime = 60000L + val delayTime = 100L // A bit sloppy, but couldn't find a better way diff --git a/app/src/main/java/com/lagradost/cloudstream3/ui/APIRepository.kt b/app/src/main/java/com/lagradost/cloudstream3/ui/APIRepository.kt index 83b4eea9..c8183829 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/ui/APIRepository.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/ui/APIRepository.kt @@ -37,7 +37,9 @@ class APIRepository(val api: MainAPI) { suspend fun search(query: String): Resource> { return safeApiCall { return@safeApiCall (api.search(query) - ?: throw ErrorLoadingException()).filter { typesActive.contains(it.type) }.toList() + ?: throw ErrorLoadingException()) +// .filter { typesActive.contains(it.type) } + .toList() } } diff --git a/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchFragment.kt b/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchFragment.kt index a20bc163..793e383f 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchFragment.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchFragment.kt @@ -21,6 +21,7 @@ import com.lagradost.cloudstream3.APIHolder.getApiProviderLangSettings import com.lagradost.cloudstream3.APIHolder.getApiSettings import com.lagradost.cloudstream3.APIHolder.getApiTypeSettings import com.lagradost.cloudstream3.mvvm.Resource +import com.lagradost.cloudstream3.mvvm.logError import com.lagradost.cloudstream3.mvvm.observe import com.lagradost.cloudstream3.ui.APIRepository import com.lagradost.cloudstream3.ui.APIRepository.Companion.providersActive @@ -35,6 +36,8 @@ import com.lagradost.cloudstream3.utils.UIHelper.fixPaddingStatusbar import com.lagradost.cloudstream3.utils.UIHelper.getGridIsCompact import com.lagradost.cloudstream3.utils.UIHelper.hideKeyboard import kotlinx.android.synthetic.main.fragment_search.* +import java.lang.Exception +import java.util.concurrent.locks.ReentrantLock class SearchFragment : Fragment() { companion object { @@ -331,16 +334,25 @@ class SearchFragment : Fragment() { } } + val listLock = ReentrantLock() observe(searchViewModel.currentSearch) { list -> - (search_master_recycler?.adapter as ParentItemAdapter?)?.apply { - items = list.map { ongoing -> - val ongoingList = HomePageList( - ongoing.apiName, - if (ongoing.data is Resource.Success) ongoing.data.value.filterSearchResponse() else ArrayList() - ) - ongoingList + try { + // https://stackoverflow.com/questions/6866238/concurrent-modification-exception-adding-to-an-arraylist + listLock.lock() + (search_master_recycler?.adapter as ParentItemAdapter?)?.apply { + items = list.map { ongoing -> + val ongoingList = HomePageList( + ongoing.apiName, + if (ongoing.data is Resource.Success) ongoing.data.value.filterSearchResponse() else ArrayList() + ) + ongoingList + } + notifyDataSetChanged() } - notifyDataSetChanged() + } catch (e: Exception) { + logError(e) + } finally { + listLock.unlock() } } diff --git a/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchViewModel.kt b/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchViewModel.kt index f5b893d7..ff6378f6 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchViewModel.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/ui/search/SearchViewModel.kt @@ -6,11 +6,18 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.lagradost.cloudstream3.APIHolder.apis import com.lagradost.cloudstream3.SearchResponse +import com.lagradost.cloudstream3.apmap import com.lagradost.cloudstream3.mvvm.Resource +import com.lagradost.cloudstream3.pmap import com.lagradost.cloudstream3.ui.APIRepository import com.lagradost.cloudstream3.ui.APIRepository.Companion.providersActive +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import okhttp3.internal.notify +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.thread data class OnGoingSearch( val apiName: String, @@ -30,7 +37,7 @@ class SearchViewModel : ViewModel() { _searchResponse.postValue(Resource.Success(ArrayList())) } - var onGoingSearch : Job? = null + var onGoingSearch: Job? = null fun searchAndCancel(query: String) { onGoingSearch?.cancel() onGoingSearch = search(query) @@ -48,11 +55,14 @@ class SearchViewModel : ViewModel() { _currentSearch.postValue(ArrayList()) - repos.filter { a -> - (providersActive.size == 0 || providersActive.contains(a.name)) - }.map { a -> - currentList.add(OnGoingSearch(a.name, a.search(query))) - _currentSearch.postValue(currentList) + withContext(Dispatchers.IO) { // This interrupts UI otherwise + repos.filter { a -> + (providersActive.size == 0 || providersActive.contains(a.name)) + }.apmap { a -> // Parallel + val search = a.search(query) + currentList.add(OnGoingSearch(a.name,search )) + _currentSearch.postValue(currentList) + } } _currentSearch.postValue(currentList) diff --git a/app/src/test/java/com/lagradost/cloudstream3/ProviderTests.kt b/app/src/test/java/com/lagradost/cloudstream3/ProviderTests.kt index 4a0bf70a..1a1ab555 100644 --- a/app/src/test/java/com/lagradost/cloudstream3/ProviderTests.kt +++ b/app/src/test/java/com/lagradost/cloudstream3/ProviderTests.kt @@ -40,7 +40,7 @@ class ProviderTests { return true } - private fun testSingleProviderApi(api: MainAPI) : Boolean { + private fun testSingleProviderApi(api: MainAPI): Boolean { val searchQueries = listOf("over", "iron", "guy") var correctResponses = 0 var searchResult: List? = null @@ -144,7 +144,7 @@ class ProviderTests { @Test fun providerCorrectHomepage() { - for (api in getAllProviders()) { + getAllProviders().pmap { api -> if (api.hasMainPage) { try { val homepage = api.getMainPage() @@ -177,13 +177,13 @@ class ProviderTests { @Test fun providerCorrect() { val providers = getAllProviders() - for ((index, api) in providers.withIndex()) { + providers.pmap { api -> try { - println("Trying $api (${index + 1}/${providers.size})") - if(testSingleProviderApi(api)) { - println("Success $api (${index + 1}/${providers.size})") + println("Trying $api") + if (testSingleProviderApi(api)) { + println("Success $api") } else { - System.err.println("Error $api (${index + 1}/${providers.size})") + System.err.println("Error $api") } } catch (e: Exception) { logError(e)