Added Zoro with Arjix (#131)

* Added Zoro with Arjix
Heavily improved search speed
Upped webview timeout to 1 minute

Co-authored-by: ArjixWasTaken <arjixg53@gmail.com>
This commit is contained in:
LagradOst 2021-10-09 14:19:26 +02:00 committed by GitHub
parent 8b88e42ec3
commit 44e0c1c606
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 384 additions and 78 deletions

View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" project-jdk-name="11" project-jdk-type="JavaSDK" />
<component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="11" project-jdk-type="JavaSDK" />
</project>

View file

@ -59,6 +59,7 @@ It merely scrapes 3rd-party websites that are publicly accessable via any regula
- [vf-film.org](https://vf-film.org)
- [asianload.cc](https://asianload.cc)
- [sflix.to](https://sflix.to)
- [zoro.to](https://zoro.to)
- [trailers.to](https://trailers.to)
- [thenos.org](https://www.thenos.org)
- [asiaflix.app](https://asiaflix.app)

View file

@ -1,5 +1,6 @@
package com.lagradost.cloudstream3
import android.annotation.SuppressLint
import android.content.Context
import androidx.preference.PreferenceManager
import com.fasterxml.jackson.databind.DeserializationFeature
@ -47,6 +48,7 @@ object APIHolder {
AsianLoadProvider(),
SflixProvider(),
ZoroProvider()
)
val restrictedApis = arrayListOf(
@ -203,6 +205,7 @@ abstract class MainAPI {
}
/** Might need a different implementation for desktop*/
@SuppressLint("NewApi")
fun base64Decode(string: String): String {
return try {
String(android.util.Base64.decode(string, android.util.Base64.DEFAULT), Charsets.ISO_8859_1)

View file

@ -78,7 +78,7 @@ class TenshiProvider : MainAPI() {
val title = section.selectFirst("h2").text()
val anime = section.select("li > a").map {
AnimeSearchResponse(
it.selectFirst(".thumb-title").text(),
it.selectFirst(".thumb-title")?.text() ?: "",
fixUrl(it.attr("href")),
this.name,
TvType.Anime,

View file

@ -0,0 +1,269 @@
package com.lagradost.cloudstream3.animeproviders
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.module.kotlin.readValue
import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.movieproviders.SflixProvider
import com.lagradost.cloudstream3.movieproviders.SflixProvider.Companion.toExtractorLink
import com.lagradost.cloudstream3.movieproviders.SflixProvider.Companion.toSubtitleFile
import com.lagradost.cloudstream3.network.WebViewResolver
import com.lagradost.cloudstream3.network.get
import com.lagradost.cloudstream3.network.text
import com.lagradost.cloudstream3.utils.ExtractorLink
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import java.net.URI
import java.util.*
class ZoroProvider : MainAPI() {
override val mainUrl: String
get() = "https://zoro.to"
override val name: String
get() = "Zoro"
override val hasQuickSearch: Boolean
get() = false
override val hasMainPage: Boolean
get() = true
override val hasChromecastSupport: Boolean
get() = true
override val hasDownloadSupport: Boolean
get() = true
override val supportedTypes: Set<TvType>
get() = setOf(
TvType.Anime,
TvType.AnimeMovie,
TvType.ONA
)
companion object {
fun getType(t: String): TvType {
return if (t.contains("OVA") || t.contains("Special")) TvType.ONA
else if (t.contains("Movie")) TvType.AnimeMovie
else TvType.Anime
}
fun getStatus(t: String): ShowStatus {
return when (t) {
"Finished Airing" -> ShowStatus.Completed
"Currently Airing" -> ShowStatus.Ongoing
else -> ShowStatus.Completed
}
}
}
fun Element.toSearchResult(): SearchResponse? {
val href = fixUrl(this.select("a").attr("href"))
val title = this.select("h3.film-name").text()
if (href.contains("/news/") || title.trim().equals("News", ignoreCase = true)) return null
val posterUrl = fixUrl(this.select("img").attr("data-src"))
val type = getType(this.select("div.fd-infor > span.fdi-item").text())
return AnimeSearchResponse(
title,
href,
this@ZoroProvider.name,
type,
posterUrl,
null,
null,
EnumSet.of(DubStatus.Subbed),
null,
null
)
}
override fun getMainPage(): HomePageResponse {
val html = get("$mainUrl/home").text
val document = Jsoup.parse(html)
val homePageList = ArrayList<HomePageList>()
document.select("div.anif-block").forEach { block ->
val header = block.select("div.anif-block-header").text().trim()
val animes = block.select("li").mapNotNull {
it.toSearchResult()
}
if (animes.isNotEmpty()) homePageList.add(HomePageList(header, animes))
}
document.select("section.block_area.block_area_home").forEach { block ->
val header = block.select("h2.cat-heading").text().trim()
val animes = block.select("div.flw-item").mapNotNull {
it.toSearchResult()
}
if (animes.isNotEmpty()) homePageList.add(HomePageList(header, animes))
}
return HomePageResponse(homePageList)
}
private data class Response(
@JsonProperty("status") val status: Boolean,
@JsonProperty("html") val html: String
)
// override fun quickSearch(query: String): List<SearchResponse> {
// val url = "$mainUrl/ajax/search/suggest?keyword=${query}"
// val html = mapper.readValue<Response>(khttp.get(url).text).html
// val document = Jsoup.parse(html)
//
// return document.select("a.nav-item").map {
// val title = it.selectFirst(".film-name")?.text().toString()
// val href = fixUrl(it.attr("href"))
// val year = it.selectFirst(".film-infor > span")?.text()?.split(",")?.get(1)?.trim()?.toIntOrNull()
// val image = it.select("img").attr("data-src")
//
// AnimeSearchResponse(
// title,
// href,
// this.name,
// TvType.TvSeries,
// image,
// year,
// null,
// EnumSet.of(DubStatus.Subbed),
// null,
// null
// )
//
// }
// }
override fun search(query: String): List<SearchResponse> {
val link = "$mainUrl/search?keyword=$query"
val html = get(link).text
val document = Jsoup.parse(html)
return document.select(".flw-item").map {
val title = it.selectFirst(".film-detail > .film-name > a")?.attr("title").toString()
val poster = it.selectFirst(".film-poster > img")?.attr("data-src")
val tvType = getType(it.selectFirst(".film-detail > .fd-infor > .fdi-item")?.text().toString())
val href = fixUrl(it.selectFirst(".film-name a").attr("href"))
AnimeSearchResponse(
title,
href,
name,
tvType,
poster,
null,
null,
EnumSet.of(DubStatus.Subbed),
null,
null
)
}
}
override fun load(url: String): LoadResponse? {
val html = get(url).text
val document = Jsoup.parse(html)
val title = document.selectFirst(".anisc-detail > .film-name")?.text().toString()
val poster = document.selectFirst(".anisc-poster img")?.attr("src")
val tags = document.select(".anisc-info a[href*=\"/genre/\"]").map { it.text() }
var year: Int? = null
var japaneseTitle: String? = null
var status: ShowStatus? = null
for (info in document.select(".anisc-info > .item.item-title")) {
val text = info?.text().toString()
when {
(year != null && japaneseTitle != null && status != null) -> break
text.contains("Premiered") && year == null ->
year = info.selectFirst(".name")?.text().toString().split(" ").last().toIntOrNull()
text.contains("Japanese") && japaneseTitle == null ->
japaneseTitle = info.selectFirst(".name")?.text().toString()
text.contains("Status") && status == null ->
status = getStatus(info.selectFirst(".name")?.text().toString())
}
}
val description = document.selectFirst(".film-description.m-hide > .text")?.text()
val animeId = URI(url).path.split("-").last()
val episodes = Jsoup.parse(
mapper.readValue<Response>(
get(
"$mainUrl/ajax/v2/episode/list/$animeId"
).text
).html
).select(".ss-list > a[href].ssl-item.ep-item").map {
val name = it?.attr("title")
AnimeEpisode(
fixUrl(it.attr("href")),
name,
null,
null,
null,
null,
it.selectFirst(".ssli-order")?.text()?.toIntOrNull()
)
}
return AnimeLoadResponse(
title,
japaneseTitle,
title,
url,
this.name,
TvType.Anime,
poster,
year,
null,
episodes,
status,
description,
tags,
)
}
override fun loadLinks(
data: String,
isCasting: Boolean,
subtitleCallback: (SubtitleFile) -> Unit,
callback: (ExtractorLink) -> Unit
): Boolean {
// Copy pasted from Sflix :)
val sources = get(
data,
interceptor = WebViewResolver(
Regex("""/getSources""")
)
).text
val mapped = mapper.readValue<SflixProvider.SourceObject>(sources)
val list = listOf(
mapped.sources to "source 1",
mapped.sources1 to "source 2",
mapped.sources2 to "source 3",
mapped.sourcesBackup to "source backup"
)
list.forEach { subList ->
subList.first?.forEach {
it?.toExtractorLink(this, subList.second)?.forEach(callback)
}
}
mapped.tracks?.forEach {
it?.toSubtitleFile()?.let { subtitleFile ->
subtitleCallback.invoke(subtitleFile)
}
}
return true
}
}

View file

@ -1,6 +1,5 @@
package com.lagradost.cloudstream3.movieproviders
import android.net.Uri
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.module.kotlin.readValue
import com.lagradost.cloudstream3.*
@ -251,59 +250,20 @@ class SflixProvider : MainAPI() {
@JsonProperty("kind") val kind: String?
)
data class Sources1(
data class Sources(
@JsonProperty("file") val file: String?,
@JsonProperty("type") val type: String?,
@JsonProperty("label") val label: String?
)
data class SourceObject(
@JsonProperty("sources") val sources: List<Sources1?>?,
@JsonProperty("sources_1") val sources1: List<Sources1?>?,
@JsonProperty("sources_2") val sources2: List<Sources1?>?,
@JsonProperty("sourcesBackup") val sourcesBackup: List<Sources1?>?,
@JsonProperty("sources") val sources: List<Sources?>?,
@JsonProperty("sources_1") val sources1: List<Sources?>?,
@JsonProperty("sources_2") val sources2: List<Sources?>?,
@JsonProperty("sourcesBackup") val sourcesBackup: List<Sources?>?,
@JsonProperty("tracks") val tracks: List<Tracks?>?
)
private fun Sources1.toExtractorLink(name: String): List<ExtractorLink>? {
return this.file?.let {
val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals("hls", ignoreCase = true)
if (isM3u8) {
M3u8Helper().m3u8Generation(M3u8Helper.M3u8Stream(this.file, null), true).map { stream ->
val qualityString = if ((stream.quality ?: 0) == 0) label ?: "" else "${stream.quality}p"
ExtractorLink(
this@SflixProvider.name,
"${this@SflixProvider.name} $qualityString $name",
stream.streamUrl,
mainUrl,
getQualityFromName(stream.quality.toString()),
true
)
}
} else {
listOf(ExtractorLink(
this@SflixProvider.name,
this.label?.let { "${this@SflixProvider.name} - $it" } ?: this@SflixProvider.name,
it,
this@SflixProvider.mainUrl,
getQualityFromName(this.type ?: ""),
false,
))
}
}
}
private fun Tracks.toSubtitleFile(): SubtitleFile? {
return this.file?.let {
SubtitleFile(
this.label ?: "Unknown",
it
)
}
}
override fun loadLinks(
data: String,
isCasting: Boolean,
@ -339,14 +299,14 @@ class SflixProvider : MainAPI() {
val mapped = mapper.readValue<SourceObject>(sources)
val list = listOf(
mapped.sources1 to "source 1",
mapped.sources2 to "source 2",
mapped.sources to "source 0",
mapped.sourcesBackup to "source 3"
mapped.sources to "source 1",
mapped.sources1 to "source 2",
mapped.sources2 to "source 3",
mapped.sourcesBackup to "source backup"
)
list.forEach { subList ->
subList.first?.forEach {
it?.toExtractorLink(subList.second)?.forEach(callback)
it?.toExtractorLink(this, subList.second)?.forEach(callback)
}
}
mapped.tracks?.forEach {
@ -357,4 +317,47 @@ class SflixProvider : MainAPI() {
return true
}
companion object {
// For re-use in Zoro
fun Sources.toExtractorLink(caller: MainAPI, name: String): List<ExtractorLink>? {
return this.file?.let {
val isM3u8 = URI(this.file).path.endsWith(".m3u8") || this.type.equals("hls", ignoreCase = true)
if (isM3u8) {
M3u8Helper().m3u8Generation(M3u8Helper.M3u8Stream(this.file, null), true).map { stream ->
val qualityString = if ((stream.quality ?: 0) == 0) label ?: "" else "${stream.quality}p"
ExtractorLink(
caller.name,
"${caller.name} $qualityString $name",
stream.streamUrl,
caller.mainUrl,
getQualityFromName(stream.quality.toString()),
true
)
}
} else {
listOf(ExtractorLink(
caller.name,
this.label?.let { "${caller.name} - $it" } ?: caller.name,
it,
caller.mainUrl,
getQualityFromName(this.type ?: ""),
false,
))
}
}
}
fun Tracks.toSubtitleFile(): SubtitleFile? {
return this.file?.let {
SubtitleFile(
this.label ?: "Unknown",
it
)
}
}
}
}

View file

@ -111,7 +111,7 @@ class VMoveeProvider : MainAPI() {
}
}
return super.loadLinks(data, isCasting, subtitleCallback, callback)
return true
}
override fun load(url: String): LoadResponse {

View file

@ -39,11 +39,14 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor {
var fixedRequest: Request? = null
main {
// Useful for debugging
// WebView.setWebContentsDebuggingEnabled(true)
webView = WebView(
AcraApplication.context ?: throw RuntimeException("No base context in WebViewResolver")
).apply {
settings.cacheMode
// Bare minimum to bypass captcha
settings.javaScriptEnabled = true
settings.domStorageEnabled = true
}
webView?.webViewClient = object : WebViewClient() {
@ -52,6 +55,7 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor {
request: WebResourceRequest
): WebResourceResponse? {
val webViewUrl = request.url.toString()
// println("Override url $webViewUrl")
if (interceptUrl.containsMatchIn(webViewUrl)) {
fixedRequest = getRequestCreator(
webViewUrl,
@ -62,6 +66,7 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor {
10,
TimeUnit.MINUTES
)
println("Web-view request finished: $webViewUrl")
destroyWebView()
}
@ -77,8 +82,9 @@ class WebViewResolver(val interceptUrl: Regex) : Interceptor {
}
var loop = 0
// Timeouts after this amount, 20s
val totalTime = 20000L
// Timeouts after this amount, 60s
val totalTime = 60000L
val delayTime = 100L
// A bit sloppy, but couldn't find a better way

View file

@ -37,7 +37,9 @@ class APIRepository(val api: MainAPI) {
suspend fun search(query: String): Resource<List<SearchResponse>> {
return safeApiCall {
return@safeApiCall (api.search(query)
?: throw ErrorLoadingException()).filter { typesActive.contains(it.type) }.toList()
?: throw ErrorLoadingException())
// .filter { typesActive.contains(it.type) }
.toList()
}
}

View file

@ -21,6 +21,7 @@ import com.lagradost.cloudstream3.APIHolder.getApiProviderLangSettings
import com.lagradost.cloudstream3.APIHolder.getApiSettings
import com.lagradost.cloudstream3.APIHolder.getApiTypeSettings
import com.lagradost.cloudstream3.mvvm.Resource
import com.lagradost.cloudstream3.mvvm.logError
import com.lagradost.cloudstream3.mvvm.observe
import com.lagradost.cloudstream3.ui.APIRepository
import com.lagradost.cloudstream3.ui.APIRepository.Companion.providersActive
@ -35,6 +36,8 @@ import com.lagradost.cloudstream3.utils.UIHelper.fixPaddingStatusbar
import com.lagradost.cloudstream3.utils.UIHelper.getGridIsCompact
import com.lagradost.cloudstream3.utils.UIHelper.hideKeyboard
import kotlinx.android.synthetic.main.fragment_search.*
import java.lang.Exception
import java.util.concurrent.locks.ReentrantLock
class SearchFragment : Fragment() {
companion object {
@ -331,16 +334,25 @@ class SearchFragment : Fragment() {
}
}
val listLock = ReentrantLock()
observe(searchViewModel.currentSearch) { list ->
(search_master_recycler?.adapter as ParentItemAdapter?)?.apply {
items = list.map { ongoing ->
val ongoingList = HomePageList(
ongoing.apiName,
if (ongoing.data is Resource.Success) ongoing.data.value.filterSearchResponse() else ArrayList()
)
ongoingList
try {
// https://stackoverflow.com/questions/6866238/concurrent-modification-exception-adding-to-an-arraylist
listLock.lock()
(search_master_recycler?.adapter as ParentItemAdapter?)?.apply {
items = list.map { ongoing ->
val ongoingList = HomePageList(
ongoing.apiName,
if (ongoing.data is Resource.Success) ongoing.data.value.filterSearchResponse() else ArrayList()
)
ongoingList
}
notifyDataSetChanged()
}
notifyDataSetChanged()
} catch (e: Exception) {
logError(e)
} finally {
listLock.unlock()
}
}

View file

@ -6,11 +6,18 @@ import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import com.lagradost.cloudstream3.APIHolder.apis
import com.lagradost.cloudstream3.SearchResponse
import com.lagradost.cloudstream3.apmap
import com.lagradost.cloudstream3.mvvm.Resource
import com.lagradost.cloudstream3.pmap
import com.lagradost.cloudstream3.ui.APIRepository
import com.lagradost.cloudstream3.ui.APIRepository.Companion.providersActive
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import okhttp3.internal.notify
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.thread
data class OnGoingSearch(
val apiName: String,
@ -30,7 +37,7 @@ class SearchViewModel : ViewModel() {
_searchResponse.postValue(Resource.Success(ArrayList()))
}
var onGoingSearch : Job? = null
var onGoingSearch: Job? = null
fun searchAndCancel(query: String) {
onGoingSearch?.cancel()
onGoingSearch = search(query)
@ -48,11 +55,14 @@ class SearchViewModel : ViewModel() {
_currentSearch.postValue(ArrayList())
repos.filter { a ->
(providersActive.size == 0 || providersActive.contains(a.name))
}.map { a ->
currentList.add(OnGoingSearch(a.name, a.search(query)))
_currentSearch.postValue(currentList)
withContext(Dispatchers.IO) { // This interrupts UI otherwise
repos.filter { a ->
(providersActive.size == 0 || providersActive.contains(a.name))
}.apmap { a -> // Parallel
val search = a.search(query)
currentList.add(OnGoingSearch(a.name,search ))
_currentSearch.postValue(currentList)
}
}
_currentSearch.postValue(currentList)

View file

@ -40,7 +40,7 @@ class ProviderTests {
return true
}
private fun testSingleProviderApi(api: MainAPI) : Boolean {
private fun testSingleProviderApi(api: MainAPI): Boolean {
val searchQueries = listOf("over", "iron", "guy")
var correctResponses = 0
var searchResult: List<SearchResponse>? = null
@ -144,7 +144,7 @@ class ProviderTests {
@Test
fun providerCorrectHomepage() {
for (api in getAllProviders()) {
getAllProviders().pmap { api ->
if (api.hasMainPage) {
try {
val homepage = api.getMainPage()
@ -177,13 +177,13 @@ class ProviderTests {
@Test
fun providerCorrect() {
val providers = getAllProviders()
for ((index, api) in providers.withIndex()) {
providers.pmap { api ->
try {
println("Trying $api (${index + 1}/${providers.size})")
if(testSingleProviderApi(api)) {
println("Success $api (${index + 1}/${providers.size})")
println("Trying $api")
if (testSingleProviderApi(api)) {
println("Success $api")
} else {
System.err.println("Error $api (${index + 1}/${providers.size})")
System.err.println("Error $api")
}
} catch (e: Exception) {
logError(e)