From 68605fb563aa287465352b91abf1d8d8fa571e0b Mon Sep 17 00:00:00 2001 From: LagradOst Date: Wed, 19 May 2021 20:44:02 +0200 Subject: [PATCH] copy paste extractors from shiro --- .idea/uiDesigner.xml | 124 +++++++++++++++++ .../com/lagradost/cloudstream3/MainAPI.kt | 5 - .../cloudstream3/utils/ExtractorApi.kt | 58 ++++++++ .../cloudstream3/utils/JsUnpacker.kt | 125 ++++++++++++++++++ .../cloudstream3/utils/extractors/MixDrop.kt | 36 +++++ .../utils/extractors/Mp4Upload.kt | 32 +++++ .../utils/extractors/MultiQuality.kt | 69 ++++++++++ .../cloudstream3/utils/extractors/Shiro.kt | 33 +++++ .../utils/extractors/StreamTape.kt | 35 +++++ .../utils/extractors/Vidstream.kt | 59 +++++++++ .../utils/extractors/XStreamCdn.kt | 70 ++++++++++ 11 files changed, 641 insertions(+), 5 deletions(-) create mode 100644 .idea/uiDesigner.xml create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/JsUnpacker.kt create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/extractors/MixDrop.kt create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Mp4Upload.kt create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/extractors/MultiQuality.kt create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Shiro.kt create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/extractors/StreamTape.kt create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Vidstream.kt create mode 100644 app/src/main/java/com/lagradost/cloudstream3/utils/extractors/XStreamCdn.kt diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 00000000..e96534fb --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt b/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt index ebec8317..001ee701 100644 --- a/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt +++ b/app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt @@ -73,11 +73,6 @@ data class Link( val referer: String?, ) -interface LinkExtractor { - val linkStart: String // THIS IS USED TO AUTO-EXTRACT LINKS FROM URL - fun extract(link: String, referer: String): ArrayList -} - enum class ShowStatus { Completed, Ongoing, diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt new file mode 100644 index 00000000..be0e33e8 --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt @@ -0,0 +1,58 @@ +package com.lagradost.cloudstream3.utils + +import com.lagradost.cloudstream3.utils.extractors.MixDrop +import com.lagradost.cloudstream3.utils.extractors.Mp4Upload +import com.lagradost.cloudstream3.utils.extractors.Shiro +import com.lagradost.cloudstream3.cloudstream3.extractors.StreamTape +import com.lagradost.cloudstream3.utils.extractors.XStreamCdn + +data class ExtractorLink( + val name: String, + val url: String, + val referer: String, + val quality: Int, + val isM3u8: Boolean = false, +) + +enum class Qualities(var value: Int) { + Unknown(0), + SD(-1), // 360p - 480p + HD(1), // 720p + FullHd(2), // 1080p + UHD(3) // 4k +} + +private val packedRegex = Regex("""eval\(function\(p,a,c,k,e,.*\)\)""") +fun getPacked(string: String): String? { + return packedRegex.find(string)?.value +} + +fun getAndUnpack(string: String): String? { + val packedText = getPacked(string) + return JsUnpacker(packedText).unpack() +} + +val APIS: Array = arrayOf( + //AllProvider(), + Shiro(), + Mp4Upload(), + StreamTape(), + MixDrop(), + XStreamCdn() +) + +fun httpsify(url: String): String { + return if (url.startsWith("//")) "https:$url" else url +} + +abstract class ExtractorApi { + abstract val name: String + abstract val mainUrl: String + abstract val requiresReferer: Boolean + + abstract fun getUrl(url: String, referer: String? = null): List? + + open fun getExtractorUrl(id: String): String{ + return id + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/JsUnpacker.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/JsUnpacker.kt new file mode 100644 index 00000000..174f1971 --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/JsUnpacker.kt @@ -0,0 +1,125 @@ +package com.lagradost.cloudstream3.utils + +import java.lang.Exception +import java.lang.StringBuilder +import java.util.HashMap +import java.util.regex.Pattern +import kotlin.math.pow + +// https://github.com/cylonu87/JsUnpacker +class JsUnpacker(packedJS: String?) { + private var packedJS: String? = null + + /** + * Detects whether the javascript is P.A.C.K.E.R. coded. + * + * @return true if it's P.A.C.K.E.R. coded. + */ + fun detect(): Boolean { + val js = packedJS!!.replace(" ", "") + val p = Pattern.compile("eval\\(function\\(p,a,c,k,e,[rd]") + val m = p.matcher(js) + return m.find() + } + + /** + * Unpack the javascript + * + * @return the javascript unpacked or null. + */ + fun unpack(): String? { + val js = packedJS + try { + var p = + Pattern.compile("""\}\s*\('(.*)',\s*(.*?),\s*(\d+),\s*'(.*?)'\.split\('\|'\)""", Pattern.DOTALL) + var m = p.matcher(js) + if (m.find() && m.groupCount() == 4) { + val payload = m.group(1).replace("\\'", "'") + val radixStr = m.group(2) + val countStr = m.group(3) + val symtab = m.group(4).split("\\|".toRegex()).toTypedArray() + var radix = 36 + var count = 0 + try { + radix = radixStr.toInt() + } catch (e: Exception) { + } + try { + count = countStr.toInt() + } catch (e: Exception) { + } + if (symtab.size != count) { + throw Exception("Unknown p.a.c.k.e.r. encoding") + } + val unbase = Unbase(radix) + p = Pattern.compile("\\b\\w+\\b") + m = p.matcher(payload) + val decoded = StringBuilder(payload) + var replaceOffset = 0 + while (m.find()) { + val word = m.group(0) + val x = unbase.unbase(word) + var value: String? = null + if (x < symtab.size) { + value = symtab[x] + } + if (value != null && value.isNotEmpty()) { + decoded.replace(m.start() + replaceOffset, m.end() + replaceOffset, value) + replaceOffset += value.length - word.length + } + } + return decoded.toString() + } + } catch (e: Exception) { + e.printStackTrace() + } + return null + } + private inner class Unbase(private val radix: Int) { + private val ALPHABET_62 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + private val ALPHABET_95 = + " !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + private var alphabet: String? = null + private var dictionary: HashMap? = null + fun unbase(str: String): Int { + var ret = 0 + if (alphabet == null) { + ret = str.toInt(radix) + } else { + val tmp = StringBuilder(str).reverse().toString() + for (i in tmp.indices) { + ret += (radix.toDouble().pow(i.toDouble()) * dictionary!![tmp.substring(i, i + 1)]!!).toInt() + } + } + return ret + } + init { + if (radix > 36) { + when { + radix < 62 -> { + alphabet = ALPHABET_62.substring(0, radix) + } + radix in 63..94 -> { + alphabet = ALPHABET_95.substring(0, radix) + } + radix == 62 -> { + alphabet = ALPHABET_62 + } + radix == 95 -> { + alphabet = ALPHABET_95 + } + } + dictionary = HashMap(95) + for (i in 0 until alphabet!!.length) { + dictionary!![alphabet!!.substring(i, i + 1)] = i + } + } + } + } + /** + * @param packedJS javascript P.A.C.K.E.R. coded. + */ + init { + this.packedJS = packedJS + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/MixDrop.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/MixDrop.kt new file mode 100644 index 00000000..9340e042 --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/MixDrop.kt @@ -0,0 +1,36 @@ +package com.lagradost.cloudstream3.utils.extractors + +import com.lagradost.cloudstream3.utils.* + +class MixDrop : ExtractorApi() { + override val name: String = "MixDrop" + override val mainUrl: String = "https://mixdrop.co" + private val srcRegex = Regex("""wurl.*?=.*?"(.*?)";""") + override val requiresReferer = true + + override fun getExtractorUrl(id: String): String { + return "$mainUrl/e/$id" + } + + override fun getUrl(url: String, referer: String?): List? { + try { + with(khttp.get(url)) { + getAndUnpack(this.text)?.let { unpackedText -> + srcRegex.find(unpackedText)?.groupValues?.get(1)?.let { link -> + return listOf( + ExtractorLink( + name, + httpsify(link), + url, + Qualities.Unknown.value, + ) + ) + } + } + } + return null + } catch (e: Exception) { + return null + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Mp4Upload.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Mp4Upload.kt new file mode 100644 index 00000000..f7f7ae99 --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Mp4Upload.kt @@ -0,0 +1,32 @@ +package com.lagradost.cloudstream3.utils.extractors + +import com.lagradost.cloudstream3.utils.* + +class Mp4Upload : ExtractorApi() { + override val name: String = "Mp4Upload" + override val mainUrl: String = "https://www.mp4upload.com" + private val srcRegex = Regex("""player\.src\("(.*?)"""") + override val requiresReferer = true + + override fun getUrl(url: String, referer: String?): List? { + try { + with(khttp.get(url)) { + getAndUnpack(this.text)?.let { unpackedText -> + srcRegex.find(unpackedText)?.groupValues?.get(1)?.let { link -> + return listOf( + ExtractorLink( + name, + link, + url, + Qualities.Unknown.value, + ) + ) + } + } + } + return null + } catch (e: Exception) { + return null + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/MultiQuality.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/MultiQuality.kt new file mode 100644 index 00000000..93c2b603 --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/MultiQuality.kt @@ -0,0 +1,69 @@ +package com.lagradost.cloudstream3.utils.extractors + +import com.lagradost.cloudstream3.utils.ExtractorApi +import com.lagradost.cloudstream3.utils.ExtractorLink +import com.lagradost.cloudstream3.utils.Qualities + +class MultiQuality : ExtractorApi() { + override val name: String = "MultiQuality" + override val mainUrl: String = "https://gogo-play.net" + private val sourceRegex = Regex("""file:\s*'(.*?)',label:\s*'(.*?)'""") + private val m3u8Regex = Regex(""".*?(\d*).m3u8""") + private val urlRegex = Regex("""(.*?)([^/]+$)""") + override val requiresReferer = false + + override fun getExtractorUrl(id: String): String { + return "$mainUrl/loadserver.php?id=$id" + } + + private fun getQuality(string: String): Int { + return when (string) { + "360" -> Qualities.SD.value + "480" -> Qualities.SD.value + "720" -> Qualities.HD.value + "1080" -> Qualities.FullHd.value + else -> Qualities.Unknown.value + } + } + + override fun getUrl(url: String, referer: String?): List? { + try { + val extractedLinksList: MutableList = mutableListOf() + with(khttp.get(url)) { + sourceRegex.findAll(this.text).forEach { sourceMatch -> + val extractedUrl = sourceMatch.groupValues[1] + // Trusting this isn't mp4, may fuck up stuff + if (extractedUrl.endsWith(".m3u8")) { + with(khttp.get(extractedUrl)) { + m3u8Regex.findAll(this.text).forEach { match -> + extractedLinksList.add( + ExtractorLink( + "$name ${match.groupValues[1]}p", + urlRegex.find(this.url)!!.groupValues[1] + match.groupValues[0], + url, + getQuality(match.groupValues[1]), + isM3u8 = true + ) + ) + } + + } + } else if (extractedUrl.endsWith(".mp4")) { + extractedLinksList.add( + ExtractorLink( + "$name ${sourceMatch.groupValues[2]}", + extractedUrl, + url.replace(" ", "%20"), + Qualities.Unknown.value, + ) + ) + } + } + return extractedLinksList + } + } catch (e: Exception) { + + } + return null + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Shiro.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Shiro.kt new file mode 100644 index 00000000..903e017f --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Shiro.kt @@ -0,0 +1,33 @@ +package com.lagradost.cloudstream3.utils.extractors + +import com.lagradost.cloudstream3.utils.ExtractorApi +import com.lagradost.cloudstream3.utils.ExtractorLink +import com.lagradost.cloudstream3.utils.Qualities +import org.jsoup.Jsoup + +class Shiro : ExtractorApi() { + override val name: String = "Shiro" + override val mainUrl: String = "https://cherry.subsplea.se" + override val requiresReferer = false + + override fun getExtractorUrl(id: String): String { + return "$mainUrl/$id" + } + + override fun getUrl(url: String, referer: String?): List? { + val headers = mapOf("Referer" to "https://shiro.is/") + val res = khttp.get(url, headers = headers).text + Jsoup.parse(res).select("source").firstOrNull()?.attr("src")?.replace("&", "?")?.let { + return listOf( + ExtractorLink( + name, + it.replace(" ", "%20"), + "https://cherry.subsplea.se/", + // UHD to give top priority + Qualities.UHD.value + ) + ) + } + return null + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/StreamTape.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/StreamTape.kt new file mode 100644 index 00000000..a8c15955 --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/StreamTape.kt @@ -0,0 +1,35 @@ +package com.lagradost.cloudstream3.cloudstream3.extractors + +import com.lagradost.cloudstream3.utils.ExtractorApi +import com.lagradost.cloudstream3.utils.ExtractorLink +import com.lagradost.cloudstream3.utils.Qualities + +class StreamTape : ExtractorApi() { + override val name: String = "StreamTape" + override val mainUrl: String = "https://streamtape.com" + override val requiresReferer = true + + // Because they add concatenation to fuck up scrapers + private val linkRegex = + Regex("""(i(|" \+ ')d(|" \+ ')=.*?&(|" \+ ')e(|" \+ ')x(|" \+ ')p(|" \+ ')i(|" \+ ')r(|" \+ ')e(|" \+ ')s(|" \+ ')=.*?&(|" \+ ')i(|" \+ ')p(|" \+ ')=.*?&(|" \+ ')t(|" \+ ')o(|" \+ ')k(|" \+ ')e(|" \+ ')n(|" \+ ')=.*)'""") + + override fun getUrl(url: String, referer: String?): List? { + try { + with(khttp.get(url)) { + linkRegex.find(this.text)?.let { + val extractedUrl = "https://streamtape.com/get_video?${it.groupValues[1]}".replace("""" + '""", "") + return listOf( + ExtractorLink( + name, + extractedUrl, + url, + Qualities.Unknown.value, + ) + ) + } + } + } catch (e: Exception) { + } + return null + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Vidstream.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Vidstream.kt new file mode 100644 index 00000000..2a3abebc --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/Vidstream.kt @@ -0,0 +1,59 @@ +package com.lagradost.cloudstream3.utils.extractors + +import com.lagradost.cloudstream3.utils.ExtractorLink +import com.lagradost.cloudstream3.utils.APIS +import com.lagradost.cloudstream3.utils.extractors.MultiQuality +import com.lagradost.cloudstream3.utils.extractors.Shiro +import org.jsoup.Jsoup + +class Vidstream { + val name: String = "Vidstream" + private val mainUrl: String = "https://gogo-stream.com" + + private fun getExtractorUrl(id: String): String { + return "$mainUrl/streaming.php?id=$id" + } + + // https://gogo-stream.com/streaming.php?id=MTE3NDg5 + fun getUrl(id: String, isCasting: Boolean = false): List { + try { + val url = getExtractorUrl(id) + with(khttp.get(url)) { + val document = Jsoup.parse(this.text) + val primaryLinks = document.select("ul.list-server-items > li.linkserver") + val extractedLinksList: MutableList = mutableListOf() + + // --- Shiro --- + val shiroUrl = Shiro().getExtractorUrl(id) + val shiroSource = Shiro().getUrl(shiroUrl) + shiroSource?.forEach { extractedLinksList.add(it) } + // --- MultiQuality --- + val multiQualityUrl = MultiQuality().getExtractorUrl(id) + val multiQualitySource = MultiQuality().getUrl(multiQualityUrl) + multiQualitySource?.forEach { extractedLinksList.add(it) } + // -------------------- + + // All vidstream links passed to extractors + primaryLinks.forEach { element -> + val link = element.attr("data-video") + //val name = element.text() + + // Matches vidstream links with extractors + APIS.filter { !it.requiresReferer || !isCasting}.forEach { api -> + if (link.startsWith(api.mainUrl)) { + val extractedLinks = api.getUrl(link, url) + if (extractedLinks?.isNotEmpty() == true) { + extractedLinks.forEach { + extractedLinksList.add(it) + } + } + } + } + } + return extractedLinksList + } + } catch (e: Exception) { + return listOf() + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/XStreamCdn.kt b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/XStreamCdn.kt new file mode 100644 index 00000000..f0af4ce5 --- /dev/null +++ b/app/src/main/java/com/lagradost/cloudstream3/utils/extractors/XStreamCdn.kt @@ -0,0 +1,70 @@ +package com.lagradost.cloudstream3.utils.extractors + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.module.kotlin.readValue +import com.lagradost.cloudstream3.mapper +import com.lagradost.cloudstream3.utils.ExtractorApi +import com.lagradost.cloudstream3.utils.ExtractorLink +import com.lagradost.cloudstream3.utils.Qualities + +class XStreamCdn : ExtractorApi() { + override val name: String = "XStreamCdn" + override val mainUrl: String = "https://fcdn.stream" + override val requiresReferer = true + + private data class ResponseData( + @JsonProperty("file") val file: String, + @JsonProperty("label") val label: String, + //val type: String // Mp4 + ) + + private data class ResponseJson( + @JsonProperty("success") val success: Boolean, + @JsonProperty("data") val data: List? + ) + + override fun getExtractorUrl(id: String): String { + return "$mainUrl/api/source/$id" + } + + private fun getQuality(string: String): Int { + return when (string) { + "360p" -> Qualities.SD.value + "480p" -> Qualities.SD.value + "720p" -> Qualities.HD.value + "1080p" -> Qualities.FullHd.value + else -> Qualities.Unknown.value + } + } + + override fun getUrl(url: String, referer: String?): List? { + try { + val headers = mapOf( + "Referer" to url, + "User-Agent" to "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0", + ) + val newUrl = url.replace("$mainUrl/v/", "$mainUrl/api/source/") + val extractedLinksList: MutableList = mutableListOf() + with(khttp.post(newUrl, headers = headers)) { + mapper.readValue(this.text)?.let { + if (it.success && it.data != null) { + it.data.forEach { data -> + extractedLinksList.add( + ExtractorLink( + "$name ${data.label}", + data.file, + url, + getQuality(data.label), + ) + ) + } + } + } + } + return extractedLinksList + } catch (e: Exception) { + } + return null + } + +} \ No newline at end of file