fuck it we ball, m3u8 download is now fixed

This commit is contained in:
LagradOst 2023-08-17 23:10:21 +02:00
parent c2b951a078
commit 590c74111c
3 changed files with 237 additions and 276 deletions

View file

@ -1,13 +1,11 @@
package com.lagradost.cloudstream3.extractors
import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.utils.Qualities
import com.lagradost.cloudstream3.USER_AGENT
import com.lagradost.cloudstream3.app
import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
import com.lagradost.cloudstream3.utils.ExtractorApi
import com.lagradost.cloudstream3.utils.ExtractorLink
import com.lagradost.cloudstream3.utils.M3u8Helper.Companion.generateM3u8
import com.fasterxml.jackson.annotation.JsonProperty
import com.lagradost.cloudstream3.utils.AppUtils.tryParseJson
import android.util.Log
import com.lagradost.cloudstream3.utils.Qualities
import java.net.URLDecoder
open class Cda: ExtractorApi() {

View file

@ -1,17 +1,16 @@
package com.lagradost.cloudstream3.utils
import com.lagradost.cloudstream3.ErrorLoadingException
import com.lagradost.cloudstream3.app
import com.lagradost.cloudstream3.mvvm.logError
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.delay
import javax.crypto.Cipher
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
import kotlin.math.pow
/** backwards api surface */
class M3u8Helper {
companion object {
private val generator = M3u8Helper()
suspend fun generateM3u8(
source: String,
streamUrl: String,
@ -20,8 +19,33 @@ class M3u8Helper {
headers: Map<String, String> = mapOf(),
name: String = source
): List<ExtractorLink> {
return generator.m3u8Generation(
M3u8Stream(
return M3u8Helper2.generateM3u8(source, streamUrl, referer, quality, headers, name)
}
}
data class M3u8Stream(
val streamUrl: String,
val quality: Int? = null,
val headers: Map<String, String> = mapOf()
)
suspend fun m3u8Generation(m3u8: M3u8Stream, returnThis: Boolean? = true): List<M3u8Stream> {
return M3u8Helper2.m3u8Generation(m3u8, returnThis)
}
}
object M3u8Helper2 {
suspend fun generateM3u8(
source: String,
streamUrl: String,
referer: String,
quality: Int? = null,
headers: Map<String, String> = mapOf(),
name: String = source
): List<ExtractorLink> {
return m3u8Generation(
M3u8Helper.M3u8Stream(
streamUrl = streamUrl,
quality = quality,
headers = headers,
@ -39,7 +63,6 @@ class M3u8Helper {
)
}
}
}
private val ENCRYPTION_DETECTION_REGEX = Regex("#EXT-X-KEY:METHOD=([^,]+),")
private val ENCRYPTION_URL_IV_REGEX =
@ -47,7 +70,8 @@ class M3u8Helper {
private val QUALITY_REGEX =
Regex("""#EXT-X-STREAM-INF:(?:(?:.*?(?:RESOLUTION=\d+x(\d+)).*?\s+(.*))|(?:.*?\s+(.*)))""")
private val TS_EXTENSION_REGEX =
Regex("""(.*\.ts.*|.*\.jpg.*)""") //.jpg here 'case vizcloud uses .jpg instead of .ts
Regex("""#EXTINF:.*\n(.+?\n)""") // fuck it we ball, who cares about the type anyways
//Regex("""(.*\.(ts|jpg|html).*)""") //.jpg here 'case vizcloud uses .jpg instead of .ts
private fun absoluteExtensionDetermination(url: String): String? {
val split = url.split("/")
@ -73,7 +97,7 @@ class M3u8Helper {
}
}.iterator()
private fun getDecrypter(
fun getDecrypter(
secretKey: ByteArray,
data: ByteArray,
iv: ByteArray = "".toByteArray()
@ -91,13 +115,8 @@ class M3u8Helper {
return st != null && (st.value.isNotEmpty() || st.destructured.component1() != "NONE")
}
data class M3u8Stream(
val streamUrl: String,
val quality: Int? = null,
val headers: Map<String, String> = mapOf()
)
private fun selectBest(qualities: List<M3u8Stream>): M3u8Stream? {
private fun selectBest(qualities: List<M3u8Helper.M3u8Stream>): M3u8Helper.M3u8Stream? {
val result = qualities.sortedBy {
if (it.quality != null && it.quality <= 1080) it.quality else 0
}.filter {
@ -113,19 +132,16 @@ class M3u8Helper {
}
private fun isNotCompleteUrl(url: String): Boolean {
return !url.contains("https://") && !url.contains("http://")
return !url.startsWith("https://") && !url.startsWith("http://")
}
suspend fun m3u8Generation(m3u8: M3u8Stream, returnThis: Boolean? = true): List<M3u8Stream> {
// return listOf(m3u8)
val list = mutableListOf<M3u8Stream>()
suspend fun m3u8Generation(m3u8: M3u8Helper.M3u8Stream, returnThis: Boolean? = true): List<M3u8Helper.M3u8Stream> {
val list = mutableListOf<M3u8Helper.M3u8Stream>()
val m3u8Parent = getParentLink(m3u8.streamUrl)
val response = app.get(m3u8.streamUrl, headers = m3u8.headers, verify = false).text
// var hasAnyContent = false
for (match in QUALITY_REGEX.findAll(response)) {
// hasAnyContent = true
var (quality, m3u8Link, m3u8Link2) = match.destructured
if (m3u8Link.isEmpty()) m3u8Link = m3u8Link2
if (absoluteExtensionDetermination(m3u8Link) == "m3u8") {
@ -136,21 +152,21 @@ class M3u8Helper {
println(m3u8.streamUrl)
}
list += m3u8Generation(
M3u8Stream(
M3u8Helper.M3u8Stream(
m3u8Link,
quality.toIntOrNull(),
m3u8.headers
), false
)
}
list += M3u8Stream(
list += M3u8Helper.M3u8Stream(
m3u8Link,
quality.toIntOrNull(),
m3u8.headers
)
}
if (returnThis != false) {
list += M3u8Stream(
list += M3u8Helper.M3u8Stream(
m3u8.streamUrl,
Qualities.Unknown.value,
m3u8.headers
@ -160,113 +176,111 @@ class M3u8Helper {
return list
}
data class LazyHlsDownloadData(
private val encryptionData: ByteArray,
private val encryptionIv: ByteArray,
private val isEncrypted: Boolean,
private val allTsLinks: List<String>,
private val relativeUrl: String,
private val headers: Map<String, String>,
) {
val size get() = allTsLinks.size
data class HlsDownloadData(
val bytes: ByteArray,
val currentIndex: Int,
val totalTs: Int,
val errored: Boolean = false
)
suspend fun hlsYield(
qualities: List<M3u8Stream>,
startIndex: Int = 0
): Iterator<HlsDownloadData> {
if (qualities.isEmpty()) return listOf(
HlsDownloadData(
byteArrayOf(),
1,
1,
true
)
).iterator()
var selected = selectBest(qualities)
if (selected == null) {
selected = qualities[0]
suspend fun resolveLinkSafe(
index: Int,
tries: Int = 3,
failDelay: Long = 3000
): ByteArray? {
for (i in 0 until tries) {
try {
return resolveLink(index)
} catch (e: IllegalArgumentException) {
return null
} catch (t: Throwable) {
delay(failDelay)
}
}
return null
}
@Throws
suspend fun resolveLink(index: Int): ByteArray {
if (index < 0 || index >= size) throw IllegalArgumentException("index must be in the bounds of the ts")
val url = allTsLinks[index]
val tsResponse = app.get(url, headers = headers, verify = false)
val tsData = tsResponse.body.bytes()
if (tsData.isEmpty()) throw ErrorLoadingException("no data")
return if (isEncrypted) {
getDecrypter(encryptionData, tsData, encryptionIv)
} else {
tsData
}
}
}
@Throws
suspend fun hslLazy(
qualities: List<M3u8Helper.M3u8Stream>
): LazyHlsDownloadData {
if (qualities.isEmpty()) throw IllegalArgumentException("qualities must be non empty")
val selected = selectBest(qualities) ?: qualities.first()
val headers = selected.headers
val streams = qualities.map { m3u8Generation(it, false) }.flatten()
//val sslVerification = if (headers.containsKey("ssl_verification")) headers["ssl_verification"].toBoolean() else true
// this selects the best quality of the qualities offered,
// due to the recursive nature of m3u8, we only go 2 depth
val secondSelection = selectBest(streams.ifEmpty { listOf(selected) })
if (secondSelection != null) {
?: throw IllegalArgumentException("qualities has no streams")
val m3u8Response =
runBlocking {
app.get(
secondSelection.streamUrl,
headers = headers,
verify = false
).text
}
var encryptionUri: String?
println("m3u8Response=$m3u8Response")
// encryption, this is because crunchy uses it
var encryptionIv = byteArrayOf()
var encryptionData = byteArrayOf()
val encryptionState = isEncrypted(m3u8Response)
if (encryptionState) {
// its safe to assume that its not going to be null
val match =
ENCRYPTION_URL_IV_REGEX.find(m3u8Response)!!.destructured // its safe to assume that its not going to be null
encryptionUri = match.component2()
ENCRYPTION_URL_IV_REGEX.find(m3u8Response)!!.groupValues
var encryptionUri = match[1]
if (isNotCompleteUrl(encryptionUri)) {
encryptionUri = "${getParentLink(secondSelection.streamUrl)}/$encryptionUri"
}
encryptionIv = match.component3().toByteArray()
val encryptionKeyResponse =
runBlocking { app.get(encryptionUri, headers = headers, verify = false) }
encryptionData = encryptionKeyResponse.body?.bytes() ?: byteArrayOf()
encryptionIv = match[2].toByteArray()
val encryptionKeyResponse = app.get(encryptionUri, headers = headers, verify = false)
encryptionData = encryptionKeyResponse.body.bytes()
}
val allTs = TS_EXTENSION_REGEX.findAll(m3u8Response)
val allTsList = allTs.toList()
val totalTs = allTsList.size
if (totalTs == 0) {
return listOf(HlsDownloadData(byteArrayOf(), 1, 1, true)).iterator()
}
var lastYield = 0
val relativeUrl = getParentLink(secondSelection.streamUrl)
var retries = 0
val tsByteGen = sequence {
loop@ for ((index, ts) in allTs.withIndex()) {
val url = if (
isNotCompleteUrl(ts.destructured.component1())
) "$relativeUrl/${ts.destructured.component1()}" else ts.destructured.component1()
val c = index + 1 + startIndex
val allTsList = TS_EXTENSION_REGEX.findAll(m3u8Response + "\n").map { ts ->
val value = ts.groupValues[1]
if (isNotCompleteUrl(value)) {
"$relativeUrl/${value}"
} else {
value
}
}.toList()
if (allTsList.isEmpty()) throw IllegalArgumentException("ts must be non empty")
while (lastYield != c) {
try {
val tsResponse =
runBlocking { app.get(url, headers = headers, verify = false) }
var tsData = tsResponse.body?.bytes() ?: byteArrayOf()
if (encryptionState) {
tsData = getDecrypter(encryptionData, tsData, encryptionIv)
yield(HlsDownloadData(tsData, c, totalTs))
lastYield = c
break
}
yield(HlsDownloadData(tsData, c, totalTs))
lastYield = c
} catch (e: Exception) {
logError(e)
if (retries == 3) {
yield(HlsDownloadData(byteArrayOf(), c, totalTs, true))
break@loop
}
++retries
Thread.sleep(2_000)
}
}
}
}
return tsByteGen.iterator()
}
return listOf(HlsDownloadData(byteArrayOf(), 1, 1, true)).iterator()
return LazyHlsDownloadData(
encryptionData = encryptionData,
encryptionIv = encryptionIv,
isEncrypted = encryptionState,
allTsLinks = allTsList,
relativeUrl = relativeUrl,
headers = headers
)
}
}

View file

@ -15,7 +15,6 @@ import androidx.annotation.RequiresApi
import androidx.core.app.NotificationCompat
import androidx.core.app.NotificationManagerCompat
import androidx.core.net.toUri
import androidx.lifecycle.LifecycleOwner
import androidx.lifecycle.MutableLiveData
import androidx.preference.PreferenceManager
import androidx.work.Data
@ -32,18 +31,15 @@ import com.lagradost.cloudstream3.MainActivity
import com.lagradost.cloudstream3.R
import com.lagradost.cloudstream3.TvType
import com.lagradost.cloudstream3.mvvm.logError
import com.lagradost.cloudstream3.mvvm.normalSafeApiCall
import com.lagradost.cloudstream3.mvvm.suspendSafeApiCall
import com.lagradost.cloudstream3.services.VideoDownloadService
import com.lagradost.cloudstream3.utils.Coroutines.ioSafe
import com.lagradost.cloudstream3.utils.Coroutines.main
import com.lagradost.cloudstream3.utils.DataStore.getKey
import com.lagradost.cloudstream3.utils.DataStore.removeKey
import com.lagradost.cloudstream3.utils.UIHelper.colorFromAttribute
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import okhttp3.internal.closeQuietly
import java.io.BufferedInputStream
@ -51,11 +47,9 @@ import java.io.File
import java.io.InputStream
import java.io.OutputStream
import java.lang.Thread.sleep
import java.net.URI
import java.net.URL
import java.net.URLConnection
import java.util.*
import kotlin.math.roundToInt
const val DOWNLOAD_CHANNEL_ID = "cloudstream3.general"
const val DOWNLOAD_CHANNEL_NAME = "Downloads"
@ -687,7 +681,8 @@ object VideoDownloadManager {
return StreamData(SUCCESS_STREAM, resume, fileLength, fileStream)
}
fun downloadThing(
@Throws
suspend fun downloadThing(
context: Context,
link: IDownloadableMinimum,
name: String,
@ -696,9 +691,9 @@ object VideoDownloadManager {
tryResume: Boolean,
parentId: Int?,
createNotificationCallback: (CreateNotificationMetadata) -> Unit,
): Int {
): Int = withContext(Dispatchers.IO) {
if (link.url.startsWith("magnet") || link.url.endsWith(".torrent")) {
return ERROR_UNKNOWN
return@withContext ERROR_UNKNOWN
}
val basePath = context.getBasePath()
@ -714,7 +709,7 @@ object VideoDownloadManager {
}
val stream = setupStream(context, name, relativePath, extension, tryResume)
if (stream.errorCode != SUCCESS_STREAM) return stream.errorCode
if (stream.errorCode != SUCCESS_STREAM) return@withContext stream.errorCode
val resume = stream.resume!!
val fileStream = stream.fileStream!!
@ -766,7 +761,7 @@ object VideoDownloadManager {
}
val bytesTotal = contentLength + resumeLength
if (extension == "mp4" && bytesTotal < 5000000) return ERROR_TOO_SMALL_CONNECTION // DATA IS LESS THAN 5MB, SOMETHING IS WRONG
if (extension == "mp4" && bytesTotal < 5000000) return@withContext ERROR_TOO_SMALL_CONNECTION // DATA IS LESS THAN 5MB, SOMETHING IS WRONG
parentId?.let {
setKey(
@ -845,11 +840,13 @@ object VideoDownloadManager {
DownloadActionType.Pause -> {
isPaused = true; updateNotification()
}
DownloadActionType.Stop -> {
isStopped = true; updateNotification()
removeKey(KEY_RESUME_PACKAGES, event.first.toString())
saveQueue()
}
DownloadActionType.Resume -> {
isPaused = false; updateNotification()
}
@ -917,15 +914,17 @@ object VideoDownloadManager {
}
// RETURN MESSAGE
return when {
return@withContext when {
isFailed -> {
parentId?.let { id -> downloadProgressEvent.invoke(Triple(id, 0, 0)) }
ERROR_CONNECTION_ERROR
}
isStopped -> {
parentId?.let { id -> downloadProgressEvent.invoke(Triple(id, 0, 0)) }
deleteFile()
}
else -> {
parentId?.let { id ->
downloadProgressEvent.invoke(
@ -989,6 +988,7 @@ object VideoDownloadManager {
found.delete()
this.createDirectory(directoryName)
}
this.isDirectory -> this.createDirectory(directoryName)
else -> this.parentFile?.createDirectory(directoryName)
}
@ -1107,7 +1107,8 @@ object VideoDownloadManager {
return SUCCESS_STOPPED
}
private fun downloadHLS(
@Throws
private suspend fun downloadHLS(
context: Context,
link: ExtractorLink,
name: String,
@ -1115,16 +1116,8 @@ object VideoDownloadManager {
parentId: Int?,
startIndex: Int?,
createNotificationCallback: (CreateNotificationMetadata) -> Unit
): Int {
): Int = withContext(Dispatchers.IO) {
val extension = "mp4"
fun logcatPrint(vararg items: Any?) {
items.forEach {
println("[HLS]: $it")
}
}
val m3u8Helper = M3u8Helper()
logcatPrint("initialised the HLS downloader.")
val m3u8 = M3u8Helper.M3u8Stream(
link.url, link.quality, mapOf("referer" to link.referer)
@ -1139,55 +1132,41 @@ object VideoDownloadManager {
) else folder
val stream = setupStream(context, name, relativePath, extension, realIndex > 0)
if (stream.errorCode != SUCCESS_STREAM) return stream.errorCode
if (stream.errorCode != SUCCESS_STREAM) return@withContext stream.errorCode
if (!stream.resume!!) realIndex = 0
val fileLengthAdd = stream.fileLength!!
val tsIterator = runBlocking {
m3u8Helper.hlsYield(listOf(m3u8), realIndex)
}
if (stream.resume != true) realIndex = 0
val fileLengthAdd = stream.fileLength ?: 0
val items = M3u8Helper2.hslLazy(listOf(m3u8))
val displayName = getDisplayName(name, extension)
val fileStream = stream.fileStream!!
val firstTs = tsIterator.next()
var isDone = false
var isFailed = false
var isPaused = false
var bytesDownloaded = firstTs.bytes.size.toLong() + fileLengthAdd
var tsProgress = 1L + realIndex
val totalTs = firstTs.totalTs.toLong()
var bytesDownloaded = fileLengthAdd
var tsProgress: Long = realIndex.toLong() + 1 // we don't want div by zero
val totalTs: Long = items.size.toLong()
fun deleteFile(): Int {
return delete(context, name, relativePath, extension, parentId, basePath.first)
}
/*
Most of the auto generated m3u8 out there have TS of the same size.
And only the last TS might have a different size.
But oh well, in cases of handmade m3u8 streams this will go all over the place ¯\_()_/¯
So ya, this calculates an estimate of how many bytes the file is going to be.
> (bytesDownloaded/tsProgress)*totalTs
*/
fun updateInfo() {
parentId?.let {
setKey(
KEY_DOWNLOAD_INFO,
it.toString(),
(parentId ?: return).toString(),
DownloadedFileInfo(
(bytesDownloaded * (totalTs / tsProgress.toFloat())).toLong(),
relativePath ?: "",
displayName,
tsProgress.toString(),
// approx bytes
totalBytes = (bytesDownloaded * (totalTs / tsProgress.toFloat())).toLong(),
relativePath = relativePath ?: "",
displayName = displayName,
extraInfo = tsProgress.toString(),
basePath = basePath.second
)
)
}
}
updateInfo()
@ -1210,9 +1189,7 @@ object VideoDownloadManager {
(bytesDownloaded * (totalTs / tsProgress.toFloat())).toLong(),
)
)
} catch (e: Exception) {
// IDK MIGHT ERROR
}
} catch (_: Throwable) {}
}
createNotificationCallback.invoke(
@ -1226,24 +1203,6 @@ object VideoDownloadManager {
)
}
fun stopIfError(ts: M3u8Helper.HlsDownloadData): Int? {
if (ts.errored || ts.bytes.isEmpty()) {
val error: Int = if (!ts.errored) {
logcatPrint("Error: No stream was found.")
ERROR_UNKNOWN
} else {
logcatPrint("Error: Failed to fetch data.")
ERROR_CONNECTION_ERROR
}
isFailed = true
fileStream.close()
deleteFile()
updateNotification()
return error
}
return null
}
val notificationCoroutine = main {
while (true) {
if (!isDone) {
@ -1261,11 +1220,11 @@ object VideoDownloadManager {
DownloadActionType.Stop -> {
isFailed = true
}
DownloadActionType.Pause -> {
isPaused =
true // Pausing is not supported since well...I need to know the index of the ts it was paused at
// it may be possible to store it in a variable, but when the app restarts it will be lost
isPaused = true
}
DownloadActionType.Resume -> {
isPaused = false
}
@ -1278,32 +1237,22 @@ object VideoDownloadManager {
try {
if (parentId != null)
downloadEvent -= downloadEventListener
} catch (e: Exception) {
logError(e)
} catch (t: Throwable) {
logError(t)
}
try {
parentId?.let {
downloadStatus.remove(it)
}
} catch (e: Exception) {
logError(e)
// IDK MIGHT ERROR
} catch (t: Throwable) {
logError(t)
}
notificationCoroutine.cancel()
}
stopIfError(firstTs).let {
if (it != null) {
closeAll()
return it
}
}
if (parentId != null)
downloadEvent += downloadEventListener
fileStream.write(firstTs.bytes)
fun onFailed() {
fileStream.close()
deleteFile()
@ -1311,31 +1260,29 @@ object VideoDownloadManager {
closeAll()
}
for (ts in tsIterator) {
for (idx in realIndex until items.size) {
while (isPaused) {
if (isFailed) {
onFailed()
return SUCCESS_STOPPED
return@withContext SUCCESS_STOPPED
}
sleep(100)
delay(100)
}
if (isFailed) {
onFailed()
return SUCCESS_STOPPED
return@withContext SUCCESS_STOPPED
}
stopIfError(ts).let {
if (it != null) {
closeAll()
return it
}
val bytes = items.resolveLinkSafe(idx) ?: run {
isFailed = true
onFailed()
return@withContext ERROR_CONNECTION_ERROR
}
fileStream.write(ts.bytes)
tsProgress = ts.currentIndex.toLong()
bytesDownloaded += ts.bytes.size.toLong()
logcatPrint("Download progress ${((tsProgress.toFloat() / totalTs.toFloat()) * 100).roundToInt()}%")
fileStream.write(bytes)
tsProgress = idx.toLong() + 1
bytesDownloaded += bytes.size.toLong()
updateInfo()
}
isDone = true
@ -1344,7 +1291,7 @@ object VideoDownloadManager {
closeAll()
updateInfo()
return SUCCESS_DOWNLOAD_DONE
return@withContext SUCCESS_DOWNLOAD_DONE
}
fun getFileName(context: Context, metadata: DownloadEpisodeMetadata): String {
@ -1379,7 +1326,7 @@ object VideoDownloadManager {
)
}
private fun downloadSingleEpisode(
private suspend fun downloadSingleEpisode(
context: Context,
source: String?,
folder: String?,
@ -1405,7 +1352,8 @@ object VideoDownloadManager {
null
)?.extraInfo?.toIntOrNull()
} else null
return downloadHLS(context, link, name, folder, ep.id, startIndex) { meta ->
return suspendSafeApiCall {
downloadHLS(context, link, name, folder, ep.id, startIndex) { meta ->
main {
createNotification(
context,
@ -1420,10 +1368,13 @@ object VideoDownloadManager {
meta.hlsTotal
)
}
}.also { extractorJob.cancel() }
}
}.also {
extractorJob.cancel()
} ?: ERROR_UNKNOWN
}
return normalSafeApiCall {
return suspendSafeApiCall {
downloadThing(context, link, name, folder, "mp4", tryResume, ep.id) { meta ->
main {
createNotification(
@ -1468,7 +1419,6 @@ object VideoDownloadManager {
DownloadResumePackage(item, index)
)
val connectionResult = withContext(Dispatchers.IO) {
normalSafeApiCall {
downloadSingleEpisode(
context,
item.source,
@ -1479,7 +1429,6 @@ object VideoDownloadManager {
resume
).also { println("Single episode finished with return code: $it") }
}
}
if (connectionResult != null && connectionResult > 0) { // SUCCESS
removeKey(KEY_RESUME_PACKAGES, id.toString())
break