Merge pull request #50 from LagradOst/hls

Added download support for HLS

app/src/main/java/com/lagradost/cloudstream3/utils/M3u8Helper.kt

@@ -0,0 +1,201 @@
+package com.lagradost.cloudstream3.utils
+
+import java.lang.Exception
+import java.util.*
+import javax.crypto.Cipher
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import kotlin.math.pow
+import com.lagradost.cloudstream3.mvvm.logError
+
+
+class M3u8Helper {
+    private val ENCRYPTION_DETECTION_REGEX = Regex("#EXT-X-KEY:METHOD=([^,]+),")
+    private val ENCRYPTION_URL_IV_REGEX = Regex("#EXT-X-KEY:METHOD=([^,]+),URI=\"([^\"]+)\"(?:,IV=(.*))?")
+    private val QUALITY_REGEX = Regex("""#EXT-X-STREAM-INF:.*RESOLUTION=\d+x(\d+).*\n(.*)""")
+    private val TS_EXTENSION_REGEX = Regex("""(.*\.ts.*)""")
+
+    private fun absoluteExtensionDetermination(url: String): String? {
+        val split = url.split("/")
+        val gg: String = split[split.size - 1].split("?")[0]
+        return if (gg.contains(".")) {
+            gg.split(".")[1].ifEmpty { null }
+        } else null
+    }
+
+    private fun toBytes16Big(n: Int): ByteArray {
+        return ByteArray(16) {
+            val fixed = n / 256.0.pow((15 - it))
+            (maxOf(0, fixed.toInt()) % 256).toByte()
+        }
+    }
+
+    private val defaultIvGen = sequence {
+        var initial = 1
+
+        while (true) {
+            yield(toBytes16Big(initial))
+            ++initial
+        }
+    }.iterator()
+
+    private fun getDecrypter(secretKey: ByteArray, data: ByteArray, iv: ByteArray = "".toByteArray()): ByteArray {
+        val ivKey = if (iv.isEmpty()) defaultIvGen.next() else iv
+        val c = Cipher.getInstance("AES/CBC/PKCS5Padding")
+        val skSpec = SecretKeySpec(secretKey, "AES")
+        val ivSpec = IvParameterSpec(ivKey)
+        c.init(Cipher.DECRYPT_MODE, skSpec, ivSpec)
+        return c.doFinal(data)
+    }
+
+    private fun isEncrypted(m3u8Data: String): Boolean {
+        val st = ENCRYPTION_DETECTION_REGEX.find(m3u8Data)
+        return st != null && (st.value.isNotEmpty() || st.destructured.component1() != "NONE")
+    }
+
+    public data class M3u8Stream(
+        val streamUrl: String,
+        val quality: Int? = null,
+        val headers: Map<String, String> = mapOf()
+    )
+
+    private fun selectBest(qualities: List<M3u8Stream>): M3u8Stream? {
+        val result = qualities.sortedBy { if (it.quality != null && it.quality <= 1080) it.quality else 0
+        }.reversed().filter {
+            listOf("m3u", "m3u8").contains(absoluteExtensionDetermination(it.streamUrl))
+        }
+        return result.getOrNull(0)
+    }
+
+    private fun getParentLink(uri: String): String {
+        val split = uri.split("/").toMutableList()
+        split.removeLast()
+        return split.joinToString("/")
+    }
+
+    private fun isCompleteUrl(url: String): Boolean {
+        return url.contains("https://") && url.contains("http://")
+    }
+
+    public fun m3u8Generation(m3u8: M3u8Stream): List<M3u8Stream> {
+        val generate = sequence {
+            val m3u8Parent = getParentLink(m3u8.streamUrl)
+            val response = khttp.get(m3u8.streamUrl, headers=m3u8.headers)
+
+            for (match in QUALITY_REGEX.findAll(response.text)) {
+                var (quality, m3u8Link) = match.destructured
+                if (absoluteExtensionDetermination(m3u8Link) == "m3u8") {
+                    if (!isCompleteUrl(m3u8Link)) {
+                        m3u8Link = "$m3u8Parent/$m3u8Link"
+                    }
+                    yieldAll(
+                        m3u8Generation(
+                            M3u8Stream(
+                                m3u8Link,
+                                quality.toIntOrNull(),
+                                m3u8.headers
+                            )
+                        )
+                    )
+                }
+                yield(
+                    M3u8Stream(
+                        m3u8Link,
+                        quality.toInt(),
+                        m3u8.headers
+                    )
+                )
+            }
+        }
+        return generate.toList()
+    }
+
+    data class HlsDownloadData(
+        val bytes: ByteArray,
+        val currentIndex: Int,
+        val totalTs: Int,
+        val errored: Boolean = false
+    )
+
+    public fun hlsYield(qualities: List<M3u8Stream>): Iterator<HlsDownloadData> {
+        if (qualities.isEmpty()) return listOf<HlsDownloadData>().iterator()
+
+        var selected = selectBest(qualities)
+        if (selected == null) {
+            selected = qualities[0]
+        }
+        val headers = selected.headers
+
+        val streams = qualities.map { m3u8Generation(it) }.flatten()
+        val sslVerification = if (headers.containsKey("ssl_verification")) headers["ssl_verification"].toBoolean() else true
+
+        val secondSelection = selectBest(streams.ifEmpty { listOf(selected) })
+        if (secondSelection != null) {
+            val m3u8Response = khttp.get(secondSelection.streamUrl, headers=headers)
+            val m3u8Data = m3u8Response.text
+
+            var encryptionUri: String? = null
+            var encryptionIv = byteArrayOf()
+            var encryptionData= byteArrayOf()
+
+            val encryptionState = isEncrypted(m3u8Data)
+
+            if (encryptionState) {
+                val match = ENCRYPTION_URL_IV_REGEX.find(m3u8Data)!!.destructured  // its safe to assume that its not going to be null
+                encryptionUri = match.component2()
+
+                if (!isCompleteUrl(encryptionUri)) {
+                    encryptionUri = "${getParentLink(secondSelection.streamUrl)}/$encryptionUri"
+                }
+
+                encryptionIv = match.component3().toByteArray()
+                val encryptionKeyResponse = khttp.get(encryptionUri, headers=headers)
+                encryptionData = encryptionKeyResponse.content
+            }
+
+            val allTs = TS_EXTENSION_REGEX.findAll(m3u8Data)
+            val totalTs = allTs.toList().size
+            if (totalTs == 0) {
+                return listOf<HlsDownloadData>().iterator()
+            }
+            var lastYield = 0
+
+            val relativeUrl = getParentLink(secondSelection.streamUrl)
+            var retries = 0
+            val tsByteGen = sequence<HlsDownloadData> {
+                loop@ for ((index, ts) in allTs.withIndex()) {
+                    val url = if (
+                        isCompleteUrl(ts.destructured.component1())
+                    ) ts.destructured.component1() else "$relativeUrl/${ts.destructured.component1()}"
+                    val c = index+1
+
+                    while (lastYield != c) {
+                        try {
+                            val tsResponse = khttp.get(url, headers=headers)
+                            var tsData = tsResponse.content
+
+                            if (encryptionState) {
+                                tsData = getDecrypter(encryptionData, tsData, encryptionIv)
+                                yield(HlsDownloadData(tsData, c, totalTs))
+                                lastYield = c
+                                break
+                            }
+                            yield(HlsDownloadData(tsData, c, totalTs))
+                            lastYield = c
+                        } catch (e: Exception) {
+                            logError(e)
+                            if (retries == 3) {
+                                yield(HlsDownloadData(byteArrayOf(), c, totalTs, true))
+                                break@loop
+                            }
+                            ++retries
+                            Thread.sleep(2_000)
+                        }
+                    }
+                }
+            }
+            return tsByteGen.iterator()
+        }
+        return listOf<HlsDownloadData>().iterator()
+    }
+}

app/src/main/java/com/lagradost/cloudstream3/utils/VideoDownloadManager.kt

@@ -31,6 +31,8 @@ import com.lagradost.cloudstream3.utils.DataStore.getKey
 import com.lagradost.cloudstream3.utils.DataStore.removeKey
 import com.lagradost.cloudstream3.utils.DataStore.setKey
 import com.lagradost.cloudstream3.utils.UIHelper.colorFromAttribute
+import com.lagradost.cloudstream3.utils.M3u8Helper
+import kotlin.math.roundToInt
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.withContext
@@ -1047,6 +1049,252 @@ object VideoDownloadManager {
         }
     }
 
+    private fun downloadHLS(
+        context: Context,
+        link: ExtractorLink,
+        name: String,
+        folder: String?,
+        parentId: Int?,
+        createNotificationCallback: (CreateNotificationMetadata) -> Unit
+    ): Int {
+        fun logcatPrint(vararg items: Any?) {
+            items.forEach {
+                println("[HLS]: $it")
+            }
+        }
+
+        val m3u8Helper = M3u8Helper()
+        logcatPrint("initialised the HLS downloader.")
+
+        val m3u8 = M3u8Helper.M3u8Stream(link.url, when (link.quality) {
+            -2 -> 360
+            -1 -> 480
+            1 -> 720
+            2 -> 1080
+            else -> null
+        }, mapOf("referer" to link.referer))
+        val tsIterator = m3u8Helper.hlsYield(listOf(m3u8))
+
+        val relativePath = (Environment.DIRECTORY_DOWNLOADS + '/' + folder + '/').replace('/', File.separatorChar)
+        val displayName = "$name.ts"
+
+        val normalPath = "${Environment.getExternalStorageDirectory()}${File.separatorChar}$relativePath$displayName"
+
+        val fileStream: OutputStream
+        val fileLength: Long
+
+        fun deleteFile(): Int {
+            if (isScopedStorage()) {
+                val lastContent = context.contentResolver.getExistingDownloadUriOrNullQ(relativePath, displayName)
+                if (lastContent != null) {
+                    context.contentResolver.delete(lastContent, null, null)
+                }
+            } else {
+                if (!File(normalPath).delete()) return ERROR_DELETING_FILE
+            }
+            parentId?.let {
+                downloadDeleteEvent.invoke(parentId)
+            }
+            return SUCCESS_STOPPED
+        }
+
+        if (isScopedStorage()) {
+            val cr = context.contentResolver ?: return ERROR_CONTENT_RESOLVER_NOT_FOUND
+
+            val currentExistingFile =
+                cr.getExistingDownloadUriOrNullQ(relativePath, displayName) // CURRENT FILE WITH THE SAME PATH
+
+            if (currentExistingFile != null) { // DELETE FILE IF FILE EXITS
+                val rowsDeleted = context.contentResolver.delete(currentExistingFile, null, null)
+                if (rowsDeleted < 1) {
+                    println("ERROR DELETING FILE!!!")
+                }
+            }
+
+            val newFileUri = if (currentExistingFile != null) {
+                currentExistingFile
+            } else {
+                val contentUri =
+                    MediaStore.Downloads.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) // USE INSTEAD OF MediaStore.Downloads.EXTERNAL_CONTENT_URI
+                //val currentMimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension)
+                val currentMimeType = "video/mp2t"
+                val newFile = ContentValues().apply {
+                    put(MediaStore.MediaColumns.DISPLAY_NAME, displayName)
+                    put(MediaStore.MediaColumns.TITLE, name)
+                    put(MediaStore.MediaColumns.MIME_TYPE, currentMimeType)
+                    put(MediaStore.MediaColumns.RELATIVE_PATH, relativePath)
+                }
+
+                cr.insert(
+                    contentUri,
+                    newFile
+                ) ?: return ERROR_MEDIA_STORE_URI_CANT_BE_CREATED
+            }
+
+            fileStream = cr.openOutputStream(newFileUri, "a")
+                ?: return ERROR_CONTENT_RESOLVER_CANT_OPEN_STREAM
+        } else {
+            // NORMAL NON SCOPED STORAGE FILE CREATION
+            val rFile = File(normalPath)
+            if (!rFile.exists()) {
+                rFile.parentFile?.mkdirs()
+                if (!rFile.createNewFile()) return ERROR_CREATE_FILE
+            } else {
+                rFile.parentFile?.mkdirs()
+                if (!rFile.delete()) return ERROR_DELETING_FILE
+                if (!rFile.createNewFile()) return ERROR_CREATE_FILE
+            }
+            fileStream = FileOutputStream(rFile, false)
+        }
+        val firstTs = tsIterator.next()
+
+        var isDone = false
+        var isFailed = false
+        var bytesDownloaded = firstTs.bytes.size.toLong()
+        var tsProgress = 1L
+        val totalTs = firstTs.totalTs.toLong()
+        /*
+            Most of the auto generated m3u8 out there have TS of the same size.
+            And only the last TS might have a different size.
+
+            But oh well, in cases of handmade m3u8 streams this will go all over the place ¯\_(ツ)_/¯
+            So ya, this calculates an estimate of how many bytes the file is going to be.
+
+            > (bytesDownloaded/tsProgress)*totalTs
+         */
+
+
+        parentId?.let {
+            context.setKey(KEY_DOWNLOAD_INFO, it.toString(), DownloadedFileInfo((bytesDownloaded/tsProgress)*totalTs, relativePath, displayName))
+        }
+
+        fun updateNotification() {
+            val type = when {
+                isDone -> DownloadType.IsDone
+                isFailed -> DownloadType.IsFailed
+                else -> DownloadType.IsDownloading
+            }
+
+            parentId?.let { id ->
+                try {
+                    downloadStatus[id] = type
+                    downloadStatusEvent.invoke(Pair(id, type))
+                    downloadProgressEvent.invoke(Triple(id, bytesDownloaded, (bytesDownloaded/tsProgress)*totalTs))
+                } catch (e: Exception) {
+                    // IDK MIGHT ERROR
+                }
+            }
+
+            createNotificationCallback.invoke(CreateNotificationMetadata(type, bytesDownloaded, (bytesDownloaded/tsProgress)*totalTs))
+        }
+
+        fun stopIfError(ts: M3u8Helper.HlsDownloadData): Int? {
+            if (ts.errored || ts.bytes.isEmpty()) {
+                val error: Int
+                error = if (!ts.errored) {
+                    logcatPrint("Error: No stream was found.")
+                    ERROR_UNKNOWN
+                } else {
+                    logcatPrint("Error: Failed to fetch data.")
+                    ERROR_CONNECTION_ERROR
+                }
+                isFailed = true
+                fileStream.close()
+                deleteFile()
+                updateNotification()
+                return error
+            }
+            return null
+        }
+
+        val notificationCoroutine = main {
+            while (true) {
+                if (!isDone) {
+                    updateNotification()
+                }
+                for (i in 1..10) {
+                    delay(100)
+                }
+            }
+        }
+
+        val downloadEventListener = { event: Pair<Int, DownloadActionType> ->
+            if (event.first == parentId) {
+                when (event.second) {
+                    DownloadActionType.Stop -> {
+                        isFailed = true
+                    }
+                    DownloadActionType.Pause -> {
+                        isFailed = true  // Pausing is not supported since well...I need to know the index of the ts it was paused at
+                        // it may be possible to store it in a variable, but when the app restarts it will be lost
+                    }
+                    else -> updateNotification()  // do nothing, since well...I don't support anything else
+                }
+            }
+        }
+
+        fun closeAll() {
+            try {
+                if (parentId != null)
+                    downloadEvent -= downloadEventListener
+            } catch (e: Exception) {
+                logError(e)
+            }
+            try {
+                parentId?.let {
+                    downloadStatus.remove(it)
+                }
+            } catch (e: Exception) {
+                logError(e)
+                // IDK MIGHT ERROR
+            }
+            notificationCoroutine.cancel()
+        }
+        
+       stopIfError(firstTs).let {
+            if (it != null) {
+                closeAll()
+                return it
+            }
+        }
+
+        if (parentId != null)
+            downloadEvent += downloadEventListener
+
+        fileStream.write(firstTs.bytes)
+
+        for (ts in tsIterator) {
+            if (isFailed) {
+                fileStream.close()
+                deleteFile()
+                updateNotification()
+                closeAll()
+                return SUCCESS_STOPPED
+            }
+            stopIfError(ts).let {
+                if (it != null) {
+                    closeAll()
+                    return it
+                }
+            }
+
+            fileStream.write(ts.bytes)
+            tsProgress = ts.currentIndex.toLong()
+            bytesDownloaded += ts.bytes.size.toLong()
+            logcatPrint("Download progress ${((tsProgress.toFloat()/totalTs.toFloat())*100).roundToInt()}%")
+        }
+        isDone = true
+        fileStream.close()
+        updateNotification()
+
+        closeAll()
+        parentId?.let {
+            context.setKey(KEY_DOWNLOAD_INFO, it.toString(), DownloadedFileInfo(bytesDownloaded, relativePath, displayName))
+        }
+
+        return SUCCESS_DOWNLOAD_DONE
+    }
+    
     private fun downloadSingleEpisode(
         context: Context,
         source: String?,
@@ -1057,6 +1305,20 @@ object VideoDownloadManager {
     ): Int {
         val name = sanitizeFilename(ep.name ?: "Episode ${ep.episode}")
 
+        if (link.isM3u8) {
+            return downloadHLS(context, link, name, folder, ep.id) { meta ->
+                createNotification(
+                    context,
+                    source,
+                    link.name,
+                    ep,
+                    meta.type,
+                    meta.bytesDownloaded,
+                    meta.bytesTotal
+                )
+            }
+        }
+
         return normalSafeApiCall {
             downloadThing(context, link, name, folder, "mp4", tryResume, ep.id) { meta ->
                 createNotification(
@@ -1221,9 +1483,8 @@ object VideoDownloadManager {
         links: List<ExtractorLink>
     ) {
         if (context == null) return
-        val validLinks = links.filter { !it.isM3u8 }
+        if (links.isNotEmpty()) {
-        if (validLinks.isNotEmpty()) {
+            downloadFromResume(context, DownloadResumePackage(DownloadItem(source, folder, ep, links), null))
-            downloadFromResume(context, DownloadResumePackage(DownloadItem(source, folder, ep, validLinks), null))
         }
     }
-}
+}