Merge pull request #50 from LagradOst/hls

Added download support for HLS
This commit is contained in:
Osten 2021-09-01 19:42:23 +00:00 committed by GitHub
commit b1dad6cd82
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 466 additions and 4 deletions

View file

@ -0,0 +1,201 @@
package com.lagradost.cloudstream3.utils
import java.lang.Exception
import java.util.*
import javax.crypto.Cipher
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
import kotlin.math.pow
import com.lagradost.cloudstream3.mvvm.logError
class M3u8Helper {
private val ENCRYPTION_DETECTION_REGEX = Regex("#EXT-X-KEY:METHOD=([^,]+),")
private val ENCRYPTION_URL_IV_REGEX = Regex("#EXT-X-KEY:METHOD=([^,]+),URI=\"([^\"]+)\"(?:,IV=(.*))?")
private val QUALITY_REGEX = Regex("""#EXT-X-STREAM-INF:.*RESOLUTION=\d+x(\d+).*\n(.*)""")
private val TS_EXTENSION_REGEX = Regex("""(.*\.ts.*)""")
private fun absoluteExtensionDetermination(url: String): String? {
val split = url.split("/")
val gg: String = split[split.size - 1].split("?")[0]
return if (gg.contains(".")) {
gg.split(".")[1].ifEmpty { null }
} else null
}
private fun toBytes16Big(n: Int): ByteArray {
return ByteArray(16) {
val fixed = n / 256.0.pow((15 - it))
(maxOf(0, fixed.toInt()) % 256).toByte()
}
}
private val defaultIvGen = sequence {
var initial = 1
while (true) {
yield(toBytes16Big(initial))
++initial
}
}.iterator()
private fun getDecrypter(secretKey: ByteArray, data: ByteArray, iv: ByteArray = "".toByteArray()): ByteArray {
val ivKey = if (iv.isEmpty()) defaultIvGen.next() else iv
val c = Cipher.getInstance("AES/CBC/PKCS5Padding")
val skSpec = SecretKeySpec(secretKey, "AES")
val ivSpec = IvParameterSpec(ivKey)
c.init(Cipher.DECRYPT_MODE, skSpec, ivSpec)
return c.doFinal(data)
}
private fun isEncrypted(m3u8Data: String): Boolean {
val st = ENCRYPTION_DETECTION_REGEX.find(m3u8Data)
return st != null && (st.value.isNotEmpty() || st.destructured.component1() != "NONE")
}
public data class M3u8Stream(
val streamUrl: String,
val quality: Int? = null,
val headers: Map<String, String> = mapOf()
)
private fun selectBest(qualities: List<M3u8Stream>): M3u8Stream? {
val result = qualities.sortedBy { if (it.quality != null && it.quality <= 1080) it.quality else 0
}.reversed().filter {
listOf("m3u", "m3u8").contains(absoluteExtensionDetermination(it.streamUrl))
}
return result.getOrNull(0)
}
private fun getParentLink(uri: String): String {
val split = uri.split("/").toMutableList()
split.removeLast()
return split.joinToString("/")
}
private fun isCompleteUrl(url: String): Boolean {
return url.contains("https://") && url.contains("http://")
}
public fun m3u8Generation(m3u8: M3u8Stream): List<M3u8Stream> {
val generate = sequence {
val m3u8Parent = getParentLink(m3u8.streamUrl)
val response = khttp.get(m3u8.streamUrl, headers=m3u8.headers)
for (match in QUALITY_REGEX.findAll(response.text)) {
var (quality, m3u8Link) = match.destructured
if (absoluteExtensionDetermination(m3u8Link) == "m3u8") {
if (!isCompleteUrl(m3u8Link)) {
m3u8Link = "$m3u8Parent/$m3u8Link"
}
yieldAll(
m3u8Generation(
M3u8Stream(
m3u8Link,
quality.toIntOrNull(),
m3u8.headers
)
)
)
}
yield(
M3u8Stream(
m3u8Link,
quality.toInt(),
m3u8.headers
)
)
}
}
return generate.toList()
}
data class HlsDownloadData(
val bytes: ByteArray,
val currentIndex: Int,
val totalTs: Int,
val errored: Boolean = false
)
public fun hlsYield(qualities: List<M3u8Stream>): Iterator<HlsDownloadData> {
if (qualities.isEmpty()) return listOf<HlsDownloadData>().iterator()
var selected = selectBest(qualities)
if (selected == null) {
selected = qualities[0]
}
val headers = selected.headers
val streams = qualities.map { m3u8Generation(it) }.flatten()
val sslVerification = if (headers.containsKey("ssl_verification")) headers["ssl_verification"].toBoolean() else true
val secondSelection = selectBest(streams.ifEmpty { listOf(selected) })
if (secondSelection != null) {
val m3u8Response = khttp.get(secondSelection.streamUrl, headers=headers)
val m3u8Data = m3u8Response.text
var encryptionUri: String? = null
var encryptionIv = byteArrayOf()
var encryptionData= byteArrayOf()
val encryptionState = isEncrypted(m3u8Data)
if (encryptionState) {
val match = ENCRYPTION_URL_IV_REGEX.find(m3u8Data)!!.destructured // its safe to assume that its not going to be null
encryptionUri = match.component2()
if (!isCompleteUrl(encryptionUri)) {
encryptionUri = "${getParentLink(secondSelection.streamUrl)}/$encryptionUri"
}
encryptionIv = match.component3().toByteArray()
val encryptionKeyResponse = khttp.get(encryptionUri, headers=headers)
encryptionData = encryptionKeyResponse.content
}
val allTs = TS_EXTENSION_REGEX.findAll(m3u8Data)
val totalTs = allTs.toList().size
if (totalTs == 0) {
return listOf<HlsDownloadData>().iterator()
}
var lastYield = 0
val relativeUrl = getParentLink(secondSelection.streamUrl)
var retries = 0
val tsByteGen = sequence<HlsDownloadData> {
loop@ for ((index, ts) in allTs.withIndex()) {
val url = if (
isCompleteUrl(ts.destructured.component1())
) ts.destructured.component1() else "$relativeUrl/${ts.destructured.component1()}"
val c = index+1
while (lastYield != c) {
try {
val tsResponse = khttp.get(url, headers=headers)
var tsData = tsResponse.content
if (encryptionState) {
tsData = getDecrypter(encryptionData, tsData, encryptionIv)
yield(HlsDownloadData(tsData, c, totalTs))
lastYield = c
break
}
yield(HlsDownloadData(tsData, c, totalTs))
lastYield = c
} catch (e: Exception) {
logError(e)
if (retries == 3) {
yield(HlsDownloadData(byteArrayOf(), c, totalTs, true))
break@loop
}
++retries
Thread.sleep(2_000)
}
}
}
}
return tsByteGen.iterator()
}
return listOf<HlsDownloadData>().iterator()
}
}

View file

@ -31,6 +31,8 @@ import com.lagradost.cloudstream3.utils.DataStore.getKey
import com.lagradost.cloudstream3.utils.DataStore.removeKey
import com.lagradost.cloudstream3.utils.DataStore.setKey
import com.lagradost.cloudstream3.utils.UIHelper.colorFromAttribute
import com.lagradost.cloudstream3.utils.M3u8Helper
import kotlin.math.roundToInt
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
@ -1047,6 +1049,252 @@ object VideoDownloadManager {
}
}
private fun downloadHLS(
context: Context,
link: ExtractorLink,
name: String,
folder: String?,
parentId: Int?,
createNotificationCallback: (CreateNotificationMetadata) -> Unit
): Int {
fun logcatPrint(vararg items: Any?) {
items.forEach {
println("[HLS]: $it")
}
}
val m3u8Helper = M3u8Helper()
logcatPrint("initialised the HLS downloader.")
val m3u8 = M3u8Helper.M3u8Stream(link.url, when (link.quality) {
-2 -> 360
-1 -> 480
1 -> 720
2 -> 1080
else -> null
}, mapOf("referer" to link.referer))
val tsIterator = m3u8Helper.hlsYield(listOf(m3u8))
val relativePath = (Environment.DIRECTORY_DOWNLOADS + '/' + folder + '/').replace('/', File.separatorChar)
val displayName = "$name.ts"
val normalPath = "${Environment.getExternalStorageDirectory()}${File.separatorChar}$relativePath$displayName"
val fileStream: OutputStream
val fileLength: Long
fun deleteFile(): Int {
if (isScopedStorage()) {
val lastContent = context.contentResolver.getExistingDownloadUriOrNullQ(relativePath, displayName)
if (lastContent != null) {
context.contentResolver.delete(lastContent, null, null)
}
} else {
if (!File(normalPath).delete()) return ERROR_DELETING_FILE
}
parentId?.let {
downloadDeleteEvent.invoke(parentId)
}
return SUCCESS_STOPPED
}
if (isScopedStorage()) {
val cr = context.contentResolver ?: return ERROR_CONTENT_RESOLVER_NOT_FOUND
val currentExistingFile =
cr.getExistingDownloadUriOrNullQ(relativePath, displayName) // CURRENT FILE WITH THE SAME PATH
if (currentExistingFile != null) { // DELETE FILE IF FILE EXITS
val rowsDeleted = context.contentResolver.delete(currentExistingFile, null, null)
if (rowsDeleted < 1) {
println("ERROR DELETING FILE!!!")
}
}
val newFileUri = if (currentExistingFile != null) {
currentExistingFile
} else {
val contentUri =
MediaStore.Downloads.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) // USE INSTEAD OF MediaStore.Downloads.EXTERNAL_CONTENT_URI
//val currentMimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension)
val currentMimeType = "video/mp2t"
val newFile = ContentValues().apply {
put(MediaStore.MediaColumns.DISPLAY_NAME, displayName)
put(MediaStore.MediaColumns.TITLE, name)
put(MediaStore.MediaColumns.MIME_TYPE, currentMimeType)
put(MediaStore.MediaColumns.RELATIVE_PATH, relativePath)
}
cr.insert(
contentUri,
newFile
) ?: return ERROR_MEDIA_STORE_URI_CANT_BE_CREATED
}
fileStream = cr.openOutputStream(newFileUri, "a")
?: return ERROR_CONTENT_RESOLVER_CANT_OPEN_STREAM
} else {
// NORMAL NON SCOPED STORAGE FILE CREATION
val rFile = File(normalPath)
if (!rFile.exists()) {
rFile.parentFile?.mkdirs()
if (!rFile.createNewFile()) return ERROR_CREATE_FILE
} else {
rFile.parentFile?.mkdirs()
if (!rFile.delete()) return ERROR_DELETING_FILE
if (!rFile.createNewFile()) return ERROR_CREATE_FILE
}
fileStream = FileOutputStream(rFile, false)
}
val firstTs = tsIterator.next()
var isDone = false
var isFailed = false
var bytesDownloaded = firstTs.bytes.size.toLong()
var tsProgress = 1L
val totalTs = firstTs.totalTs.toLong()
/*
Most of the auto generated m3u8 out there have TS of the same size.
And only the last TS might have a different size.
But oh well, in cases of handmade m3u8 streams this will go all over the place ¯\_()_/¯
So ya, this calculates an estimate of how many bytes the file is going to be.
> (bytesDownloaded/tsProgress)*totalTs
*/
parentId?.let {
context.setKey(KEY_DOWNLOAD_INFO, it.toString(), DownloadedFileInfo((bytesDownloaded/tsProgress)*totalTs, relativePath, displayName))
}
fun updateNotification() {
val type = when {
isDone -> DownloadType.IsDone
isFailed -> DownloadType.IsFailed
else -> DownloadType.IsDownloading
}
parentId?.let { id ->
try {
downloadStatus[id] = type
downloadStatusEvent.invoke(Pair(id, type))
downloadProgressEvent.invoke(Triple(id, bytesDownloaded, (bytesDownloaded/tsProgress)*totalTs))
} catch (e: Exception) {
// IDK MIGHT ERROR
}
}
createNotificationCallback.invoke(CreateNotificationMetadata(type, bytesDownloaded, (bytesDownloaded/tsProgress)*totalTs))
}
fun stopIfError(ts: M3u8Helper.HlsDownloadData): Int? {
if (ts.errored || ts.bytes.isEmpty()) {
val error: Int
error = if (!ts.errored) {
logcatPrint("Error: No stream was found.")
ERROR_UNKNOWN
} else {
logcatPrint("Error: Failed to fetch data.")
ERROR_CONNECTION_ERROR
}
isFailed = true
fileStream.close()
deleteFile()
updateNotification()
return error
}
return null
}
val notificationCoroutine = main {
while (true) {
if (!isDone) {
updateNotification()
}
for (i in 1..10) {
delay(100)
}
}
}
val downloadEventListener = { event: Pair<Int, DownloadActionType> ->
if (event.first == parentId) {
when (event.second) {
DownloadActionType.Stop -> {
isFailed = true
}
DownloadActionType.Pause -> {
isFailed = true // Pausing is not supported since well...I need to know the index of the ts it was paused at
// it may be possible to store it in a variable, but when the app restarts it will be lost
}
else -> updateNotification() // do nothing, since well...I don't support anything else
}
}
}
fun closeAll() {
try {
if (parentId != null)
downloadEvent -= downloadEventListener
} catch (e: Exception) {
logError(e)
}
try {
parentId?.let {
downloadStatus.remove(it)
}
} catch (e: Exception) {
logError(e)
// IDK MIGHT ERROR
}
notificationCoroutine.cancel()
}
stopIfError(firstTs).let {
if (it != null) {
closeAll()
return it
}
}
if (parentId != null)
downloadEvent += downloadEventListener
fileStream.write(firstTs.bytes)
for (ts in tsIterator) {
if (isFailed) {
fileStream.close()
deleteFile()
updateNotification()
closeAll()
return SUCCESS_STOPPED
}
stopIfError(ts).let {
if (it != null) {
closeAll()
return it
}
}
fileStream.write(ts.bytes)
tsProgress = ts.currentIndex.toLong()
bytesDownloaded += ts.bytes.size.toLong()
logcatPrint("Download progress ${((tsProgress.toFloat()/totalTs.toFloat())*100).roundToInt()}%")
}
isDone = true
fileStream.close()
updateNotification()
closeAll()
parentId?.let {
context.setKey(KEY_DOWNLOAD_INFO, it.toString(), DownloadedFileInfo(bytesDownloaded, relativePath, displayName))
}
return SUCCESS_DOWNLOAD_DONE
}
private fun downloadSingleEpisode(
context: Context,
source: String?,
@ -1057,6 +1305,20 @@ object VideoDownloadManager {
): Int {
val name = sanitizeFilename(ep.name ?: "Episode ${ep.episode}")
if (link.isM3u8) {
return downloadHLS(context, link, name, folder, ep.id) { meta ->
createNotification(
context,
source,
link.name,
ep,
meta.type,
meta.bytesDownloaded,
meta.bytesTotal
)
}
}
return normalSafeApiCall {
downloadThing(context, link, name, folder, "mp4", tryResume, ep.id) { meta ->
createNotification(
@ -1221,9 +1483,8 @@ object VideoDownloadManager {
links: List<ExtractorLink>
) {
if (context == null) return
val validLinks = links.filter { !it.isM3u8 }
if (validLinks.isNotEmpty()) {
downloadFromResume(context, DownloadResumePackage(DownloadItem(source, folder, ep, validLinks), null))
if (links.isNotEmpty()) {
downloadFromResume(context, DownloadResumePackage(DownloadItem(source, folder, ep, links), null))
}
}
}
}