Merge pull request #50 from LagradOst/hls
Added download support for HLS
This commit is contained in:
Osten
GitHub
commit
b1dad6cd82
|
|
@ -0,0 +1,201 @@
|
|||
package com.lagradost.cloudstream3.utils
|
||||
|
||||
import java.lang.Exception
|
||||
import java.util.*
|
||||
import javax.crypto.Cipher
|
||||
import javax.crypto.spec.IvParameterSpec
|
||||
import javax.crypto.spec.SecretKeySpec
|
||||
import kotlin.math.pow
|
||||
import com.lagradost.cloudstream3.mvvm.logError
|
||||
|
||||
|
||||
class M3u8Helper {
|
||||
private val ENCRYPTION_DETECTION_REGEX = Regex("#EXT-X-KEY:METHOD=([^,]+),")
|
||||
private val ENCRYPTION_URL_IV_REGEX = Regex("#EXT-X-KEY:METHOD=([^,]+),URI=\"([^\"]+)\"(?:,IV=(.*))?")
|
||||
private val QUALITY_REGEX = Regex("""#EXT-X-STREAM-INF:.*RESOLUTION=\d+x(\d+).*\n(.*)""")
|
||||
private val TS_EXTENSION_REGEX = Regex("""(.*\.ts.*)""")
|
||||
|
||||
private fun absoluteExtensionDetermination(url: String): String? {
|
||||
val split = url.split("/")
|
||||
val gg: String = split[split.size - 1].split("?")[0]
|
||||
return if (gg.contains(".")) {
|
||||
gg.split(".")[1].ifEmpty { null }
|
||||
} else null
|
||||
}
|
||||
|
||||
private fun toBytes16Big(n: Int): ByteArray {
|
||||
return ByteArray(16) {
|
||||
val fixed = n / 256.0.pow((15 - it))
|
||||
(maxOf(0, fixed.toInt()) % 256).toByte()
|
||||
}
|
||||
}
|
||||
|
||||
private val defaultIvGen = sequence {
|
||||
var initial = 1
|
||||
|
||||
while (true) {
|
||||
yield(toBytes16Big(initial))
|
||||
++initial
|
||||
}
|
||||
}.iterator()
|
||||
|
||||
private fun getDecrypter(secretKey: ByteArray, data: ByteArray, iv: ByteArray = "".toByteArray()): ByteArray {
|
||||
val ivKey = if (iv.isEmpty()) defaultIvGen.next() else iv
|
||||
val c = Cipher.getInstance("AES/CBC/PKCS5Padding")
|
||||
val skSpec = SecretKeySpec(secretKey, "AES")
|
||||
val ivSpec = IvParameterSpec(ivKey)
|
||||
c.init(Cipher.DECRYPT_MODE, skSpec, ivSpec)
|
||||
return c.doFinal(data)
|
||||
}
|
||||
|
||||
private fun isEncrypted(m3u8Data: String): Boolean {
|
||||
val st = ENCRYPTION_DETECTION_REGEX.find(m3u8Data)
|
||||
return st != null && (st.value.isNotEmpty() || st.destructured.component1() != "NONE")
|
||||
}
|
||||
|
||||
public data class M3u8Stream(
|
||||
val streamUrl: String,
|
||||
val quality: Int? = null,
|
||||
val headers: Map<String, String> = mapOf()
|
||||
)
|
||||
|
||||
private fun selectBest(qualities: List<M3u8Stream>): M3u8Stream? {
|
||||
val result = qualities.sortedBy { if (it.quality != null && it.quality <= 1080) it.quality else 0
|
||||
}.reversed().filter {
|
||||
listOf("m3u", "m3u8").contains(absoluteExtensionDetermination(it.streamUrl))
|
||||
}
|
||||
return result.getOrNull(0)
|
||||
}
|
||||
|
||||
private fun getParentLink(uri: String): String {
|
||||
val split = uri.split("/").toMutableList()
|
||||
split.removeLast()
|
||||
return split.joinToString("/")
|
||||
}
|
||||
|
||||
private fun isCompleteUrl(url: String): Boolean {
|
||||
return url.contains("https://") && url.contains("http://")
|
||||
}
|
||||
|
||||
public fun m3u8Generation(m3u8: M3u8Stream): List<M3u8Stream> {
|
||||
val generate = sequence {
|
||||
val m3u8Parent = getParentLink(m3u8.streamUrl)
|
||||
val response = khttp.get(m3u8.streamUrl, headers=m3u8.headers)
|
||||
|
||||
for (match in QUALITY_REGEX.findAll(response.text)) {
|
||||
var (quality, m3u8Link) = match.destructured
|
||||
if (absoluteExtensionDetermination(m3u8Link) == "m3u8") {
|
||||
if (!isCompleteUrl(m3u8Link)) {
|
||||
m3u8Link = "$m3u8Parent/$m3u8Link"
|
||||
}
|
||||
yieldAll(
|
||||
m3u8Generation(
|
||||
M3u8Stream(
|
||||
m3u8Link,
|
||||
quality.toIntOrNull(),
|
||||
m3u8.headers
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
yield(
|
||||
M3u8Stream(
|
||||
m3u8Link,
|
||||
quality.toInt(),
|
||||
m3u8.headers
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
return generate.toList()
|
||||
}
|
||||
|
||||
data class HlsDownloadData(
|
||||
val bytes: ByteArray,
|
||||
val currentIndex: Int,
|
||||
val totalTs: Int,
|
||||
val errored: Boolean = false
|
||||
)
|
||||
|
||||
public fun hlsYield(qualities: List<M3u8Stream>): Iterator<HlsDownloadData> {
|
||||
if (qualities.isEmpty()) return listOf<HlsDownloadData>().iterator()
|
||||
|
||||
var selected = selectBest(qualities)
|
||||
if (selected == null) {
|
||||
selected = qualities[0]
|
||||
}
|
||||
val headers = selected.headers
|
||||
|
||||
val streams = qualities.map { m3u8Generation(it) }.flatten()
|
||||
val sslVerification = if (headers.containsKey("ssl_verification")) headers["ssl_verification"].toBoolean() else true
|
||||
|
||||
val secondSelection = selectBest(streams.ifEmpty { listOf(selected) })
|
||||
if (secondSelection != null) {
|
||||
val m3u8Response = khttp.get(secondSelection.streamUrl, headers=headers)
|
||||
val m3u8Data = m3u8Response.text
|
||||
|
||||
var encryptionUri: String? = null
|
||||
var encryptionIv = byteArrayOf()
|
||||
var encryptionData= byteArrayOf()
|
||||
|
||||
val encryptionState = isEncrypted(m3u8Data)
|
||||
|
||||
if (encryptionState) {
|
||||
val match = ENCRYPTION_URL_IV_REGEX.find(m3u8Data)!!.destructured // its safe to assume that its not going to be null
|
||||
encryptionUri = match.component2()
|
||||
|
||||
if (!isCompleteUrl(encryptionUri)) {
|
||||
encryptionUri = "${getParentLink(secondSelection.streamUrl)}/$encryptionUri"
|
||||
}
|
||||
|
||||
encryptionIv = match.component3().toByteArray()
|
||||
val encryptionKeyResponse = khttp.get(encryptionUri, headers=headers)
|
||||
encryptionData = encryptionKeyResponse.content
|
||||
}
|
||||
|
||||
val allTs = TS_EXTENSION_REGEX.findAll(m3u8Data)
|
||||
val totalTs = allTs.toList().size
|
||||
if (totalTs == 0) {
|
||||
return listOf<HlsDownloadData>().iterator()
|
||||
}
|
||||
var lastYield = 0
|
||||
|
||||
val relativeUrl = getParentLink(secondSelection.streamUrl)
|
||||
var retries = 0
|
||||
val tsByteGen = sequence<HlsDownloadData> {
|
||||
loop@ for ((index, ts) in allTs.withIndex()) {
|
||||
val url = if (
|
||||
isCompleteUrl(ts.destructured.component1())
|
||||
) ts.destructured.component1() else "$relativeUrl/${ts.destructured.component1()}"
|
||||
val c = index+1
|
||||
|
||||
while (lastYield != c) {
|
||||
try {
|
||||
val tsResponse = khttp.get(url, headers=headers)
|
||||
var tsData = tsResponse.content
|
||||
|
||||
if (encryptionState) {
|
||||
tsData = getDecrypter(encryptionData, tsData, encryptionIv)
|
||||
yield(HlsDownloadData(tsData, c, totalTs))
|
||||
lastYield = c
|
||||
break
|
||||
}
|
||||
yield(HlsDownloadData(tsData, c, totalTs))
|
||||
lastYield = c
|
||||
} catch (e: Exception) {
|
||||
logError(e)
|
||||
if (retries == 3) {
|
||||
yield(HlsDownloadData(byteArrayOf(), c, totalTs, true))
|
||||
break@loop
|
||||
}
|
||||
++retries
|
||||
Thread.sleep(2_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return tsByteGen.iterator()
|
||||
}
|
||||
return listOf<HlsDownloadData>().iterator()
|
||||
}
|
||||
}
|
||||
|
|
@ -31,6 +31,8 @@ import com.lagradost.cloudstream3.utils.DataStore.getKey
|
|||
import com.lagradost.cloudstream3.utils.DataStore.removeKey
|
||||
import com.lagradost.cloudstream3.utils.DataStore.setKey
|
||||
import com.lagradost.cloudstream3.utils.UIHelper.colorFromAttribute
|
||||
import com.lagradost.cloudstream3.utils.M3u8Helper
|
||||
import kotlin.math.roundToInt
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.withContext
|
||||
|
|
@ -1047,6 +1049,252 @@ object VideoDownloadManager {
|
|||
}
|
||||
}
|
||||
|
||||
private fun downloadHLS(
|
||||
context: Context,
|
||||
link: ExtractorLink,
|
||||
name: String,
|
||||
folder: String?,
|
||||
parentId: Int?,
|
||||
createNotificationCallback: (CreateNotificationMetadata) -> Unit
|
||||
): Int {
|
||||
fun logcatPrint(vararg items: Any?) {
|
||||
items.forEach {
|
||||
println("[HLS]: $it")
|
||||
}
|
||||
}
|
||||
|
||||
val m3u8Helper = M3u8Helper()
|
||||
logcatPrint("initialised the HLS downloader.")
|
||||
|
||||
val m3u8 = M3u8Helper.M3u8Stream(link.url, when (link.quality) {
|
||||
-2 -> 360
|
||||
-1 -> 480
|
||||
1 -> 720
|
||||
2 -> 1080
|
||||
else -> null
|
||||
}, mapOf("referer" to link.referer))
|
||||
val tsIterator = m3u8Helper.hlsYield(listOf(m3u8))
|
||||
|
||||
val relativePath = (Environment.DIRECTORY_DOWNLOADS + '/' + folder + '/').replace('/', File.separatorChar)
|
||||
val displayName = "$name.ts"
|
||||
|
||||
val normalPath = "${Environment.getExternalStorageDirectory()}${File.separatorChar}$relativePath$displayName"
|
||||
|
||||
val fileStream: OutputStream
|
||||
val fileLength: Long
|
||||
|
||||
fun deleteFile(): Int {
|
||||
if (isScopedStorage()) {
|
||||
val lastContent = context.contentResolver.getExistingDownloadUriOrNullQ(relativePath, displayName)
|
||||
if (lastContent != null) {
|
||||
context.contentResolver.delete(lastContent, null, null)
|
||||
}
|
||||
} else {
|
||||
if (!File(normalPath).delete()) return ERROR_DELETING_FILE
|
||||
}
|
||||
parentId?.let {
|
||||
downloadDeleteEvent.invoke(parentId)
|
||||
}
|
||||
return SUCCESS_STOPPED
|
||||
}
|
||||
|
||||
if (isScopedStorage()) {
|
||||
val cr = context.contentResolver ?: return ERROR_CONTENT_RESOLVER_NOT_FOUND
|
||||
|
||||
val currentExistingFile =
|
||||
cr.getExistingDownloadUriOrNullQ(relativePath, displayName) // CURRENT FILE WITH THE SAME PATH
|
||||
|
||||
if (currentExistingFile != null) { // DELETE FILE IF FILE EXITS
|
||||
val rowsDeleted = context.contentResolver.delete(currentExistingFile, null, null)
|
||||
if (rowsDeleted < 1) {
|
||||
println("ERROR DELETING FILE!!!")
|
||||
}
|
||||
}
|
||||
|
||||
val newFileUri = if (currentExistingFile != null) {
|
||||
currentExistingFile
|
||||
} else {
|
||||
val contentUri =
|
||||
MediaStore.Downloads.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY) // USE INSTEAD OF MediaStore.Downloads.EXTERNAL_CONTENT_URI
|
||||
//val currentMimeType = MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension)
|
||||
val currentMimeType = "video/mp2t"
|
||||
val newFile = ContentValues().apply {
|
||||
put(MediaStore.MediaColumns.DISPLAY_NAME, displayName)
|
||||
put(MediaStore.MediaColumns.TITLE, name)
|
||||
put(MediaStore.MediaColumns.MIME_TYPE, currentMimeType)
|
||||
put(MediaStore.MediaColumns.RELATIVE_PATH, relativePath)
|
||||
}
|
||||
|
||||
cr.insert(
|
||||
contentUri,
|
||||
newFile
|
||||
) ?: return ERROR_MEDIA_STORE_URI_CANT_BE_CREATED
|
||||
}
|
||||
|
||||
fileStream = cr.openOutputStream(newFileUri, "a")
|
||||
?: return ERROR_CONTENT_RESOLVER_CANT_OPEN_STREAM
|
||||
} else {
|
||||
// NORMAL NON SCOPED STORAGE FILE CREATION
|
||||
val rFile = File(normalPath)
|
||||
if (!rFile.exists()) {
|
||||
rFile.parentFile?.mkdirs()
|
||||
if (!rFile.createNewFile()) return ERROR_CREATE_FILE
|
||||
} else {
|
||||
rFile.parentFile?.mkdirs()
|
||||
if (!rFile.delete()) return ERROR_DELETING_FILE
|
||||
if (!rFile.createNewFile()) return ERROR_CREATE_FILE
|
||||
}
|
||||
fileStream = FileOutputStream(rFile, false)
|
||||
}
|
||||
val firstTs = tsIterator.next()
|
||||
|
||||
var isDone = false
|
||||
var isFailed = false
|
||||
var bytesDownloaded = firstTs.bytes.size.toLong()
|
||||
var tsProgress = 1L
|
||||
val totalTs = firstTs.totalTs.toLong()
|
||||
/*
|
||||
Most of the auto generated m3u8 out there have TS of the same size.
|
||||
And only the last TS might have a different size.
|
||||
|
||||
But oh well, in cases of handmade m3u8 streams this will go all over the place ¯\_(ツ)_/¯
|
||||
So ya, this calculates an estimate of how many bytes the file is going to be.
|
||||
|
||||
> (bytesDownloaded/tsProgress)*totalTs
|
||||
*/
|
||||
|
||||
|
||||
parentId?.let {
|
||||
context.setKey(KEY_DOWNLOAD_INFO, it.toString(), DownloadedFileInfo((bytesDownloaded/tsProgress)*totalTs, relativePath, displayName))
|
||||
}
|
||||
|
||||
fun updateNotification() {
|
||||
val type = when {
|
||||
isDone -> DownloadType.IsDone
|
||||
isFailed -> DownloadType.IsFailed
|
||||
else -> DownloadType.IsDownloading
|
||||
}
|
||||
|
||||
parentId?.let { id ->
|
||||
try {
|
||||
downloadStatus[id] = type
|
||||
downloadStatusEvent.invoke(Pair(id, type))
|
||||
downloadProgressEvent.invoke(Triple(id, bytesDownloaded, (bytesDownloaded/tsProgress)*totalTs))
|
||||
} catch (e: Exception) {
|
||||
// IDK MIGHT ERROR
|
||||
}
|
||||
}
|
||||
|
||||
createNotificationCallback.invoke(CreateNotificationMetadata(type, bytesDownloaded, (bytesDownloaded/tsProgress)*totalTs))
|
||||
}
|
||||
|
||||
fun stopIfError(ts: M3u8Helper.HlsDownloadData): Int? {
|
||||
if (ts.errored || ts.bytes.isEmpty()) {
|
||||
val error: Int
|
||||
error = if (!ts.errored) {
|
||||
logcatPrint("Error: No stream was found.")
|
||||
ERROR_UNKNOWN
|
||||
} else {
|
||||
logcatPrint("Error: Failed to fetch data.")
|
||||
ERROR_CONNECTION_ERROR
|
||||
}
|
||||
isFailed = true
|
||||
fileStream.close()
|
||||
deleteFile()
|
||||
updateNotification()
|
||||
return error
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
val notificationCoroutine = main {
|
||||
while (true) {
|
||||
if (!isDone) {
|
||||
updateNotification()
|
||||
}
|
||||
for (i in 1..10) {
|
||||
delay(100)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val downloadEventListener = { event: Pair<Int, DownloadActionType> ->
|
||||
if (event.first == parentId) {
|
||||
when (event.second) {
|
||||
DownloadActionType.Stop -> {
|
||||
isFailed = true
|
||||
}
|
||||
DownloadActionType.Pause -> {
|
||||
isFailed = true // Pausing is not supported since well...I need to know the index of the ts it was paused at
|
||||
// it may be possible to store it in a variable, but when the app restarts it will be lost
|
||||
}
|
||||
else -> updateNotification() // do nothing, since well...I don't support anything else
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun closeAll() {
|
||||
try {
|
||||
if (parentId != null)
|
||||
downloadEvent -= downloadEventListener
|
||||
} catch (e: Exception) {
|
||||
logError(e)
|
||||
}
|
||||
try {
|
||||
parentId?.let {
|
||||
downloadStatus.remove(it)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
logError(e)
|
||||
// IDK MIGHT ERROR
|
||||
}
|
||||
notificationCoroutine.cancel()
|
||||
}
|
||||
|
||||
stopIfError(firstTs).let {
|
||||
if (it != null) {
|
||||
closeAll()
|
||||
return it
|
||||
}
|
||||
}
|
||||
|
||||
if (parentId != null)
|
||||
downloadEvent += downloadEventListener
|
||||
|
||||
fileStream.write(firstTs.bytes)
|
||||
|
||||
for (ts in tsIterator) {
|
||||
if (isFailed) {
|
||||
fileStream.close()
|
||||
deleteFile()
|
||||
updateNotification()
|
||||
closeAll()
|
||||
return SUCCESS_STOPPED
|
||||
}
|
||||
stopIfError(ts).let {
|
||||
if (it != null) {
|
||||
closeAll()
|
||||
return it
|
||||
}
|
||||
}
|
||||
|
||||
fileStream.write(ts.bytes)
|
||||
tsProgress = ts.currentIndex.toLong()
|
||||
bytesDownloaded += ts.bytes.size.toLong()
|
||||
logcatPrint("Download progress ${((tsProgress.toFloat()/totalTs.toFloat())*100).roundToInt()}%")
|
||||
}
|
||||
isDone = true
|
||||
fileStream.close()
|
||||
updateNotification()
|
||||
|
||||
closeAll()
|
||||
parentId?.let {
|
||||
context.setKey(KEY_DOWNLOAD_INFO, it.toString(), DownloadedFileInfo(bytesDownloaded, relativePath, displayName))
|
||||
}
|
||||
|
||||
return SUCCESS_DOWNLOAD_DONE
|
||||
}
|
||||
|
||||
private fun downloadSingleEpisode(
|
||||
context: Context,
|
||||
source: String?,
|
||||
|
|
@ -1057,6 +1305,20 @@ object VideoDownloadManager {
|
|||
): Int {
|
||||
val name = sanitizeFilename(ep.name ?: "Episode ${ep.episode}")
|
||||
|
||||
if (link.isM3u8) {
|
||||
return downloadHLS(context, link, name, folder, ep.id) { meta ->
|
||||
createNotification(
|
||||
context,
|
||||
source,
|
||||
link.name,
|
||||
ep,
|
||||
meta.type,
|
||||
meta.bytesDownloaded,
|
||||
meta.bytesTotal
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return normalSafeApiCall {
|
||||
downloadThing(context, link, name, folder, "mp4", tryResume, ep.id) { meta ->
|
||||
createNotification(
|
||||
|
|
@ -1221,9 +1483,8 @@ object VideoDownloadManager {
|
|||
links: List<ExtractorLink>
|
||||
) {
|
||||
if (context == null) return
|
||||
val validLinks = links.filter { !it.isM3u8 }
|
||||
if (validLinks.isNotEmpty()) {
|
||||
downloadFromResume(context, DownloadResumePackage(DownloadItem(source, folder, ep, validLinks), null))
|
||||
if (links.isNotEmpty()) {
|
||||
downloadFromResume(context, DownloadResumePackage(DownloadItem(source, folder, ep, links), null))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue