MohmedIbrahim/cloudstream
1
0
Fork 0
forked from recloudstream/cloudstream
Code Issues Pull requests Projects Releases Packages Wiki Activity

switched yt implementation to newpipe

Browse source
This commit is contained in:
LagradOst 2022-06-17 19:25:41 +02:00
parent f64b2f7f1b
commit 3d00015a0c
12 changed files with 240 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
app/build.gradle
View file

@ -74,6 +74,8 @@ android {
}
}
compileOptions {
coreLibraryDesugaringEnabled true
sourceCompatibility JavaVersion.VERSION_1_8
targetCompatibility JavaVersion.VERSION_1_8
}
@ -143,7 +145,7 @@ dependencies {
implementation 'com.jaredrummler:colorpicker:1.1.0'
//run JS
implementation 'org.mozilla:rhino:1.7R4'
implementation 'org.mozilla:rhino:1.7.14'
// TorrentStream
//implementation 'com.github.TorrentStream:TorrentStream-Android:2.7.0'
@ -175,6 +177,11 @@ dependencies {
// used for subtitle decoding https://github.com/albfernandez/juniversalchardet
implementation 'com.github.albfernandez:juniversalchardet:2.4.0'
// play yt
implementation 'com.github.HaarigerHarald:android-youtubeExtractor:master-SNAPSHOT'
// slow af yt
//implementation 'com.github.HaarigerHarald:android-youtubeExtractor:master-SNAPSHOT'
// newpipe yt
implementation 'com.github.TeamNewPipe:NewPipeExtractor:master-SNAPSHOT'
coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:1.1.5'
}

83
app/src/main/java/com/lagradost/cloudstream3/DownloaderTestImpl.kt Normal file
View file

@ -0,0 +1,83 @@
package com.lagradost.cloudstream3
import okhttp3.OkHttpClient
import okhttp3.RequestBody
import org.schabi.newpipe.extractor.downloader.Downloader
import org.schabi.newpipe.extractor.downloader.Request
import org.schabi.newpipe.extractor.downloader.Response
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException
import java.util.concurrent.TimeUnit
class DownloaderTestImpl private constructor(builder: OkHttpClient.Builder) : Downloader() {
private val client: OkHttpClient
override fun execute(request: Request): Response {
val httpMethod: String = request.httpMethod()
val url: String = request.url()
val headers: Map<String, List<String>> = request.headers()
val dataToSend: ByteArray? = request.dataToSend()
var requestBody: RequestBody? = null
if (dataToSend != null) {
requestBody = RequestBody.create(null, dataToSend)
}
val requestBuilder: okhttp3.Request.Builder = okhttp3.Request.Builder()
.method(httpMethod, requestBody).url(url)
.addHeader("User-Agent", USER_AGENT)
for ((headerName, headerValueList) in headers) {
if (headerValueList.size > 1) {
requestBuilder.removeHeader(headerName)
for (headerValue in headerValueList) {
requestBuilder.addHeader(headerName, headerValue)
}
} else if (headerValueList.size == 1) {
requestBuilder.header(headerName, headerValueList[0])
}
}
val response = client.newCall(requestBuilder.build()).execute()
if (response.code == 429) {
response.close()
throw ReCaptchaException("reCaptcha Challenge requested", url)
}
val body = response.body
var responseBodyToReturn: String? = null
if (body != null) {
responseBodyToReturn = body.string()
}
val latestUrl = response.request.url.toString()
return Response(
response.code, response.message, response.headers.toMultimap(),
responseBodyToReturn, latestUrl
)
}
companion object {
private const val USER_AGENT =
"Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0"
private var instance: DownloaderTestImpl? = null
/**
* It's recommended to call exactly once in the entire lifetime of the application.
*
* @param builder if null, default builder will be used
* @return a new instance of [DownloaderTestImpl]
*/
fun init(builder: OkHttpClient.Builder?): DownloaderTestImpl? {
instance = DownloaderTestImpl(
builder ?: OkHttpClient.Builder()
)
return instance
}
fun getInstance(): DownloaderTestImpl? {
if (instance == null) {
init(null)
}
return instance
}
}
init {
client = builder.readTimeout(30, TimeUnit.SECONDS).build()
}
}

58
app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt
View file

@ -19,6 +19,7 @@ import com.lagradost.cloudstream3.syncproviders.AccountManager.Companion.malApi
import com.lagradost.cloudstream3.ui.player.SubtitleData
import com.lagradost.cloudstream3.utils.AppUtils.toJson
import com.lagradost.cloudstream3.utils.ExtractorLink
import com.lagradost.cloudstream3.utils.loadExtractor
import okhttp3.Interceptor
import java.text.SimpleDateFormat
import java.util.*
@ -854,7 +855,7 @@ interface LoadResponse {
var rating: Int? // 0-10000
var tags: List<String>?
var duration: Int? // in minutes
var trailers: List<String>?
var trailers: List<ExtractorLink>?
var recommendations: List<SearchResponse>?
var actors: List<ActorData>?
var comingSoon: Boolean
@ -897,27 +898,34 @@ interface LoadResponse {
addImdbId(imdbUrlToIdNullable(url))
}
/**better to set trailers directly instead of calling this multiple times*/
fun LoadResponse.addTrailer(trailerUrl: String?) {
/**better to call addTrailer with mutible trailers directly instead of calling this multiple times*/
suspend fun LoadResponse.addTrailer(trailerUrl: String?, referer: String? = null) {
if (trailerUrl == null) return
val newTrailers = loadExtractor(trailerUrl, referer)
addTrailer(newTrailers)
}
fun LoadResponse.addTrailer(newTrailers: List<ExtractorLink>) {
if (this.trailers == null) {
this.trailers = listOf(trailerUrl)
this.trailers = newTrailers
} else {
val update = this.trailers?.toMutableList()
update?.add(trailerUrl)
val update = this.trailers?.toMutableList() ?: mutableListOf()
update.addAll(newTrailers)
this.trailers = update
}
}
fun LoadResponse.addTrailer(trailerUrls: List<String>?) {
suspend fun LoadResponse.addTrailer(trailerUrls: List<String>?, referer: String? = null) {
if (trailerUrls == null) return
if (this.trailers == null) {
this.trailers = trailerUrls
} else {
val update = this.trailers?.toMutableList()
update?.addAll(trailerUrls)
this.trailers = update
val newTrailers = trailerUrls.apmap { trailerUrl ->
try {
loadExtractor(trailerUrl, referer)
} catch (e: Exception) {
logError(e)
emptyList()
}
}.flatten().distinct()
addTrailer(newTrailers)
}
fun LoadResponse.addImdbId(id: String?) {
@ -997,7 +1005,7 @@ data class TorrentLoadResponse(
override var rating: Int? = null,
override var tags: List<String>? = null,
override var duration: Int? = null,
override var trailers: List<String>? = null,
override var trailers: List<ExtractorLink>? = null,
override var recommendations: List<SearchResponse>? = null,
override var actors: List<ActorData>? = null,
override var comingSoon: Boolean = false,
@ -1025,7 +1033,7 @@ data class AnimeLoadResponse(
override var rating: Int? = null,
override var duration: Int? = null,
override var trailers: List<String>? = null,
override var trailers: List<ExtractorLink>? = null,
override var recommendations: List<SearchResponse>? = null,
override var actors: List<ActorData>? = null,
override var comingSoon: Boolean = false,
@ -1038,12 +1046,12 @@ fun AnimeLoadResponse.addEpisodes(status: DubStatus, episodes: List<Episode>?) {
this.episodes[status] = episodes
}
fun MainAPI.newAnimeLoadResponse(
suspend fun MainAPI.newAnimeLoadResponse(
name: String,
url: String,
type: TvType,
comingSoonIfNone: Boolean = true,
initializer: AnimeLoadResponse.() -> Unit = { },
initializer: suspend AnimeLoadResponse.() -> Unit = { },
): AnimeLoadResponse {
val builder = AnimeLoadResponse(name = name, url = url, apiName = this.name, type = type)
builder.initializer()
@ -1072,7 +1080,7 @@ data class MovieLoadResponse(
override var rating: Int? = null,
override var tags: List<String>? = null,
override var duration: Int? = null,
override var trailers: List<String>? = null,
override var trailers: List<ExtractorLink>? = null,
override var recommendations: List<SearchResponse>? = null,
override var actors: List<ActorData>? = null,
override var comingSoon: Boolean = false,
@ -1080,12 +1088,12 @@ data class MovieLoadResponse(
override var posterHeaders: Map<String, String>? = null,
) : LoadResponse
fun <T> MainAPI.newMovieLoadResponse(
suspend fun <T> MainAPI.newMovieLoadResponse(
name: String,
url: String,
type: TvType,
data: T?,
initializer: MovieLoadResponse.() -> Unit = { }
initializer: suspend MovieLoadResponse.() -> Unit = { }
): MovieLoadResponse {
// just in case
if (data is String) return newMovieLoadResponse(
@ -1108,12 +1116,12 @@ fun <T> MainAPI.newMovieLoadResponse(
return builder
}
fun MainAPI.newMovieLoadResponse(
suspend fun MainAPI.newMovieLoadResponse(
name: String,
url: String,
type: TvType,
dataUrl: String,
initializer: MovieLoadResponse.() -> Unit = { }
initializer: suspend MovieLoadResponse.() -> Unit = { }
): MovieLoadResponse {
val builder = MovieLoadResponse(
name = name,
@ -1193,7 +1201,7 @@ data class TvSeriesLoadResponse(
override var rating: Int? = null,
override var tags: List<String>? = null,
override var duration: Int? = null,
override var trailers: List<String>? = null,
override var trailers: List<ExtractorLink>? = null,
override var recommendations: List<SearchResponse>? = null,
override var actors: List<ActorData>? = null,
override var comingSoon: Boolean = false,
@ -1201,12 +1209,12 @@ data class TvSeriesLoadResponse(
override var posterHeaders: Map<String, String>? = null,
) : LoadResponse
fun MainAPI.newTvSeriesLoadResponse(
suspend fun MainAPI.newTvSeriesLoadResponse(
name: String,
url: String,
type: TvType,
episodes: List<Episode>,
initializer: TvSeriesLoadResponse.() -> Unit = { }
initializer: suspend TvSeriesLoadResponse.() -> Unit = { }
): TvSeriesLoadResponse {
val builder = TvSeriesLoadResponse(
name = name,

18
app/src/main/java/com/lagradost/cloudstream3/MainActivity.kt
View file

@ -76,6 +76,7 @@ import kotlinx.android.synthetic.main.fragment_result_swipe.*
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.schabi.newpipe.extractor.NewPipe
import java.io.File
import kotlin.concurrent.thread
@ -366,9 +367,23 @@ class MainActivity : AppCompatActivity(), ColorPickerDialogListener {
}
fun test() {
//val youtubeLink = "https://www.youtube.com/watch?v=TxB48MEAmZw"
/*thread {
val youtubeLink = "https://www.youtube.com/watch?v=Zxem9rqJ5S0"
val url = YoutubeStreamLinkHandlerFactory.getInstance().fromUrl(youtubeLink)
println("ID:::: ${url.id}")
NewPipe.init(DownloaderTestImpl.getInstance())
val service = ServiceList.YouTube
val s = object : YoutubeStreamExtractor(
service,
url
) {
}
s.fetchPage()
val streams = s.videoStreams
println("STREAMS: ${streams.map { "url = "+ it.url + " extra= " + it.height + "|" + it.isVideoOnly + "\n" }}")
}*/
/*
runBlocking {
@ -598,6 +613,7 @@ class MainActivity : AppCompatActivity(), ColorPickerDialogListener {
loadCache()
test()
NewPipe.init(DownloaderTestImpl.getInstance())
/*nav_view.setOnNavigationItemSelectedListener { item ->
when (item.itemId) {
R.id.navigation_home -> {

6
app/src/main/java/com/lagradost/cloudstream3/animeproviders/KuronimeProvider.kt
View file

@ -3,10 +3,11 @@ package com.lagradost.cloudstream3.animeproviders
import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.LoadResponse.Companion.addTrailer
import com.lagradost.cloudstream3.mvvm.safeApiCall
import com.lagradost.cloudstream3.utils.*
import com.lagradost.cloudstream3.utils.ExtractorLink
import com.lagradost.cloudstream3.utils.getQualityFromName
import com.lagradost.cloudstream3.utils.loadExtractor
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import java.util.ArrayList
class KuronimeProvider : MainAPI() {
override var mainUrl = "https://185.231.223.254"
@ -139,7 +140,6 @@ class KuronimeProvider : MainAPI() {
plot = description
addTrailer(trailer)
this.tags = tags
trailers = listOf(trailer)
}
}

68
app/src/main/java/com/lagradost/cloudstream3/extractors/YoutubeExtractor.kt Normal file
View file

@ -0,0 +1,68 @@
package com.lagradost.cloudstream3.extractors
import com.lagradost.cloudstream3.ErrorLoadingException
import com.lagradost.cloudstream3.mvvm.Resource
import com.lagradost.cloudstream3.mvvm.safeApiCall
import com.lagradost.cloudstream3.utils.ExtractorApi
import com.lagradost.cloudstream3.utils.ExtractorLink
import org.schabi.newpipe.extractor.ServiceList
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory
import org.schabi.newpipe.extractor.stream.VideoStream
class YoutubeExtractor : ExtractorApi() {
override val mainUrl = "https://www.youtube.com"
override val requiresReferer = false
override val name = "YouTube"
companion object {
private var ytVideos: MutableMap<String, List<VideoStream>> = mutableMapOf()
}
override fun getExtractorUrl(id: String): String {
return "https://www.youtube.com/watch?v=$id"
}
override suspend fun getUrl(url: String, referer: String?): List<ExtractorLink>? {
val streams = safeApiCall {
val streams = ytVideos[url] ?: let {
val link =
YoutubeStreamLinkHandlerFactory.getInstance().fromUrl(url)
val s = object : YoutubeStreamExtractor(
ServiceList.YouTube,
link
) {
}
s.fetchPage()
val streams = s.videoStreams ?: return@let emptyList()
ytVideos[url] = streams
streams
}
if (streams.isEmpty()) {
throw ErrorLoadingException("No Youtube streams")
}
streams
//streams.sortedBy { it.height }
// .firstOrNull { !it.isVideoOnly && it.height > 0 }
// ?: throw ErrorLoadingException("No valid Youtube stream")
}
if (streams is Resource.Success) {
return streams.value.mapNotNull {
if (it.isVideoOnly || it.height <= 0) return@mapNotNull null
ExtractorLink(
this.name,
this.name,
it.url ?: return@mapNotNull null,
"",
it.height
)
}
} else {
return null
}
}
}

4
app/src/main/java/com/lagradost/cloudstream3/metaproviders/TmdbProvider.kt
View file

@ -95,7 +95,7 @@ open class TmdbProvider : MainAPI() {
}
}
private fun TvShow.toLoadResponse(): TvSeriesLoadResponse {
private suspend fun TvShow.toLoadResponse(): TvSeriesLoadResponse {
val episodes = this.seasons?.filter { !disableSeasonZero || (it.season_number ?: 0) != 0 }
?.mapNotNull { season ->
season.episodes?.map { episode ->
@ -167,7 +167,7 @@ open class TmdbProvider : MainAPI() {
}
}
private fun Movie.toLoadResponse(): MovieLoadResponse {
private suspend fun Movie.toLoadResponse(): MovieLoadResponse {
return newMovieLoadResponse(
this.title ?: this.original_title, getUrl(id, false), TvType.Movie, TmdbLink(
this.imdb_id,

3
app/src/main/java/com/lagradost/cloudstream3/movieproviders/NginxProvider.kt
View file

@ -1,6 +1,7 @@
package com.lagradost.cloudstream3.movieproviders
import com.lagradost.cloudstream3.*
import com.lagradost.cloudstream3.LoadResponse.Companion.addTrailer
import com.lagradost.cloudstream3.utils.ExtractorLink
import com.lagradost.cloudstream3.utils.Qualities
@ -95,7 +96,7 @@ class NginxProvider : MainAPI() {
this.plot = description
this.rating = ratingAverage
this.tags = tagsList
this.trailers = trailer
addTrailer(trailer)
addPoster(poster, authHeader)
}
} else // a tv serie

57
app/src/main/java/com/lagradost/cloudstream3/ui/player/CS3IPlayer.kt
View file

@ -1,17 +1,11 @@
package com.lagradost.cloudstream3.ui.player
import android.annotation.SuppressLint
import android.content.Context
import android.net.Uri
import android.os.Handler
import android.os.Looper
import android.util.Log
import android.util.SparseArray
import android.widget.FrameLayout
import androidx.core.util.forEach
import at.huber.youtubeExtractor.VideoMeta
import at.huber.youtubeExtractor.YouTubeExtractor
import at.huber.youtubeExtractor.YtFile
import com.google.android.exoplayer2.*
import com.google.android.exoplayer2.database.StandaloneDatabaseProvider
import com.google.android.exoplayer2.ext.okhttp.OkHttpDataSource
@ -31,7 +25,6 @@ import com.google.android.exoplayer2.upstream.cache.SimpleCache
import com.google.android.exoplayer2.util.MimeTypes
import com.google.android.exoplayer2.video.VideoSize
import com.lagradost.cloudstream3.APIHolder.getApiFromName
import com.lagradost.cloudstream3.ErrorLoadingException
import com.lagradost.cloudstream3.USER_AGENT
import com.lagradost.cloudstream3.app
import com.lagradost.cloudstream3.mvvm.logError
@ -39,7 +32,6 @@ import com.lagradost.cloudstream3.mvvm.normalSafeApiCall
import com.lagradost.cloudstream3.ui.subtitles.SaveCaptionStyle
import com.lagradost.cloudstream3.utils.ExtractorLink
import com.lagradost.cloudstream3.utils.ExtractorUri
import com.lagradost.cloudstream3.utils.Qualities
import com.lagradost.cloudstream3.utils.SubtitleHelper.fromTwoLettersToLanguage
import java.io.File
import javax.net.ssl.HttpsURLConnection
@ -332,7 +324,6 @@ class CS3IPlayer : IPlayer {
}
companion object {
private var ytVideos: MutableMap<String, YtFile> = mutableMapOf()
private var simpleCache: SimpleCache? = null
var requestSubtitleUpdate: (() -> Unit)? = null
@ -883,57 +874,9 @@ class CS3IPlayer : IPlayer {
return Pair(subSources, activeSubtitles)
}
fun loadYtFile(context: Context, yt: YtFile) {
loadOnlinePlayer(
context,
ExtractorLink(
"YouTube",
"",
yt.url,
"",
yt.format?.height ?: Qualities.Unknown.value
)
)
}
private fun loadOnlinePlayer(context: Context, link: ExtractorLink) {
Log.i(TAG, "loadOnlinePlayer $link")
try {
if (link.url.contains("youtube.com")) {
val ytLink = link.url.replace("/embed/", "/watch?v=")
ytVideos[ytLink]?.let {
loadYtFile(context, it)
return
}
val ytExtractor =
@SuppressLint("StaticFieldLeak")
object : YouTubeExtractor(context) {
override fun onExtractionComplete(
ytFiles: SparseArray<YtFile>?,
videoMeta: VideoMeta?
) {
var yt: YtFile? = null
ytFiles?.forEach { _, value ->
if ((yt?.format?.height ?: 0) < (value.format?.height
?: -1) && (value.format?.audioBitrate ?: -1) > 0
) {
yt = value
}
}
yt?.let { ytf ->
ytVideos[ytLink] = ytf
loadYtFile(context, ytf)
} ?: run {
playerError?.invoke(ErrorLoadingException("No Link"))
}
}
}
Log.i(TAG, "YouTube extraction on $ytLink")
ytExtractor.extract(ytLink)
return
}
currentLink = link
if (ignoreSSL) {

14
app/src/main/java/com/lagradost/cloudstream3/ui/result/ResultFragment.kt
View file

@ -602,7 +602,7 @@ class ResultFragment : ResultTrailerPlayer() {
setFormatText(result_meta_rating, R.string.rating_format, rating?.div(1000f))
}
var currentTrailers: List<String> = emptyList()
var currentTrailers: List<ExtractorLink> = emptyList()
var currentTrailerIndex = 0
override fun nextMirror() {
@ -626,13 +626,7 @@ class ResultFragment : ResultTrailerPlayer() {
player.loadPlayer(
ctx,
false,
ExtractorLink(
"",
"Trailer",
trailer,
"",
Qualities.Unknown.value
),
null,
startPosition = 0L,
subtitles = emptySet(),
@ -649,14 +643,14 @@ class ResultFragment : ResultTrailerPlayer() {
result_trailer_loading?.isVisible = isSuccess
}
private fun setTrailers(trailers: List<String>?) {
private fun setTrailers(trailers: List<ExtractorLink>?) {
context?.let { ctx ->
if (ctx.isTvSettings()) return
val settingsManager = PreferenceManager.getDefaultSharedPreferences(ctx)
val showTrailers =
settingsManager.getBoolean(ctx.getString(R.string.show_trailers_key), true)
if (!showTrailers) return
currentTrailers = trailers ?: emptyList()
currentTrailers = trailers?.sortedBy { -it.quality } ?: emptyList()
loadTrailer()
}
}
@ -769,7 +763,7 @@ class ResultFragment : ResultTrailerPlayer() {
player_open_source?.setOnClickListener {
currentTrailers.getOrNull(currentTrailerIndex)?.let {
context?.openBrowser(it)
context?.openBrowser(it.url)
}
}

4
app/src/main/java/com/lagradost/cloudstream3/ui/result/ResultViewModel.kt
View file

@ -118,7 +118,7 @@ class ResultViewModel : ViewModel() {
}
var lastMeta: SyncAPI.SyncResult? = null
private fun applyMeta(resp: LoadResponse, meta: SyncAPI.SyncResult?): LoadResponse {
private suspend fun applyMeta(resp: LoadResponse, meta: SyncAPI.SyncResult?): LoadResponse {
if (meta == null) return resp
lastMeta = meta
return resp.apply {
@ -145,7 +145,7 @@ class ResultViewModel : ViewModel() {
}
}
fun setMeta(meta: SyncAPI.SyncResult) {
fun setMeta(meta: SyncAPI.SyncResult) = viewModelScope.launch {
Log.i(TAG, "setMeta")
(result.value as? Resource.Success<LoadResponse>?)?.value?.let { resp ->
_resultResponse.postValue(Resource.Success(applyMeta(resp, meta)))

14
app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt
View file

@ -106,6 +106,19 @@ suspend fun loadExtractor(
return false
}
suspend fun loadExtractor(
url: String,
referer: String? = null,
): List<ExtractorLink> {
for (extractor in extractorApis) {
if (url.startsWith(extractor.mainUrl)) {
return extractor.getSafeUrl(url, referer) ?: emptyList()
}
}
return emptyList()
}
val extractorApis: Array<ExtractorApi> = arrayOf(
//AllProvider(),
WcoStream(),
@ -205,6 +218,7 @@ val extractorApis: Array<ExtractorApi> = arrayOf(
KotakAnimeid(),
Neonime8n(),
Neonime7n(),
YoutubeExtractor(),
)
fun getExtractorApiFromName(name: String): ExtractorApi {