Improve tests (#1142)

app/src/main/java/com/lagradost/cloudstream3/MainAPI.kt

@@ -622,7 +622,7 @@ abstract class MainAPI {
     /**Used for testing and can be used to disable the providers if WebView is not available*/
     open val usesWebView = false
 
-    /** Determines which plugin a given provider is from */
+    /** Determines which plugin a given provider is from. This is the full path to the plugin. */
     var sourcePlugin: String? = null
 
     open val hasMainPage = false

app/src/main/java/com/lagradost/cloudstream3/plugins/Plugin.kt

@@ -67,6 +67,7 @@ abstract class Plugin {
      * This will contain your resources if you specified requiresResources in gradle
      */
     var resources: Resources? = null
+    /** Full file path to the plugin. */
     var __filename: String? = null
 
     /**

app/src/main/java/com/lagradost/cloudstream3/plugins/PluginManager.kt

@@ -18,7 +18,6 @@ import com.lagradost.cloudstream3.APIHolder.getApiProviderLangSettings
 import com.lagradost.cloudstream3.APIHolder.removePluginMapping
 import com.lagradost.cloudstream3.AcraApplication.Companion.getActivity
 import com.lagradost.cloudstream3.AcraApplication.Companion.getKey
-import com.lagradost.cloudstream3.AcraApplication.Companion.removeKey
 import com.lagradost.cloudstream3.AcraApplication.Companion.setKey
 import com.lagradost.cloudstream3.CommonActivity.showToast
 import com.lagradost.cloudstream3.MainAPI.Companion.settingsForProvider
@@ -518,7 +517,7 @@ object PluginManager {
                 return true
             }
 
-            pluginInstance.__filename = fileName
+            pluginInstance.__filename = file.absolutePath
             if (manifest.requiresResources) {
                 Log.d(TAG, "Loading resources for ${data.internalName}")
                 // based on https://stackoverflow.com/questions/7483568/dynamic-resource-loading-from-other-apk

app/src/main/java/com/lagradost/cloudstream3/ui/settings/testing/TestResultAdapter.kt

@@ -2,26 +2,31 @@ package com.lagradost.cloudstream3.ui.settings.testing
 
 import android.app.AlertDialog
 import android.view.LayoutInflater
-import android.view.View
 import android.view.ViewGroup
 import android.widget.ImageView
 import android.widget.TextView
+import android.widget.Toast
 import androidx.core.content.ContextCompat
 import androidx.recyclerview.widget.RecyclerView
+import com.lagradost.cloudstream3.CommonActivity.showToast
 import com.lagradost.cloudstream3.MainAPI
 import com.lagradost.cloudstream3.R
 import com.lagradost.cloudstream3.databinding.ProviderTestItemBinding
 import com.lagradost.cloudstream3.mvvm.getAllMessages
 import com.lagradost.cloudstream3.mvvm.getStackTracePretty
+import com.lagradost.cloudstream3.plugins.PluginManager
 import com.lagradost.cloudstream3.utils.AppUtils
+import com.lagradost.cloudstream3.utils.Coroutines.ioSafe
+import com.lagradost.cloudstream3.utils.Coroutines.runOnMainThread
 import com.lagradost.cloudstream3.utils.SubtitleHelper.getFlagFromIso
 import com.lagradost.cloudstream3.utils.TestingUtils
+import java.io.File
 
 class TestResultAdapter(override val items: MutableList<Pair<MainAPI, TestingUtils.TestResultProvider>>) :
     AppUtils.DiffAdapter<Pair<MainAPI, TestingUtils.TestResultProvider>>(items) {
     override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): RecyclerView.ViewHolder {
         return ProviderTestViewHolder(
-            ProviderTestItemBinding.inflate(LayoutInflater.from(parent.context), parent,false)
+            ProviderTestItemBinding.inflate(LayoutInflater.from(parent.context), parent, false)
             //LayoutInflater.from(parent.context)
             //    .inflate(R.layout.provider_test_item, parent, false),
         )
@@ -36,7 +41,8 @@ class TestResultAdapter(override val items: MutableList<Pair<MainAPI, TestingUti
         }
     }
 
-    inner class ProviderTestViewHolder(binding: ProviderTestItemBinding) : RecyclerView.ViewHolder(binding.root) {
+    inner class ProviderTestViewHolder(binding: ProviderTestItemBinding) :
+        RecyclerView.ViewHolder(binding.root) {
         private val languageText: TextView = binding.langIcon
         private val providerTitle: TextView = binding.mainText
         private val statusText: TextView = binding.passedFailedMarker
@@ -52,7 +58,11 @@ class TestResultAdapter(override val items: MutableList<Pair<MainAPI, TestingUti
             providerTitle.text = api.name
 
             val (resultText, resultColor) = if (result.success) {
+                if (result.log.any { it.level == TestingUtils.Logger.LogLevel.Warning }) {
+                    R.string.test_warning to R.color.colorTestWarning
+                } else {
                     R.string.test_passed to R.color.colorTestPass
+                }
             } else {
                 R.string.test_failed to R.color.colorTestFail
             }
@@ -62,17 +72,43 @@ class TestResultAdapter(override val items: MutableList<Pair<MainAPI, TestingUti
 
             val stackTrace = result.exception?.getStackTracePretty(false)?.ifBlank { null }
             val messages = result.exception?.getAllMessages()?.ifBlank { null }
+            val resultLog = result.log.joinToString("\n")
             val fullLog =
-                result.log + (messages?.let { "\n\n$it" } ?: "") + (stackTrace?.let { "\n\n$it" } ?: "")
+                resultLog +
+                        (messages?.let { "\n\nError: $it" } ?: "") +
+                        (stackTrace?.let { "\n\n$it" } ?: "")
 
-            failDescription.text = messages?.lastLine() ?: result.log.lastLine()
+            failDescription.text = messages?.lastLine() ?: resultLog.lastLine()
 
             logButton.setOnClickListener {
                 val builder: AlertDialog.Builder =
                     AlertDialog.Builder(it.context, R.style.AlertDialogCustom)
                 builder.setMessage(fullLog)
                     .setTitle(R.string.test_log)
-                    .show()
+                    // Ok button just closes the dialog
+                    .setPositiveButton(R.string.ok) { _, _ -> }
+
+                api.sourcePlugin?.let { path ->
+                    val pluginFile = File(path)
+                    // Cannot delete a deleted plugin
+                    if (!pluginFile.exists()) return@let
+
+                    builder.setNegativeButton(R.string.delete_plugin) { _, _ ->
+                        ioSafe {
+                            val success = PluginManager.deletePlugin(pluginFile)
+
+                            runOnMainThread {
+                                if (success) {
+                                    showToast(R.string.plugin_deleted, Toast.LENGTH_SHORT)
+                                } else {
+                                    showToast(R.string.error, Toast.LENGTH_SHORT)
+                                }
+                            }
+                        }
+                    }
+                }
+
+                builder.show()
             }
         }
     }

app/src/main/java/com/lagradost/cloudstream3/ui/settings/testing/TestViewModel.kt

@@ -95,7 +95,7 @@ class TestViewModel : ViewModel() {
         providers.clear()
         updateProgress()
 
-        TestingUtils.getDeferredProviderTests(scope ?: return, apis, ::println) { api, result ->
+        TestingUtils.getDeferredProviderTests(scope ?: return, apis) { api, result ->
             addProvider(api, result)
         }
     }

app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt

@@ -1015,7 +1015,7 @@ abstract class ExtractorApi {
     abstract val mainUrl: String
     abstract val requiresReferer: Boolean
 
-    /** Determines which plugin a given extractor is from */
+    /** Determines which plugin a given provider is from. This is the full path to the plugin. */
     var sourcePlugin: String? = null
 
     //suspend fun getSafeUrl(url: String, referer: String? = null): List<ExtractorLink>? {

app/src/main/java/com/lagradost/cloudstream3/utils/TestingUtils.kt

@@ -13,16 +13,55 @@ object TestingUtils {
         }
     }
 
-    class TestResultSearch(val results: List<SearchResponse>) : TestResult(true)
+    class Logger {
-    class TestResultLoad(val extractorData: String) : TestResult(true)
+        enum class LogLevel {
+            Normal,
+            Warning,
+            Error;
+        }
 
-    class TestResultProvider(success: Boolean, val log: String, val exception: Throwable?) :
+        data class Message(val level: LogLevel, val message: String) {
+            override fun toString(): String {
+                val level = when (this.level) {
+                    LogLevel.Normal -> ""
+                    LogLevel.Warning -> "Warning: "
+                    LogLevel.Error -> "Error: "
+                }
+                return "$level$message"
+            }
+        }
+
+        private val messageLog = mutableListOf<Message>()
+
+        fun getRawLog(): List<Message> = messageLog
+
+        fun log(message: String) {
+            messageLog.add(Message(LogLevel.Normal, message))
+        }
+
+        fun warn(message: String) {
+            messageLog.add(Message(LogLevel.Warning, message))
+        }
+
+        fun error(message: String) {
+            messageLog.add(Message(LogLevel.Error, message))
+        }
+    }
+
+    class TestResultList(val results: List<SearchResponse>) : TestResult(true)
+    class TestResultLoad(val extractorData: String, val shouldLoadLinks: Boolean) : TestResult(true)
+
+    class TestResultProvider(
+        success: Boolean,
+        val log: List<Logger.Message>,
+        val exception: Throwable?
+    ) :
         TestResult(success)
 
     @Throws(AssertionError::class, CancellationException::class)
     suspend fun testHomepage(
         api: MainAPI,
-        logger: (String) -> Unit
+        logger: Logger
     ): TestResult {
         if (api.hasMainPage) {
             try {
@@ -31,22 +70,33 @@ object TestingUtils {
                     api.getMainPage(1, MainPageRequest(f.name, f.data, f.horizontalImages))
                 when {
                     homepage == null -> {
-                        logger.invoke("Homepage provider ${api.name} did not correctly load homepage!")
+                        logger.error("Provider ${api.name} did not correctly load homepage!")
                     }
+
                     homepage.items.isEmpty() -> {
-                        logger.invoke("Homepage provider ${api.name} does not contain any items!")
+                        logger.warn("Provider ${api.name} does not contain any homepage rows!")
                     }
+
                     homepage.items.any { it.list.isEmpty() } -> {
-                        logger.invoke("Homepage provider ${api.name} does not have any items on result!")
+                        logger.warn("Provider ${api.name} does not have any items in a homepage row!")
                     }
                 }
+                val homePageList = homepage?.items?.flatMap { it.list } ?: emptyList()
+                return TestResultList(homePageList)
             } catch (e: Throwable) {
-                if (e is NotImplementedError) {
+                when (e) {
+                    is NotImplementedError -> {
                         Assert.fail("Provider marked as hasMainPage, while in reality is has not been implemented")
-                } else if (e is CancellationException) {
+                    }
+
+                    is CancellationException -> {
                         throw e
                     }
-                logError(e)
+
+                    else -> {
+                        e.message?.let { logger.warn("Exception thrown when loading homepage: \"$it\"") }
+                    }
+                }
             }
         }
         return TestResult.Pass
@@ -54,11 +104,13 @@ object TestingUtils {
 
     @Throws(AssertionError::class, CancellationException::class)
     private suspend fun testSearch(
-        api: MainAPI
+        api: MainAPI,
+        testQueries: List<String>,
+        logger: Logger,
     ): TestResult {
-        val searchQueries = listOf("over", "iron", "guy")
+        val searchResults = testQueries.firstNotNullOfOrNull { query ->
-        val searchResults = searchQueries.firstNotNullOfOrNull { query ->
             try {
+                logger.log("Searching for: $query")
                 api.search(query).takeIf { !it.isNullOrEmpty() }
             } catch (e: Throwable) {
                 if (e is NotImplementedError) {
@@ -72,12 +124,11 @@ object TestingUtils {
         }
 
         return if (searchResults.isNullOrEmpty()) {
-            Assert.fail("Api ${api.name} did not return any valid search responses")
+            Assert.fail("Api ${api.name} did not return any search responses")
             TestResult.Fail // Should not be reached
         } else {
-            TestResultSearch(searchResults)
+            TestResultList(searchResults)
         }
-
     }
 
 
@@ -85,31 +136,27 @@ object TestingUtils {
     private suspend fun testLoad(
         api: MainAPI,
         result: SearchResponse,
-        logger: (String) -> Unit
+        logger: Logger
     ): TestResult {
         try {
-            Assert.assertEquals(
+            if (result.apiName != api.name) {
-                "Invalid apiName on SearchResponse on ${api.name}",
+                logger.warn("Wrong apiName on SearchResponse: ${api.name} != ${result.apiName}")
-                result.apiName,
+            }
-                api.name
-            )
 
             val loadResponse = api.load(result.url)
 
             if (loadResponse == null) {
-                logger.invoke("Returned null loadResponse on ${result.url} on ${api.name}")
+                logger.error("Returned null loadResponse on ${result.url} on ${api.name}")
                 return TestResult.Fail
             }
 
-            Assert.assertEquals(
+            if (loadResponse.apiName != api.name) {
-                "Invalid apiName on LoadResponse on ${api.name}",
+                logger.warn("Wrong apiName on LoadResponse: ${api.name} != ${loadResponse.apiName}")
-                loadResponse.apiName,
+            }
-                result.apiName
+
-            )
+            if (!api.supportedTypes.contains(loadResponse.type)) {
-            Assert.assertTrue(
+                logger.warn("Api ${api.name} on load does not contain any of the supportedTypes: ${loadResponse.type}")
-                "Api ${api.name} on load does not contain any of the supportedTypes: ${loadResponse.type}",
+            }
-                api.supportedTypes.contains(loadResponse.type)
-            )
 
             val url = when (loadResponse) {
                 is AnimeLoadResponse -> {
@@ -117,39 +164,43 @@ object TestingUtils {
                         loadResponse.episodes.keys.isEmpty() || loadResponse.episodes.keys.any { loadResponse.episodes[it].isNullOrEmpty() }
 
                     if (gotNoEpisodes) {
-                        logger.invoke("Api ${api.name} got no episodes on ${loadResponse.url}")
+                        logger.error("Api ${api.name} got no episodes on ${loadResponse.url}")
                         return TestResult.Fail
                     }
 
                     (loadResponse.episodes[loadResponse.episodes.keys.firstOrNull()])?.firstOrNull()?.data
                 }
+
                 is MovieLoadResponse -> {
                     val gotNoEpisodes = loadResponse.dataUrl.isBlank()
                     if (gotNoEpisodes) {
-                        logger.invoke("Api ${api.name} got no movie on ${loadResponse.url}")
+                        logger.error("Api ${api.name} got no movie on ${loadResponse.url}")
                         return TestResult.Fail
                     }
 
                     loadResponse.dataUrl
                 }
+
                 is TvSeriesLoadResponse -> {
                     val gotNoEpisodes = loadResponse.episodes.isEmpty()
                     if (gotNoEpisodes) {
-                        logger.invoke("Api ${api.name} got no episodes on ${loadResponse.url}")
+                        logger.error("Api ${api.name} got no episodes on ${loadResponse.url}")
                         return TestResult.Fail
                     }
                     loadResponse.episodes.firstOrNull()?.data
                 }
+
                 is LiveStreamLoadResponse -> {
                     loadResponse.dataUrl
                 }
+
                 else -> {
-                    logger.invoke("Unknown load response: ${loadResponse.javaClass.name}")
+                    logger.error("Unknown load response: ${loadResponse.javaClass.name}")
                     return TestResult.Fail
                 }
             } ?: return TestResult.Fail
 
-            return TestResultLoad(url)
+            return TestResultLoad(url, loadResponse.type != TvType.CustomMedia)
 
 //            val loadTest = testLoadResponse(api, load, logger)
 //            if (loadTest is TestResultLoad) {
@@ -174,7 +225,7 @@ object TestingUtils {
     private suspend fun testLinkLoading(
         api: MainAPI,
         url: String?,
-        logger: (String) -> Unit
+        logger: Logger
     ): TestResult {
         Assert.assertNotNull("Api ${api.name} has invalid url on episode", url)
         if (url == null) return TestResult.Fail // Should never trigger
@@ -182,7 +233,7 @@ object TestingUtils {
         var linksLoaded = 0
         try {
             val success = api.loadLinks(url, false, {}) { link ->
-                logger.invoke("Video loaded: ${link.name}")
+                logger.log("Video loaded: ${link.name}")
                 Assert.assertTrue(
                     "Api ${api.name} returns link with invalid url ${link.url}",
                     link.url.length > 4
@@ -190,7 +241,7 @@ object TestingUtils {
                 linksLoaded++
             }
             if (success) {
-                logger.invoke("Links loaded: $linksLoaded")
+                logger.log("Links loaded: $linksLoaded")
                 return TestResult(linksLoaded > 0)
             } else {
                 Assert.fail("Api ${api.name} returns false on loadLinks() with $linksLoaded links loaded")
@@ -200,8 +251,9 @@ object TestingUtils {
                 is NotImplementedError -> {
                     Assert.fail("Provider has not implemented loadLinks()")
                 }
+
                 else -> {
-                    logger.invoke("Failed link loading on ${api.name} using data: $url")
+                    logger.error("Failed link loading on ${api.name} using data: $url")
                     throw e
                 }
             }
@@ -212,53 +264,57 @@ object TestingUtils {
     fun getDeferredProviderTests(
         scope: CoroutineScope,
         providers: Array<MainAPI>,
-        logger: (String) -> Unit,
         callback: (MainAPI, TestResultProvider) -> Unit
     ) {
         providers.forEach { api ->
             scope.launch {
-                var log = ""
+                val logger = Logger()
-                fun addToLog(string: String) {
-                    log += string + "\n"
-                    logger.invoke(string)
-                }
-                fun getLog(): String {
-                    return log.removeSuffix("\n")
-                }
 
                 val result = try {
-                    addToLog("Trying ${api.name}")
+                    logger.log("Trying ${api.name}")
 
                     // Test Homepage
-                    val homepage = testHomepage(api, logger).success
+                    val homepage = testHomepage(api, logger)
-                    Assert.assertTrue("Homepage failed to load", homepage)
+                    Assert.assertTrue("Homepage failed to load", homepage.success)
+                    val homePageList = (homepage as? TestResultList)?.results ?: emptyList()
 
                     // Test Search Results
-                    val searchResults = testSearch(api)
+                    val searchQueries =
+                        // Use the first 3 home page results as queries since they are guaranteed to exist
+                        (homePageList.take(3).map { it.name } +
+                                // If home page is sparse then use generic search queries
+                                listOf("over", "iron", "guy")).take(3)
+
+                    val searchResults = testSearch(api, searchQueries, logger)
                     Assert.assertTrue("Failed to get search results", searchResults.success)
-                    searchResults as TestResultSearch
+                    searchResults as TestResultList
 
                     // Test Load and LoadLinks
                     // Only try the first 3 search results to prevent spamming
                     val success = searchResults.results.take(3).any { searchResponse ->
-                        addToLog("Testing search result: ${searchResponse.url}")
+                        logger.log("Testing search result: ${searchResponse.url}")
-                        val loadResponse = testLoad(api, searchResponse, ::addToLog)
+                        val loadResponse = testLoad(api, searchResponse, logger)
                         if (loadResponse !is TestResultLoad) {
                             false
                         } else {
-                            testLinkLoading(api, loadResponse.extractorData, ::addToLog).success
+                            if (loadResponse.shouldLoadLinks) {
+                                testLinkLoading(api, loadResponse.extractorData, logger).success
+                            } else {
+                                logger.log("Skipping link loading test")
+                                true
+                            }
                         }
                     }
 
                     if (success) {
-                        logger.invoke("Success ${api.name}")
+                        logger.log("Success ${api.name}")
-                        TestResultProvider(true, getLog(), null)
+                        TestResultProvider(true, logger.getRawLog(), null)
                     } else {
-                        logger.invoke("Error ${api.name}")
+                        logger.error("Link loading failed")
-                        TestResultProvider(false, getLog(), null)
+                        TestResultProvider(false, logger.getRawLog(), null)
                     }
                 } catch (e: Throwable) {
-                    TestResultProvider(false, getLog(), e)
+                    TestResultProvider(false, logger.getRawLog(), e)
                 }
                 callback.invoke(api, result)
             }

app/src/main/res/values/colors.xml

@@ -88,4 +88,5 @@
 
     <color name="colorTestPass">#48E484</color>
     <color name="colorTestFail">#ea596e</color>
+    <color name="colorTestWarning">#FF9800</color>
 </resources>

app/src/main/res/values/strings.xml

@@ -304,6 +304,7 @@
     <string name="start">Start</string>
     <string name="test_failed">Failed</string>
     <string name="test_passed">Passed</string>
+    <string name="test_warning">Warning</string>
     <string name="resume">Resume</string>
     <string name="go_back_30">-30</string>
     <string name="go_forward_30">+30</string>
@@ -609,6 +610,7 @@
     <string name="plugin">plugins</string>
     <string name="delete_repository_plugins">This will also delete all repository plugins</string>
     <string name="delete_repository">Delete repository</string>
+    <string name="delete_plugin">Delete plugin</string>
     <string name="setup_extensions_subtext">Download the list of sites you want to use</string>
     <string name="plugins_downloaded" formatted="true">Downloaded: %d</string>
     <string name="plugins_disabled" formatted="true">Disabled: %d</string>