Merge remote-tracking branch 'origin/master'

app/build.gradle

@@ -171,4 +171,7 @@ dependencies {
     implementation 'com.facebook.shimmer:shimmer:0.5.0'
 
     implementation "androidx.tvprovider:tvprovider:1.0.0"
+
+    // used for subtitle decoding https://github.com/albfernandez/juniversalchardet
+    implementation 'com.github.albfernandez:juniversalchardet:2.4.0'
 }

app/src/main/java/com/lagradost/cloudstream3/ui/player/AbstractPlayerFragment.kt

@@ -394,6 +394,7 @@ abstract class AbstractPlayerFragment(
     override fun onDestroy() {
         playerEventListener = null
         keyEventListener = null
+        canEnterPipMode = false
         SubtitlesFragment.applyStyleEvent -= ::onSubStyleChanged
 
         keepScreenOn(false)

app/src/main/java/com/lagradost/cloudstream3/ui/player/CustomSubtitleDecoderFactory.kt

@@ -1,6 +1,8 @@
 package com.lagradost.cloudstream3.ui.player
 
+import android.content.Context
 import android.util.Log
+import androidx.preference.PreferenceManager
 import com.google.android.exoplayer2.Format
 import com.google.android.exoplayer2.text.SubtitleDecoder
 import com.google.android.exoplayer2.text.SubtitleDecoderFactory
@@ -11,13 +13,29 @@ import com.google.android.exoplayer2.text.subrip.SubripDecoder
 import com.google.android.exoplayer2.text.ttml.TtmlDecoder
 import com.google.android.exoplayer2.text.webvtt.WebvttDecoder
 import com.google.android.exoplayer2.util.MimeTypes
+import com.lagradost.cloudstream3.R
 import com.lagradost.cloudstream3.mvvm.logError
+import org.mozilla.universalchardet.UniversalDetector
 import java.nio.ByteBuffer
 
-
 class CustomDecoder : SubtitleDecoder {
     companion object {
+        fun updateForcedEncoding(context: Context) {
+            val settingsManager = PreferenceManager.getDefaultSharedPreferences(context)
+            val value = settingsManager.getString(
+                context.getString(R.string.subtitles_encoding_key),
+                null
+            )
+            overrideEncoding = if (value.isNullOrBlank()) {
+                null
+            } else {
+                value
+            }
+        }
+
+        private const val UTF_8 = "UTF-8"
         private const val TAG = "CustomDecoder"
+        private var overrideEncoding: String? = null
         var regexSubtitlesToRemoveCaptions = false
         val bloatRegex =
             listOf(
@@ -65,18 +83,67 @@ class CustomDecoder : SubtitleDecoder {
             if (realDecoder == null) {
                 inputBuffer.data?.let { data ->
                     // this way we read the subtitle file and decide what decoder to use instead of relying on mimetype
+                    Log.i(TAG, "Got data from queueInputBuffer")
 
-                    val pos = data.position()
+                    var (str, charset) = try {
                         data.position(0)
-                    val arr = ByteArray(minOf(data.remaining(), 100))
+                        val fullDataArr = ByteArray(data.remaining())
-                    data.get(arr)
+                        data.get(fullDataArr)
-                    data.position(pos)
+                        val encoding = try {
+                            val encoding = overrideEncoding ?: run {
+                                val detector = UniversalDetector()
 
+                                detector.handleData(fullDataArr, 0, fullDataArr.size)
+                                detector.dataEnd()
+
+                                detector.detectedCharset // "windows-1256"
+                            }
+
+                            Log.i(
+                                TAG,
+                                "Detected encoding with charset $encoding and override = $overrideEncoding"
+                            )
+                            encoding ?: UTF_8
+                        } catch (e: Exception) {
+                            Log.e(TAG, "Failed to detect encoding throwing error")
+                            logError(e)
+                            UTF_8
+                        }
+
+                        var (fullStr, charset) = try {
+                            val set = charset(encoding)
+                            Pair(String(fullDataArr, set), set)
+                        } catch (e: Exception) {
+                            Log.e(TAG, "Failed to parse using encoding $encoding")
+                            logError(e)
+                            Pair(fullDataArr.decodeToString(), charset(UTF_8))
+                        }
+
+                        bloatRegex.forEach { rgx ->
+                            fullStr = fullStr.replace(rgx, "\n")
+                        }
+
+                        fullStr.replace(Regex("(\r\n|\r|\n){2,}"), "\n")
+                        // fullStr = "1\n00:00:01,616 --> 00:00:40,200\n" +
+                        //         "تــــرجــمة"
+
+                        Log.i(
+                            TAG,
+                            "Encoded Text start: " + fullStr.substring(
+                                0,
+                                minOf(fullStr.length, 300)
+                            )
+                        )
+                        Pair(fullStr, charset)
+                    } catch (e: Exception) {
+                        Log.e(TAG, "Failed to parse text returning plain data")
+                        logError(e)
+                        return
+                    }
                     //https://emptycharacter.com/
                     //https://www.fileformat.info/info/unicode/char/200b/index.htm
-                    val str = trimStr(arr.decodeToString())
+                    //val str = trimStr(arr.decodeToString())
-                    Log.i(TAG, "Got data from queueInputBuffer")
+                    //Log.i(TAG, "first string is >>>$str<<<")
-                    Log.i(TAG, "first string is >>>$str<<<")
                     if (str.isNotEmpty()) {
                         //https://github.com/LagradOst/CloudStream-2/blob/ddd774ee66810137ff7bd65dae70bcf3ba2d2489/CloudStreamForms/CloudStreamForms/Script/MainChrome.cs#L388
                         realDecoder = when {
@@ -93,33 +160,21 @@ class CustomDecoder : SubtitleDecoder {
                             TAG,
                             "Decoder selected: $realDecoder"
                         )
-                        val decoder = realDecoder
+                        realDecoder?.let { decoder ->
-                        if (decoder != null) {
                             decoder.dequeueInputBuffer()?.let { buff ->
                                 if (regexSubtitlesToRemoveCaptions && decoder::class.java != SsaDecoder::class.java) {
-                                    try {
-                                        data.position(0)
-                                        val fullDataArr = ByteArray(data.remaining())
-                                        data.get(fullDataArr)
-                                        var fullStr = trimStr(fullDataArr.decodeToString())
-
-                                        bloatRegex.forEach { rgx ->
-                                            fullStr = fullStr.replace(rgx, "\n")
-                                        }
                                     captionRegex.forEach { rgx ->
-                                            fullStr = fullStr.replace(rgx, "\n")
+                                        str = str.replace(rgx, "\n")
+                                    }
                                 }
-                                        fullStr.replace(Regex("(\r\n|\r|\n){2,}"), "\n")
 
-                                        buff.data = ByteBuffer.wrap(fullStr.toByteArray())
+                                buff.data = ByteBuffer.wrap(str.toByteArray(charset = charset))
-                                    } catch (e: Exception) {
+
-                                        data.position(pos)
-                                        buff.data = data
-                                    }
-                                } else {
-                                    buff.data = data
-                                }
                                 decoder.queueInputBuffer(buff)
+                                Log.i(
+                                    TAG,
+                                    "Decoder queueInputBuffer successfully"
+                                )
                             }
                             CS3IPlayer.requestSubtitleUpdate?.invoke()
                         }

app/src/main/java/com/lagradost/cloudstream3/ui/player/GeneratorPlayer.kt

@@ -15,7 +15,6 @@ import androidx.core.view.isVisible
 import androidx.lifecycle.ViewModelProvider
 import androidx.preference.PreferenceManager
 import com.google.android.exoplayer2.util.MimeTypes
-import com.google.android.material.button.MaterialButton
 import com.hippo.unifile.UniFile
 import com.lagradost.cloudstream3.*
 import com.lagradost.cloudstream3.APIHolder.getApiFromNameNull
@@ -24,6 +23,7 @@ import com.lagradost.cloudstream3.mvvm.Resource
 import com.lagradost.cloudstream3.mvvm.logError
 import com.lagradost.cloudstream3.mvvm.normalSafeApiCall
 import com.lagradost.cloudstream3.mvvm.observe
+import com.lagradost.cloudstream3.ui.player.CustomDecoder.Companion.updateForcedEncoding
 import com.lagradost.cloudstream3.ui.player.PlayerSubtitleHelper.Companion.toSubtitleMimeType
 import com.lagradost.cloudstream3.ui.result.ResultEpisode
 import com.lagradost.cloudstream3.ui.result.ResultFragment
@@ -32,11 +32,13 @@ import com.lagradost.cloudstream3.ui.settings.SettingsFragment.Companion.isTvSet
 import com.lagradost.cloudstream3.ui.subtitles.SubtitlesFragment
 import com.lagradost.cloudstream3.utils.*
 import com.lagradost.cloudstream3.utils.Coroutines.ioSafe
+import com.lagradost.cloudstream3.utils.SingleSelectionHelper.showDialog
 import com.lagradost.cloudstream3.utils.UIHelper.dismissSafe
 import com.lagradost.cloudstream3.utils.UIHelper.hideSystemUI
 import com.lagradost.cloudstream3.utils.UIHelper.popCurrentPage
 import kotlinx.android.synthetic.main.fragment_player.*
 import kotlinx.android.synthetic.main.player_custom_layout.*
+import kotlinx.android.synthetic.main.player_select_source_and_subs.*
 import kotlinx.coroutines.Job
 
 class GeneratorPlayer : FullScreenPlayer() {
@@ -241,14 +243,8 @@ class GeneratorPlayer : FullScreenPlayer() {
                 val sourceDialog = sourceBuilder.create()
                 selectSourceDialog = sourceDialog
                 sourceDialog.show()
-                val providerList =
+                val providerList = sourceDialog.sort_providers
-                    sourceDialog.findViewById<ListView>(R.id.sort_providers)!!
+                val subtitleList = sourceDialog.sort_subtitles
-                val subtitleList =
-                    sourceDialog.findViewById<ListView>(R.id.sort_subtitles)!!
-                val applyButton =
-                    sourceDialog.findViewById<MaterialButton>(R.id.apply_btt)!!
-                val cancelButton =
-                    sourceDialog.findViewById<MaterialButton>(R.id.cancel_btt)!!
 
                 val footer: TextView =
                     layoutInflater.inflate(R.layout.sort_bottom_footer_add_choice, null) as TextView
@@ -314,11 +310,56 @@ class GeneratorPlayer : FullScreenPlayer() {
                     subtitleList.setItemChecked(which, true)
                 }
 
-                cancelButton.setOnClickListener {
+                sourceDialog.cancel_btt?.setOnClickListener {
                     sourceDialog.dismissSafe(activity)
                 }
 
-                applyButton.setOnClickListener {
+                sourceDialog.subtitles_encoding_format?.apply {
+                    val settingsManager = PreferenceManager.getDefaultSharedPreferences(ctx)
+
+                    val prefNames = ctx.resources.getStringArray(R.array.subtitles_encoding_list)
+                    val prefValues = ctx.resources.getStringArray(R.array.subtitles_encoding_values)
+
+                    val value = settingsManager.getString(
+                        ctx.getString(R.string.subtitles_encoding_key),
+                        null
+                    )
+                    val index = prefValues.indexOf(value)
+                    text = prefNames[if(index == -1) 0 else index]
+                }
+
+                sourceDialog.subtitles_click_settings?.setOnClickListener {
+                    val settingsManager = PreferenceManager.getDefaultSharedPreferences(ctx)
+
+                    val prefNames = ctx.resources.getStringArray(R.array.subtitles_encoding_list)
+                    val prefValues = ctx.resources.getStringArray(R.array.subtitles_encoding_values)
+
+                    val currentPrefMedia =
+                        settingsManager.getString(
+                            getString(R.string.subtitles_encoding_key),
+                            null
+                        )
+
+                    val index = prefValues.indexOf(currentPrefMedia)
+                    sourceDialog.dismissSafe(activity)
+                    activity?.showDialog(
+                        prefNames.toList(),
+                        if(index == -1) 0 else index,
+                        ctx.getString(R.string.subtitles_encoding),
+                        true,
+                        {}) {
+                        settingsManager.edit()
+                            .putString(
+                                ctx.getString(R.string.subtitles_encoding_key),
+                                prefValues[it]
+                            )
+                            .apply()
+                        println("FORCED ENCODING: ${prefValues[it]}")
+                        updateForcedEncoding(ctx)
+                    }
+                }
+
+                sourceDialog.apply_btt?.setOnClickListener {
                     var init = false
                     if (sourceIndex != startSource) {
                         init = true
@@ -477,15 +518,24 @@ class GeneratorPlayer : FullScreenPlayer() {
     ): SubtitleData? {
         val langCode = preferredAutoSelectSubtitles ?: return null
         val lang = SubtitleHelper.fromTwoLettersToLanguage(langCode) ?: return null
+        if (downloads) {
+            return subtitles.firstOrNull { sub ->
+                (sub.origin == SubtitleOrigin.DOWNLOADED_FILE && sub.name == context?.getString(
+                    R.string.default_subtitles
+                ))
+            }
+        }
 
-        if (settings)
+        sortSubs(subtitles).firstOrNull { sub ->
-            subtitles.firstOrNull { sub ->
             val t = sub.name.replace(Regex("[^A-Za-z]"), " ").trim()
-                t == lang || t.startsWith("$lang ")
+            (settings || (downloads && sub.origin == SubtitleOrigin.DOWNLOADED_FILE)) && t == lang || t.startsWith(
-                        || t == langCode
+                "$lang "
+            ) || t == langCode
         }?.let { sub ->
             return sub
         }
+
+        // post check in case both did not catch anything
         if (downloads) {
             return subtitles.firstOrNull { sub ->
                 (sub.origin == SubtitleOrigin.DOWNLOADED_FILE || sub.name == context?.getString(
@@ -493,10 +543,11 @@ class GeneratorPlayer : FullScreenPlayer() {
                 ))
             }
         }
+
         return null
     }
 
-    private fun autoSelectFromSettings() {
+    private fun autoSelectFromSettings(): Boolean {
         // auto select subtitle based of settings
         val langCode = preferredAutoSelectSubtitles
 
@@ -506,31 +557,36 @@ class GeneratorPlayer : FullScreenPlayer() {
                     if (setSubtitles(sub)) {
                         player.reloadPlayer(ctx)
                         player.handleEvent(CSPlayerEvent.Play)
+                        return true
                     }
                 }
             }
         }
+        return false
     }
 
-    private fun autoSelectFromDownloads() {
+    private fun autoSelectFromDownloads(): Boolean {
         if (player.getCurrentPreferredSubtitle() == null) {
             getAutoSelectSubtitle(currentSubs, settings = false, downloads = true)?.let { sub ->
                 context?.let { ctx ->
                     if (setSubtitles(sub)) {
                         player.reloadPlayer(ctx)
                         player.handleEvent(CSPlayerEvent.Play)
+                        return true
                     }
                 }
             }
         }
+        return false
     }
 
     private fun autoSelectSubtitles() {
         normalSafeApiCall {
-            autoSelectFromSettings()
+            if (!autoSelectFromSettings()) {
                 autoSelectFromDownloads()
             }
         }
+    }
 
     @SuppressLint("SetTextI18n")
     fun setTitle() {
@@ -567,7 +623,11 @@ class GeneratorPlayer : FullScreenPlayer() {
                             if (season == null)
                                 " - ${ctx.getString(R.string.episode)} $episode"
                             else
-                                " \"${ctx.getString(R.string.season_short)}${season}:${ctx.getString(R.string.episode_short)}${episode}\""
+                                " \"${ctx.getString(R.string.season_short)}${season}:${
+                                    ctx.getString(
+                                        R.string.episode_short
+                                    )
+                                }${episode}\""
                         else "") + if (subName.isNullOrBlank() || subName == headerName) "" else " - $subName"
             } else {
                 ""
@@ -649,6 +709,7 @@ class GeneratorPlayer : FullScreenPlayer() {
             val settingsManager = PreferenceManager.getDefaultSharedPreferences(ctx)
             titleRez = settingsManager.getInt(getString(R.string.prefer_limit_title_rez_key), 3)
             limitTitle = settingsManager.getInt(getString(R.string.prefer_limit_title_key), 0)
+            updateForcedEncoding(ctx)
         }
 
         unwrapBundle(savedInstanceState)

app/src/main/java/com/lagradost/cloudstream3/utils/ExtractorApi.kt

@@ -61,6 +61,7 @@ enum class Qualities(var value: Int) {
                 0 -> "Auto"
                 Unknown.value -> ""
                 P2160.value -> "4K"
+                null -> ""
                 else -> "${qual}p"
             }
         }

app/src/main/res/layout/player_select_source_and_subs.xml

@@ -62,20 +62,38 @@
                     android:orientation="horizontal"
                     tools:ignore="UseCompoundDrawables">
 
-                <TextView
+                <LinearLayout
-                        android:layout_width="match_parent"
+                        android:foreground="?attr/selectableItemBackgroundBorderless"
-                        android:layout_height="wrap_content"
+                        android:id="@+id/subtitles_click_settings"
                         android:layout_rowWeight="1"
-                        android:layout_marginTop="10dp"
-                        android:paddingStart="?android:attr/listPreferredItemPaddingStart"
                         android:paddingTop="10dp"
-                        android:paddingEnd="?android:attr/listPreferredItemPaddingEnd"
                         android:paddingBottom="10dp"
+                        android:layout_marginTop="10dp"
+                        android:orientation="horizontal"
+                        android:layout_width="match_parent"
+                        android:layout_height="wrap_content">
+
+                    <TextView
+                            android:layout_width="wrap_content"
+                            android:layout_height="wrap_content"
+                            android:paddingStart="?android:attr/listPreferredItemPaddingStart"
+                            android:paddingEnd="?android:attr/listPreferredItemPaddingEnd"
                             android:text="@string/pick_subtitle"
                             android:textColor="?attr/textColor"
                             android:textSize="20sp"
                             android:textStyle="bold" />
 
+                    <TextView
+                            android:textSize="15sp"
+                            android:id="@+id/subtitles_encoding_format"
+                            android:textColor="?attr/textColor"
+                            android:layout_gravity="center"
+                            android:gravity="center"
+                            tools:text="Thai (TIS 620-2533/ISO 8859-11)"
+                            android:layout_width="wrap_content"
+                            android:layout_height="wrap_content" />
+                </LinearLayout>
+
                 <ImageView
                         android:layout_width="25dp"
                         android:layout_height="wrap_content"
@@ -112,6 +130,7 @@
             android:layout_width="match_parent"
             android:layout_height="60dp">
 
+
         <com.google.android.material.button.MaterialButton
                 style="@style/WhiteButton"
                 android:layout_gravity="center_vertical|end"

app/src/main/res/layout/sort_bottom_footer_add_choice.xml

@@ -4,4 +4,5 @@
         style="@style/CheckLabel"
         android:id="@android:id/text1"
         tools:text="hello"
-        app:drawableStartCompat="@drawable/ic_baseline_add_24" />
+        app:drawableStartCompat="@drawable/ic_baseline_add_24"
+        app:drawableTint="?attr/textColor" />

app/src/main/res/values/array.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<resources>
+<resources xmlns:tools="http://schemas.android.com/tools">
     <array name="cast_mini_controller_control_buttons">
         <item>@id/cast_button_type_rewind_30_seconds</item>
         <item>@id/cast_button_type_play_pause_toggle</item>
@@ -269,4 +269,94 @@
         <item>Amoled</item>
         <item>Light</item>
     </string-array>
+
+    <!--https://github.com/videolan/vlc-android/blob/72ccfb93db027b49855760001d1a930fa657c5a8/application/resources/src/main/res/values/arrays.xml#L266-->
+    <string-array name="subtitles_encoding_list" tools:ignore="TypographyDashes">
+        <item>@string/automatic</item>
+        <item>Universal (UTF-8)</item>
+        <item>Universal (UTF-16)</item>
+        <item>Universal (big endian UTF-16)</item>
+        <item>Universal (little endian UTF-16)</item>
+        <item>Universal, Chinese (GB18030)</item>
+        <item>Western European (Latin-9)</item>
+        <item>Western European (Windows-1252)</item>
+        <item>Western European (IBM 00850)</item>
+        <item>Eastern European (Latin-2)</item>
+        <item>Eastern European (Windows-1250)</item>
+        <item>Esperanto (Latin-3)</item>
+        <item>Nordic (Latin-6)</item>
+        <item>Cyrillic (Windows-1251)</item>
+        <item>Russian (KOI8-R)</item>
+        <item>Ukrainian (KOI8-U)</item>
+        <item>Arabic (ISO 8859-6)</item>
+        <item>Arabic (Windows-1256)</item>
+        <item>Greek (ISO 8859-7)</item>
+        <item>Greek (Windows-1253)</item>
+        <item>Hebrew (ISO 8859-8)</item>
+        <item>Hebrew (Windows-1255)</item>
+        <item>Turkish (ISO 8859-9)</item>
+        <item>Turkish (Windows-1254)</item>
+        <item>Thai (TIS 620-2533/ISO 8859-11)</item>
+        <item>Thai (Windows-874)</item>
+        <item>Baltic (Latin-7)</item>
+        <item>Baltic (Windows-1257)</item>
+        <item>Celtic (Latin-8)</item>
+        <item>South-Eastern European (Latin-10)</item>
+        <item>Simplified Chinese (ISO-2022-CN-EXT)</item>
+        <item>Simplified Chinese Unix (EUC-CN)</item>
+        <item>Japanese (7-bits JIS/ISO-2022-JP-2)</item>
+        <item>Japanese Unix (EUC-JP)</item>
+        <item>Japanese (Shift JIS)</item>
+        <item>Korean (EUC-KR/CP949)</item>
+        <item>Korean (ISO-2022-KR)</item>
+        <item>Traditional Chinese (Big5)</item>
+        <item>Traditional Chinese Unix (EUC-TW)</item>
+        <item>Hong-Kong Supplementary (HKSCS)</item>
+        <item>Vietnamese (VISCII)</item>
+        <item>Vietnamese (Windows-1258)</item>
+    </string-array>
+    <string-array name="subtitles_encoding_values" translatable="false" tools:ignore="TypographyDashes">
+        <item></item>
+        <item>UTF-8</item>
+        <item>UTF-16</item>
+        <item>UTF-16BE</item>
+        <item>UTF-16LE</item>
+        <item>GB18030</item>
+        <item>ISO-8859-15</item>
+        <item>Windows-1252</item>
+        <item>IBM850</item>
+        <item>ISO-8859-2</item>
+        <item>Windows-1250</item>
+        <item>ISO-8859-3</item>
+        <item>ISO-8859-10</item>
+        <item>Windows-1251</item>
+        <item>KOI8-R</item>
+        <item>KOI8-U</item>
+        <item>ISO-8859-6</item>
+        <item>Windows-1256</item>
+        <item>ISO-8859-7</item>
+        <item>Windows-1253</item>
+        <item>ISO-8859-8</item>
+        <item>Windows-1255</item>
+        <item>ISO-8859-9</item>
+        <item>Windows-1254</item>
+        <item>ISO-8859-11</item>
+        <item>Windows-874</item>
+        <item>ISO-8859-13</item>
+        <item>Windows-1257</item>
+        <item>ISO-8859-14</item>
+        <item>ISO-8859-16</item>
+        <item>ISO-2022-CN-EXT</item>
+        <item>EUC-CN</item>
+        <item>ISO-2022-JP-2</item>
+        <item>EUC-JP</item>
+        <item>Shift_JIS</item>
+        <item>CP949</item>
+        <item>ISO-2022-KR</item>
+        <item>Big5</item>
+        <item>ISO-2022-TW</item>
+        <item>Big5-HKSCS</item>
+        <item>VISCII</item>
+        <item>Windows-1258</item>
+    </string-array>
 </resources>

app/src/main/res/values/strings.xml

@@ -51,6 +51,7 @@
     <string name="show_logcat_key" translatable="false">show_logcat_key</string>
     <string name="bottom_title_key" translatable="false">bottom_title_key</string>
     <string name="poster_ui_key" translatable="false">poster_ui_key</string>
+    <string name="subtitles_encoding_key" translatable="false">subtitles_encoding_key</string>
 
     <!-- FORMAT MIGHT TRANSLATE, WILL CAUSE CRASH IF APPLIED WRONG -->
     <string name="extra_info_format" translatable="false" formatted="true">%d %s | %sMB</string>
@@ -418,6 +419,7 @@
     <string name="provider_lang_settings">Provider languages</string>
     <string name="app_layout">App Layout</string>
     <string name="preferred_media_settings">Preferred media</string>
+    <string name="subtitles_encoding">Subtitle encoding</string>
     <string name="category_preferred_media_and_lang">Preferred media and language</string>
     <string name="category_ui">User interface</string>