feat: add remote sync capability - fix sync/restore logic

app/src/main/java/com/lagradost/cloudstream3/syncproviders/BackupAPI.kt

@@ -52,7 +52,7 @@ interface BackupAPI<LOGIN_DATA> {
         private val ioScope = CoroutineScope(Dispatchers.IO)
 
         fun SharedPreferences.logHistoryChanged(path: String, source: BackupUtils.RestoreSource) {
-            edit().putLong("$SYNC_HISTORY_PREFIX${source.prefix}$path", System.currentTimeMillis())
+            edit().putLong("${source.syncPrefix}$path", System.currentTimeMillis())
                 .apply()
         }
     }
@@ -132,7 +132,7 @@ interface BackupAPI<LOGIN_DATA> {
 
         val executionTime = measureTimeMillis {
             result = try {
-                JSONCompare.compareJSON(old, new, JSONCompareMode.LENIENT)
+                JSONCompare.compareJSON(old, new, JSONCompareMode.NON_EXTENSIBLE)
             } catch (e: Exception) {
                 null
             }
@@ -157,7 +157,7 @@ interface BackupAPI<LOGIN_DATA> {
         }.keys
 
         val onlyLocalKeys = currentSync.keys.filter { !newSync.containsKey(it) }
-        val missingKeys = getMissingKeys(currentData, newData)
+        val missingKeys = getAllMissingKeys(currentData, newData)
 
         return (missingKeys + onlyLocalKeys + changedKeys).toSet()
     }
@@ -166,26 +166,41 @@ interface BackupAPI<LOGIN_DATA> {
         data.syncMeta._Long.orEmpty().filter { it.key.startsWith(SYNC_HISTORY_PREFIX) }
 
 
-    // 🤮
-    private fun getMissingKeys(
+    private fun getAllMissingKeys(
         old: BackupUtils.BackupFile,
         new: BackupUtils.BackupFile
-    ): List<String> = mutableListOf(
-        *getMissing(old.settings._Bool, new.settings._Bool),
-        *getMissing(old.settings._Long, new.settings._Long),
-        *getMissing(old.settings._Float, new.settings._Float),
-        *getMissing(old.settings._Int, new.settings._Int),
-        *getMissing(old.settings._String, new.settings._String),
-        *getMissing(old.settings._StringSet, new.settings._StringSet),
-        *getMissing(old.datastore._Bool, new.datastore._Bool),
-        *getMissing(old.datastore._Long, new.datastore._Long),
-        *getMissing(old.datastore._Float, new.datastore._Float),
-        *getMissing(old.datastore._Int, new.datastore._Int),
-        *getMissing(old.datastore._String, new.datastore._String),
-        *getMissing(old.datastore._StringSet, new.datastore._StringSet),
-    )
+    ): List<String> = BackupUtils.RestoreSource
+        .values()
+        .filter { it != BackupUtils.RestoreSource.SYNC }
+        .fold(mutableListOf()) { acc, source ->
+            acc.addAll(getMissingKeysPrefixed(source, old, new))
+            acc
+        }
+
+    private fun getMissingKeysPrefixed(
+        restoreSource: BackupUtils.RestoreSource,
+        old: BackupUtils.BackupFile,
+        new: BackupUtils.BackupFile
+    ): List<String> {
+        val oldSource = old.getData(restoreSource)
+        val newSource = new.getData(restoreSource)
+        val prefixToMatch = restoreSource.syncPrefix
+
+        return listOf(
+            *getMissing(oldSource._Bool, newSource._Bool),
+            *getMissing(oldSource._Long, newSource._Long),
+            *getMissing(oldSource._Float, newSource._Float),
+            *getMissing(oldSource._Int, newSource._Int),
+            *getMissing(oldSource._String, newSource._String),
+            *getMissing(oldSource._StringSet, newSource._StringSet),
+        ).map {
+            prefixToMatch + it
+        }
+    }
+
 
     private fun getMissing(old: Map<String, *>?, new: Map<String, *>?): Array<String> =
-        new.orEmpty().keys.subtract(old.orEmpty().keys).toTypedArray()
+        (new.orEmpty().keys - old.orEmpty().keys)
+            .toTypedArray()
 
 }

app/src/main/java/com/lagradost/cloudstream3/syncproviders/providers/GoogleDriveApi.kt

@@ -39,6 +39,8 @@ import java.util.Date
  *
  * | State    | Priority | Description
  * |---------:|:--------:|---------------------------------------------------------------------
+ * | Progress | 1        | When scheduler has queued upload job (but is not working in backupApi
+ * |          |          | yet) we should postpone download and prioritize upload
  * | Waiting  | 2        | Add button to manually trigger sync
  * | Waiting  | 3        | Move "https://chiff.github.io/cloudstream-sync/google-drive"
  * | Waiting  | 5        | Choose what should be synced and recheck `invalidKeys` in createBackupScheduler
@@ -50,6 +52,7 @@ import java.util.Date
  * |          |          | dont update sync meta if not needed
  * | Solved   | 4        | Implement backup before user quits application
  * | Solved   | 1        | Do not write sync meta when user is not syncing data
+ * | Solved   | 1        | Fix sync/restore bugs
  */
 class GoogleDriveApi(index: Int) :
     InAppOAuth2APIManager(index),

app/src/main/java/com/lagradost/cloudstream3/utils/BackupUtils.kt

@@ -55,12 +55,13 @@ object BackupUtils {
         DATA, SETTINGS, SYNC;
 
         val prefix = "$name/"
+        val syncPrefix = "${BackupAPI.SYNC_HISTORY_PREFIX}$prefix"
     }
 
     /**
      * No sensitive or breaking data in the backup
      * */
-    private val nonTransferableKeys = listOf(
+    val nonTransferableKeys = listOf(
         // When sharing backup we do not want to transfer what is essentially the password
         ANILIST_TOKEN_KEY,
         ANILIST_CACHED_LIST,
@@ -132,7 +133,7 @@ object BackupUtils {
             return successfulRestore
         }
 
-        private fun getData(source: RestoreSource) = when (source) {
+        fun getData(source: RestoreSource) = when (source) {
             RestoreSource.SYNC -> syncMeta
             RestoreSource.DATA -> datastore
             RestoreSource.SETTINGS -> settings
@@ -194,6 +195,8 @@ object BackupUtils {
         restoreKeys: Set<String>? = null,
         vararg restoreSources: RestoreSource
     ) {
+        Log.d(BackupAPI.LOG_KEY, "will restore keys = $restoreKeys")
+
         for (restoreSource in restoreSources) {
             val restoreData = RestoreMapData()
 
@@ -346,12 +349,18 @@ object BackupUtils {
         val successfulRestore = mutableSetOf<String>()
 
         if (!restoreKeys.isNullOrEmpty()) {
-            val prefixToMatch = "${BackupAPI.SYNC_HISTORY_PREFIX}${restoreSource.prefix}"
+            var prefixToMatch = restoreSource.syncPrefix
+            var prefixToRemove = prefixToMatch
+
+            if (restoreSource == RestoreSource.SYNC) {
+                prefixToMatch = BackupAPI.SYNC_HISTORY_PREFIX
+                prefixToRemove = ""
+            }
 
             val restore = restoreKeys.filter {
                 it.startsWith(prefixToMatch)
             }.map {
-                it.removePrefix(prefixToMatch)
+                it.removePrefix(prefixToRemove)
             }
 
             restoreOnlyThese.addAll(restore)
@@ -359,19 +368,19 @@ object BackupUtils {
 
 
         map?.filter {
-            var isTransferable = it.key.isTransferable()
+            var isTransferable = it.key.withoutPrefix(restoreSource).isTransferable()
 
             if (isTransferable && restoreOnlyThese.isNotEmpty()) {
-                isTransferable = restoreOnlyThese.contains(it.key)
+                isTransferable = restoreOnlyThese.contains(it.key.withoutPrefix(restoreSource))
             }
 
             if (isTransferable) {
-                successfulRestore.add(it.key)
+                successfulRestore.add(it.key.withoutPrefix(restoreSource))
             }
 
             isTransferable
         }?.forEach {
-            setKeyRaw(it.key, it.value, restoreSource)
+            setKeyRaw(it.key.withoutPrefix(restoreSource), it.value, restoreSource)
         }
 
         return RestoreMapData(
@@ -379,4 +388,8 @@ object BackupUtils {
             successfulRestore
         )
     }
-}
+}
+
+private fun String.withoutPrefix(restoreSource: BackupUtils.RestoreSource) =
+    // will not remove sync prefix because it wont match (its not a bug its a feature ¯\_(ツ)_/¯ )
+    removePrefix(restoreSource.prefix)

app/src/main/java/com/lagradost/cloudstream3/utils/DataStore.kt

@@ -113,7 +113,7 @@ object DataStore {
 
                 backupScheduler.work(
                     BackupAPI.PreferencesSchedulerData(
-                        prefs,
+                        getSyncPrefs(),
                         path,
                         oldValueExists,
                         false,
@@ -146,7 +146,7 @@ object DataStore {
 
             backupScheduler.work(
                 BackupAPI.PreferencesSchedulerData(
-                    prefs,
+                    getSyncPrefs(),
                     path,
                     oldValue,
                     newValue,

app/src/main/java/com/lagradost/cloudstream3/utils/Scheduler.kt

@@ -10,6 +10,7 @@ import com.lagradost.cloudstream3.syncproviders.BackupAPI.Companion.logHistoryCh
 import com.lagradost.cloudstream3.ui.home.HOME_BOOKMARK_VALUE_LIST
 import com.lagradost.cloudstream3.ui.player.PLAYBACK_SPEED_KEY
 import com.lagradost.cloudstream3.ui.player.RESIZE_MODE_KEY
+import com.lagradost.cloudstream3.utils.BackupUtils.nonTransferableKeys
 
 class Scheduler<INPUT>(
     private val throttleTimeMs: Long,
@@ -19,8 +20,11 @@ class Scheduler<INPUT>(
     companion object {
         var SCHEDULER_ID = 1
 
-        private val invalidSchedulerKeys = listOf(
+        // these will not run upload scheduler, however only `nonTransferableKeys` are not stored
+        private val invalidUploadTriggerKeys = listOf(
+            *nonTransferableKeys.toTypedArray(),
             VideoDownloadManager.KEY_DOWNLOAD_INFO,
+            DOWNLOAD_HEADER_CACHE,
             PLAYBACK_SPEED_KEY,
             HOME_BOOKMARK_VALUE_LIST,
             RESIZE_MODE_KEY
@@ -33,8 +37,6 @@ class Scheduler<INPUT>(
                     throw IllegalStateException()
                 }
 
-                input.syncPrefs.logHistoryChanged(input.storeKey, input.source)
-
                 AccountManager.BackupApis.forEach {
                     it.addToQueue(
                         input.storeKey,
@@ -52,7 +54,7 @@ class Scheduler<INPUT>(
                     return@Scheduler false
                 }
 
-                val hasInvalidKey = invalidSchedulerKeys.contains(input.storeKey)
+                val hasInvalidKey = invalidUploadTriggerKeys.contains(input.storeKey)
                 if (hasInvalidKey) {
                     return@Scheduler false
                 }
@@ -62,6 +64,7 @@ class Scheduler<INPUT>(
                     return@Scheduler false
                 }
 
+                input.syncPrefs.logHistoryChanged(input.storeKey, input.source)
                 return@Scheduler true
             }
         )