recloudstream/cloudstream
2
2
Fork 1
mirror of https://github.com/recloudstream/cloudstream.git synced 2024-08-15 01:53:11 +00:00
Code Issues Releases Wiki Activity

feat: add remote sync capability - fix sync/restore logic

Browse source
This commit is contained in:
Martin Filo 2023-05-08 19:35:24 +02:00
parent de3019b1d8
commit dee51d8695
5 changed files with 69 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

55
app/src/main/java/com/lagradost/cloudstream3/syncproviders/BackupAPI.kt
View file

@ -52,7 +52,7 @@ interface BackupAPI<LOGIN_DATA> {
private val ioScope = CoroutineScope(Dispatchers.IO) private val ioScope = CoroutineScope(Dispatchers.IO)
fun SharedPreferences.logHistoryChanged(path: String, source: BackupUtils.RestoreSource) { fun SharedPreferences.logHistoryChanged(path: String, source: BackupUtils.RestoreSource) {
edit().putLong("$SYNC_HISTORY_PREFIX${source.prefix}$path", System.currentTimeMillis()) edit().putLong("${source.syncPrefix}$path", System.currentTimeMillis())
.apply() .apply()
} }
} }
@ -132,7 +132,7 @@ interface BackupAPI<LOGIN_DATA> {
val executionTime = measureTimeMillis { val executionTime = measureTimeMillis {
result = try { result = try {
JSONCompare.compareJSON(old, new, JSONCompareMode.LENIENT) JSONCompare.compareJSON(old, new, JSONCompareMode.NON_EXTENSIBLE)
} catch (e: Exception) { } catch (e: Exception) {
null null
} }
@ -157,7 +157,7 @@ interface BackupAPI<LOGIN_DATA> {
}.keys }.keys
val onlyLocalKeys = currentSync.keys.filter { !newSync.containsKey(it) } val onlyLocalKeys = currentSync.keys.filter { !newSync.containsKey(it) }
val missingKeys = getMissingKeys(currentData, newData) val missingKeys = getAllMissingKeys(currentData, newData)
return (missingKeys + onlyLocalKeys + changedKeys).toSet() return (missingKeys + onlyLocalKeys + changedKeys).toSet()
} }
@ -166,26 +166,41 @@ interface BackupAPI<LOGIN_DATA> {
data.syncMeta._Long.orEmpty().filter { it.key.startsWith(SYNC_HISTORY_PREFIX) } data.syncMeta._Long.orEmpty().filter { it.key.startsWith(SYNC_HISTORY_PREFIX) }
// 🤮 private fun getAllMissingKeys(
private fun getMissingKeys(
old: BackupUtils.BackupFile, old: BackupUtils.BackupFile,
new: BackupUtils.BackupFile new: BackupUtils.BackupFile
): List<String> = mutableListOf( ): List<String> = BackupUtils.RestoreSource
*getMissing(old.settings._Bool, new.settings._Bool), .values()
*getMissing(old.settings._Long, new.settings._Long), .filter { it != BackupUtils.RestoreSource.SYNC }
*getMissing(old.settings._Float, new.settings._Float), .fold(mutableListOf()) { acc, source ->
*getMissing(old.settings._Int, new.settings._Int), acc.addAll(getMissingKeysPrefixed(source, old, new))
*getMissing(old.settings._String, new.settings._String), acc
*getMissing(old.settings._StringSet, new.settings._StringSet), }
*getMissing(old.datastore._Bool, new.datastore._Bool),
*getMissing(old.datastore._Long, new.datastore._Long), private fun getMissingKeysPrefixed(
*getMissing(old.datastore._Float, new.datastore._Float), restoreSource: BackupUtils.RestoreSource,
*getMissing(old.datastore._Int, new.datastore._Int), old: BackupUtils.BackupFile,
*getMissing(old.datastore._String, new.datastore._String), new: BackupUtils.BackupFile
*getMissing(old.datastore._StringSet, new.datastore._StringSet), ): List<String> {
) val oldSource = old.getData(restoreSource)
val newSource = new.getData(restoreSource)
val prefixToMatch = restoreSource.syncPrefix
return listOf(
*getMissing(oldSource._Bool, newSource._Bool),
*getMissing(oldSource._Long, newSource._Long),
*getMissing(oldSource._Float, newSource._Float),
*getMissing(oldSource._Int, newSource._Int),
*getMissing(oldSource._String, newSource._String),
*getMissing(oldSource._StringSet, newSource._StringSet),
).map {
prefixToMatch + it
}
}
private fun getMissing(old: Map<String, *>?, new: Map<String, *>?): Array<String> = private fun getMissing(old: Map<String, *>?, new: Map<String, *>?): Array<String> =
new.orEmpty().keys.subtract(old.orEmpty().keys).toTypedArray() (new.orEmpty().keys - old.orEmpty().keys)
.toTypedArray()
} }

3
app/src/main/java/com/lagradost/cloudstream3/syncproviders/providers/GoogleDriveApi.kt
View file

@ -39,6 +39,8 @@ import java.util.Date
* *
* | State | Priority | Description * | State | Priority | Description
* |---------:|:--------:|--------------------------------------------------------------------- * |---------:|:--------:|---------------------------------------------------------------------
* | Progress | 1 | When scheduler has queued upload job (but is not working in backupApi
* | | | yet) we should postpone download and prioritize upload
* | Waiting | 2 | Add button to manually trigger sync * | Waiting | 2 | Add button to manually trigger sync
* | Waiting | 3 | Move "https://chiff.github.io/cloudstream-sync/google-drive" * | Waiting | 3 | Move "https://chiff.github.io/cloudstream-sync/google-drive"
* | Waiting | 5 | Choose what should be synced and recheck `invalidKeys` in createBackupScheduler * | Waiting | 5 | Choose what should be synced and recheck `invalidKeys` in createBackupScheduler
@ -50,6 +52,7 @@ import java.util.Date
* | | | dont update sync meta if not needed * | | | dont update sync meta if not needed
* | Solved | 4 | Implement backup before user quits application * | Solved | 4 | Implement backup before user quits application
* | Solved | 1 | Do not write sync meta when user is not syncing data * | Solved | 1 | Do not write sync meta when user is not syncing data
* | Solved | 1 | Fix sync/restore bugs
*/ */
class GoogleDriveApi(index: Int) : class GoogleDriveApi(index: Int) :
InAppOAuth2APIManager(index), InAppOAuth2APIManager(index),

31
app/src/main/java/com/lagradost/cloudstream3/utils/BackupUtils.kt
View file

@ -55,12 +55,13 @@ object BackupUtils {
DATA, SETTINGS, SYNC; DATA, SETTINGS, SYNC;
val prefix = "$name/" val prefix = "$name/"
val syncPrefix = "${BackupAPI.SYNC_HISTORY_PREFIX}$prefix"
} }
/** /**
* No sensitive or breaking data in the backup * No sensitive or breaking data in the backup
* */ * */
private val nonTransferableKeys = listOf( val nonTransferableKeys = listOf(
// When sharing backup we do not want to transfer what is essentially the password // When sharing backup we do not want to transfer what is essentially the password
ANILIST_TOKEN_KEY, ANILIST_TOKEN_KEY,
ANILIST_CACHED_LIST, ANILIST_CACHED_LIST,
@ -132,7 +133,7 @@ object BackupUtils {
return successfulRestore return successfulRestore
} }
private fun getData(source: RestoreSource) = when (source) { fun getData(source: RestoreSource) = when (source) {
RestoreSource.SYNC -> syncMeta RestoreSource.SYNC -> syncMeta
RestoreSource.DATA -> datastore RestoreSource.DATA -> datastore
RestoreSource.SETTINGS -> settings RestoreSource.SETTINGS -> settings
@ -194,6 +195,8 @@ object BackupUtils {
restoreKeys: Set<String>? = null, restoreKeys: Set<String>? = null,
vararg restoreSources: RestoreSource vararg restoreSources: RestoreSource
) { ) {
Log.d(BackupAPI.LOG_KEY, "will restore keys = $restoreKeys")
for (restoreSource in restoreSources) { for (restoreSource in restoreSources) {
val restoreData = RestoreMapData() val restoreData = RestoreMapData()
@ -346,12 +349,18 @@ object BackupUtils {
val successfulRestore = mutableSetOf<String>() val successfulRestore = mutableSetOf<String>()
if (!restoreKeys.isNullOrEmpty()) { if (!restoreKeys.isNullOrEmpty()) {
val prefixToMatch = "${BackupAPI.SYNC_HISTORY_PREFIX}${restoreSource.prefix}" var prefixToMatch = restoreSource.syncPrefix
var prefixToRemove = prefixToMatch
if (restoreSource == RestoreSource.SYNC) {
prefixToMatch = BackupAPI.SYNC_HISTORY_PREFIX
prefixToRemove = ""
}
val restore = restoreKeys.filter { val restore = restoreKeys.filter {
it.startsWith(prefixToMatch) it.startsWith(prefixToMatch)
}.map { }.map {
it.removePrefix(prefixToMatch) it.removePrefix(prefixToRemove)
} }
restoreOnlyThese.addAll(restore) restoreOnlyThese.addAll(restore)
@ -359,19 +368,19 @@ object BackupUtils {
map?.filter { map?.filter {
var isTransferable = it.key.isTransferable() var isTransferable = it.key.withoutPrefix(restoreSource).isTransferable()
if (isTransferable && restoreOnlyThese.isNotEmpty()) { if (isTransferable && restoreOnlyThese.isNotEmpty()) {
isTransferable = restoreOnlyThese.contains(it.key) isTransferable = restoreOnlyThese.contains(it.key.withoutPrefix(restoreSource))
} }
if (isTransferable) { if (isTransferable) {
successfulRestore.add(it.key) successfulRestore.add(it.key.withoutPrefix(restoreSource))
} }
isTransferable isTransferable
}?.forEach { }?.forEach {
setKeyRaw(it.key, it.value, restoreSource) setKeyRaw(it.key.withoutPrefix(restoreSource), it.value, restoreSource)
} }
return RestoreMapData( return RestoreMapData(
@ -379,4 +388,8 @@ object BackupUtils {
successfulRestore successfulRestore
) )
} }
} }
private fun String.withoutPrefix(restoreSource: BackupUtils.RestoreSource) =
// will not remove sync prefix because it wont match (its not a bug its a feature ¯\_(ツ)_/¯ )
removePrefix(restoreSource.prefix)

4
app/src/main/java/com/lagradost/cloudstream3/utils/DataStore.kt
View file

@ -113,7 +113,7 @@ object DataStore {
backupScheduler.work( backupScheduler.work(
BackupAPI.PreferencesSchedulerData( BackupAPI.PreferencesSchedulerData(
prefs, getSyncPrefs(),
path, path,
oldValueExists, oldValueExists,
false, false,
@ -146,7 +146,7 @@ object DataStore {
backupScheduler.work( backupScheduler.work(
BackupAPI.PreferencesSchedulerData( BackupAPI.PreferencesSchedulerData(
prefs, getSyncPrefs(),
path, path,
oldValue, oldValue,
newValue, newValue,

11
app/src/main/java/com/lagradost/cloudstream3/utils/Scheduler.kt
View file

@ -10,6 +10,7 @@ import com.lagradost.cloudstream3.syncproviders.BackupAPI.Companion.logHistoryCh
import com.lagradost.cloudstream3.ui.home.HOME_BOOKMARK_VALUE_LIST import com.lagradost.cloudstream3.ui.home.HOME_BOOKMARK_VALUE_LIST
import com.lagradost.cloudstream3.ui.player.PLAYBACK_SPEED_KEY import com.lagradost.cloudstream3.ui.player.PLAYBACK_SPEED_KEY
import com.lagradost.cloudstream3.ui.player.RESIZE_MODE_KEY import com.lagradost.cloudstream3.ui.player.RESIZE_MODE_KEY
import com.lagradost.cloudstream3.utils.BackupUtils.nonTransferableKeys
class Scheduler<INPUT>( class Scheduler<INPUT>(
private val throttleTimeMs: Long, private val throttleTimeMs: Long,
@ -19,8 +20,11 @@ class Scheduler<INPUT>(
companion object { companion object {
var SCHEDULER_ID = 1 var SCHEDULER_ID = 1
private val invalidSchedulerKeys = listOf( // these will not run upload scheduler, however only `nonTransferableKeys` are not stored
private val invalidUploadTriggerKeys = listOf(
*nonTransferableKeys.toTypedArray(),
VideoDownloadManager.KEY_DOWNLOAD_INFO, VideoDownloadManager.KEY_DOWNLOAD_INFO,
DOWNLOAD_HEADER_CACHE,
PLAYBACK_SPEED_KEY, PLAYBACK_SPEED_KEY,
HOME_BOOKMARK_VALUE_LIST, HOME_BOOKMARK_VALUE_LIST,
RESIZE_MODE_KEY RESIZE_MODE_KEY
@ -33,8 +37,6 @@ class Scheduler<INPUT>(
throw IllegalStateException() throw IllegalStateException()
} }
input.syncPrefs.logHistoryChanged(input.storeKey, input.source)
AccountManager.BackupApis.forEach { AccountManager.BackupApis.forEach {
it.addToQueue( it.addToQueue(
input.storeKey, input.storeKey,
@ -52,7 +54,7 @@ class Scheduler<INPUT>(
return@Scheduler false return@Scheduler false
} }
val hasInvalidKey = invalidSchedulerKeys.contains(input.storeKey) val hasInvalidKey = invalidUploadTriggerKeys.contains(input.storeKey)
if (hasInvalidKey) { if (hasInvalidKey) {
return@Scheduler false return@Scheduler false
} }
@ -62,6 +64,7 @@ class Scheduler<INPUT>(
return@Scheduler false return@Scheduler false
} }
input.syncPrefs.logHistoryChanged(input.storeKey, input.source)
return@Scheduler true return@Scheduler true
} }
) )