MS-1538 Sync worker error propagation fix

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/EventSyncStateProcessor.kt

@@ -19,6 +19,7 @@ import com.simprints.infra.eventsync.sync.common.didFailBecauseReloginRequired
 import com.simprints.infra.eventsync.sync.common.didFailBecauseTooManyRequests
 import com.simprints.infra.eventsync.sync.common.filterByTags
 import com.simprints.infra.eventsync.sync.common.getEstimatedOutageTime
+import com.simprints.infra.eventsync.sync.common.hasAnyFailureReason
 import com.simprints.infra.eventsync.sync.common.sortByScheduledTime
 import com.simprints.infra.eventsync.sync.down.workers.extractDownSyncMaxCount
 import com.simprints.infra.eventsync.sync.down.workers.extractDownSyncProgress
@@ -134,7 +135,7 @@ class EventSyncStateProcessor @Inject constructor(
         }
 
     private fun WorkInfo.toEventSyncWorkerState(): EventSyncWorkerState = fromWorkInfo(
-        state = state,
+        state = if (hasAnyFailureReason()) WorkInfo.State.FAILED else state,
         failedBecauseReloginRequired = didFailBecauseReloginRequired(),
         failedBecauseCloudIntegration = didFailBecauseCloudIntegration(),
         failedBecauseBackendMaintenance = didFailBecauseBackendMaintenance(),

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/common/WorkInfo.ext.kt

@@ -25,3 +25,10 @@ internal fun WorkInfo.didFailBecauseCommCarePermissionMissing(): Boolean =
     this.outputData.getBoolean(OUTPUT_FAILED_BECAUSE_COMMCARE_PERMISSION_MISSING, false)
 
 internal fun WorkInfo.getEstimatedOutageTime(): Long = this.outputData.getLong(OUTPUT_ESTIMATED_MAINTENANCE_TIME, 0L)
+
+internal fun WorkInfo.hasAnyFailureReason(): Boolean =
+    didFailBecauseReloginRequired() ||
+        didFailBecauseCloudIntegration() ||
+        didFailBecauseBackendMaintenance() ||
+        didFailBecauseTooManyRequests() ||
+        didFailBecauseCommCarePermissionMissing()

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/down/tasks/BaseEventDownSyncTask.kt

@@ -71,6 +71,7 @@ internal abstract class BaseEventDownSyncTask(
         val requestId = UUID.randomUUID().toString()
         var result: EventFetchResult? = null
         var errorType: String? = null
+        var errorToRethrow: Throwable? = null
 
         try {
             result = fetchEvents(operation, scope, requestId)
@@ -98,18 +99,19 @@ internal abstract class BaseEventDownSyncTask(
             lastOperation = lastOperation.copy(state = COMPLETE, lastSyncTime = timeHelper.now().ms)
             emitProgress(lastOperation, count, result.totalCount)
         } catch (t: Throwable) {
-            if (shouldRethrowError(t)) {
-                throw t
-            }
-
-            Simber.i("Down sync error", t, tag = SYNC)
             errorType = t.javaClass.simpleName
 
             lastOperation = processBatchedEvents(operation, batchOfEventsToProcess, lastOperation, project)
             emitProgress(lastOperation, count, count)
 
             lastOperation = lastOperation.copy(state = FAILED, lastSyncTime = timeHelper.now().ms)
             emitProgress(lastOperation, count, count)
+
+            if (shouldRethrowError(t)) {
+                errorToRethrow = t
+            } else {
+                Simber.i("Down sync error", t, tag = SYNC)
+            }
         }
 
         if (count > 0 || errorType != null) {
@@ -136,6 +138,8 @@ internal abstract class BaseEventDownSyncTask(
                 ),
             )
         }
+
+        errorToRethrow?.let { throw it }
     }
 
     private suspend fun FlowCollector<EventDownSyncProgress>.emitProgress(

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/down/tasks/SimprintsEventDownSyncTask.kt

@@ -6,9 +6,12 @@ import com.simprints.infra.config.store.ConfigRepository
 import com.simprints.infra.enrolment.records.repository.EnrolmentRecordRepository
 import com.simprints.infra.events.EventRepository
 import com.simprints.infra.eventsync.event.remote.EventRemoteDataSource
+import com.simprints.infra.eventsync.event.remote.exceptions.TooManyRequestsException
 import com.simprints.infra.eventsync.status.down.EventDownSyncScopeRepository
 import com.simprints.infra.eventsync.status.down.domain.EventDownSyncOperation
 import com.simprints.infra.eventsync.sync.common.EnrolmentRecordFactory
+import com.simprints.infra.network.exceptions.BackendMaintenanceException
+import com.simprints.infra.network.exceptions.SyncCloudIntegrationException
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.catch
 import kotlinx.coroutines.flow.consumeAsFlow
@@ -59,6 +62,9 @@ internal class SimprintsEventDownSyncTask @Inject constructor(
 
     override fun shouldRethrowError(throwable: Throwable): Boolean {
         // Return true to re-throw specific exceptions that should not be handled by the base class
-        return throwable is RemoteDbNotSignedInException
+        return throwable is RemoteDbNotSignedInException ||
+            throwable is BackendMaintenanceException ||
+            throwable is SyncCloudIntegrationException ||
+            throwable is TooManyRequestsException
     }
 }

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/down/workers/BaseEventDownSyncDownloaderWorker.kt

@@ -5,6 +5,7 @@ import androidx.work.WorkInfo
 import androidx.work.WorkerParameters
 import androidx.work.workDataOf
 import com.simprints.core.workers.SimCoroutineWorker
+import com.simprints.infra.authstore.exceptions.RemoteDbNotSignedInException
 import com.simprints.infra.config.store.ConfigRepository
 import com.simprints.infra.enrolment.records.repository.local.migration.RealmToRoomMigrationFlagsStore
 import com.simprints.infra.events.EventRepository
@@ -46,7 +47,11 @@ internal abstract class BaseEventDownSyncDownloaderWorker(
 
     abstract fun createDownSyncTask(): BaseEventDownSyncTask
 
-    abstract fun handleSyncException(t: Throwable): Result
+    abstract fun handleSyncException(
+        t: Throwable,
+        count: Int,
+        max: Int?,
+    ): Result
 
     override suspend fun doWork(): Result {
         // Check if the migration is in progress before starting the sync
@@ -66,14 +71,14 @@ internal abstract class BaseEventDownSyncDownloaderWorker(
     protected open suspend fun performDownSync(): Result = withContext(dispatcher) {
         showProgressNotification()
         crashlyticsLog("Started")
+        val workerId = id.toString()
+        var count = syncCache.readProgress(workerId)
+        var max: Int? = syncCache.readMax(workerId)
         try {
-            val workerId = id.toString()
-            var count = syncCache.readProgress(workerId)
-            var max: Int? = syncCache.readMax(workerId)
             val project = configRepository.getProject()
 
             if (project == null) {
-                fail(IllegalStateException("User is not signed in"))
+                throw RemoteDbNotSignedInException()
             } else {
                 createDownSyncTask().downSync(this, getDownSyncOperation(), getEventScope(), project).collect {
                     count = it.progress
@@ -93,7 +98,7 @@ internal abstract class BaseEventDownSyncDownloaderWorker(
                 )
             }
         } catch (t: Throwable) {
-            handleSyncException(t)
+            handleSyncException(t, count, max)
         }
     }
 

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/down/workers/CommCareEventSyncDownloaderWorker.kt

@@ -2,17 +2,20 @@
 
 import android.content.Context
 import androidx.hilt.work.HiltWorker
+import androidx.work.Data
 import androidx.work.WorkerParameters
-import androidx.work.workDataOf
 import com.simprints.core.DispatcherBG
+import com.simprints.infra.authstore.exceptions.RemoteDbNotSignedInException
 import com.simprints.infra.config.store.ConfigRepository
 import com.simprints.infra.enrolment.records.repository.local.migration.RealmToRoomMigrationFlagsStore
 import com.simprints.infra.events.EventRepository
 import com.simprints.infra.eventsync.status.down.EventDownSyncScopeRepository
 import com.simprints.infra.eventsync.sync.common.EventSyncCache
+import com.simprints.infra.eventsync.sync.common.OUTPUT_FAILED_BECAUSE_RELOGIN_REQUIRED
 import com.simprints.infra.eventsync.sync.common.OUTPUT_FAILED_BECAUSE_COMMCARE_PERMISSION_MISSING
 import com.simprints.infra.eventsync.sync.down.tasks.BaseEventDownSyncTask
 import com.simprints.infra.eventsync.sync.down.tasks.CommCareEventSyncTask
+import com.simprints.infra.logging.Simber
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedInject
 import kotlinx.coroutines.CoroutineDispatcher
@@ -40,9 +43,35 @@
     ) {
     override fun createDownSyncTask(): BaseEventDownSyncTask = commCareSyncTask
 
-    override fun handleSyncException(t: Throwable) = when (t) {
-        is IllegalArgumentException -> fail(t, t.message)
-        is SecurityException -> fail(t, t.message, workDataOf(OUTPUT_FAILED_BECAUSE_COMMCARE_PERMISSION_MISSING to true))
-        else -> retry(t)
+    override fun handleSyncException(
+        t: Throwable,
+        count: Int,
+        max: Int?,
+    ): Result {
+        val outputData = Data.Builder()
+            .putInt(OUTPUT_DOWN_SYNC, count)
+            .putInt(OUTPUT_DOWN_MAX_SYNC, max ?: 0)
+
+        when (t) {
+            is SecurityException -> {
+                outputData.putBoolean(OUTPUT_FAILED_BECAUSE_COMMCARE_PERMISSION_MISSING, true)
+                Simber.i("Down-sync completed with recoverable issue", t, tag = tag)
+            }
+
+            is RemoteDbNotSignedInException -> {
+                outputData.putBoolean(OUTPUT_FAILED_BECAUSE_RELOGIN_REQUIRED, true)
+                Simber.i("Down-sync completed with recoverable issue", t, tag = tag)
+            }
+
+            is IllegalArgumentException -> {
+                Simber.i("Down-sync completed with recoverable issue", t, tag = tag)
+            }
+
+            else -> {
+                Simber.e("Down-sync completed with unexpected issue", t, tag = tag)
+            }
+        }
+
+        return success(outputData.build(), "Completed with down-sync error: ${t.message}")
     }
 }

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/down/workers/SimprintsEventDownSyncDownloaderWorker.kt

@@ -2,8 +2,8 @@
 
 import android.content.Context
 import androidx.hilt.work.HiltWorker
+import androidx.work.Data
 import androidx.work.WorkerParameters
-import androidx.work.workDataOf
 import com.simprints.core.DispatcherBG
 import com.simprints.infra.authstore.exceptions.RemoteDbNotSignedInException
 import com.simprints.infra.config.store.ConfigRepository
@@ -19,6 +19,7 @@
 import com.simprints.infra.eventsync.sync.common.OUTPUT_FAILED_BECAUSE_TOO_MANY_REQUESTS
 import com.simprints.infra.eventsync.sync.down.tasks.BaseEventDownSyncTask
 import com.simprints.infra.eventsync.sync.down.tasks.SimprintsEventDownSyncTask
+import com.simprints.infra.logging.Simber
 import com.simprints.infra.network.exceptions.BackendMaintenanceException
 import com.simprints.infra.network.exceptions.SyncCloudIntegrationException
 import dagger.assisted.Assisted
@@ -48,21 +49,47 @@
     ) {
     override fun createDownSyncTask(): BaseEventDownSyncTask = downSyncTask
 
-    override fun handleSyncException(t: Throwable) = when (t) {
-        is IllegalArgumentException -> fail(t, t.message)
+    override fun handleSyncException(
+        t: Throwable,
+        count: Int,
+        max: Int?,
+    ): Result {
+        val outputData = Data.Builder()
+            .putInt(OUTPUT_DOWN_SYNC, count)
+            .putInt(OUTPUT_DOWN_MAX_SYNC, max ?: 0)
 
-        is BackendMaintenanceException -> fail(
-            t,
-            t.message,
-            workDataOf(
-                OUTPUT_FAILED_BECAUSE_BACKEND_MAINTENANCE to true,
-                OUTPUT_ESTIMATED_MAINTENANCE_TIME to t.estimatedOutage,
-            ),
-        )
+        when (t) {
+            is BackendMaintenanceException -> {
+                outputData
+                    .putBoolean(OUTPUT_FAILED_BECAUSE_BACKEND_MAINTENANCE, true)
+                    .putLong(OUTPUT_ESTIMATED_MAINTENANCE_TIME, t.estimatedOutage ?: 0L)
+                Simber.i("Down-sync completed with recoverable issue", t, tag = tag)
+            }
 
-        is SyncCloudIntegrationException -> fail(t, t.message, workDataOf(OUTPUT_FAILED_BECAUSE_CLOUD_INTEGRATION to true))
-        is TooManyRequestsException -> fail(t, t.message, workDataOf(OUTPUT_FAILED_BECAUSE_TOO_MANY_REQUESTS to true))
-        is RemoteDbNotSignedInException -> fail(t, t.message, workDataOf(OUTPUT_FAILED_BECAUSE_RELOGIN_REQUIRED to true))
-        else -> retry(t)
+            is SyncCloudIntegrationException -> {
+                outputData.putBoolean(OUTPUT_FAILED_BECAUSE_CLOUD_INTEGRATION, true)
+                Simber.i("Down-sync completed with recoverable issue", t, tag = tag)
+            }
+
+            is TooManyRequestsException -> {
+                outputData.putBoolean(OUTPUT_FAILED_BECAUSE_TOO_MANY_REQUESTS, true)
+                Simber.i("Down-sync completed with recoverable issue", t, tag = tag)
+            }
+
+            is RemoteDbNotSignedInException -> {
+                outputData.putBoolean(OUTPUT_FAILED_BECAUSE_RELOGIN_REQUIRED, true)
+                Simber.i("Down-sync completed with recoverable issue", t, tag = tag)
+            }
+
+            is IllegalArgumentException -> {
+                Simber.i("Down-sync completed with recoverable issue", t, tag = tag)
+            }
+
+            else -> {
+                Simber.e("Down-sync completed with unexpected issue", t, tag = tag)
+            }
+        }
+
+        return success(outputData.build(), "Completed with down-sync error: ${t.message}")
     }
 }

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/master/EventEndSyncReporterWorker.kt

@@ -9,10 +9,13 @@ import com.simprints.core.workers.SimCoroutineWorker
 import com.simprints.infra.events.EventRepository
 import com.simprints.infra.events.event.domain.models.scope.EventScopeEndCause
 import com.simprints.infra.eventsync.sync.common.EventSyncCache
+import com.simprints.infra.eventsync.sync.common.SyncWorkersInfoProvider
+import com.simprints.infra.eventsync.sync.common.hasAnyFailureReason
 import com.simprints.infra.logging.Simber
 import dagger.assisted.Assisted
 import dagger.assisted.AssistedInject
 import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.flow.firstOrNull
 import kotlinx.coroutines.withContext
 
 /**
@@ -25,6 +28,7 @@ internal class EventEndSyncReporterWorker @AssistedInject constructor(
     @Assisted params: WorkerParameters,
     private val syncCache: EventSyncCache,
     private val eventRepository: EventRepository,
+    private val syncWorkersInfoProvider: SyncWorkersInfoProvider,
     private val timeHelper: TimeHelper,
     @param:DispatcherBG private val dispatcher: CoroutineDispatcher,
 ) : SimCoroutineWorker(appContext, params) {
@@ -46,7 +50,15 @@ internal class EventEndSyncReporterWorker @AssistedInject constructor(
             }
 
             if (!syncId.isNullOrEmpty()) {
-                syncCache.storeLastSuccessfulSyncTime(timeHelper.now())
+                val hasWorkerFailures = syncWorkersInfoProvider
+                    .getSyncWorkerInfos(syncId)
+                    .firstOrNull()
+                    .orEmpty()
+                    .any { it.hasAnyFailureReason() }
+
+                if (!hasWorkerFailures) {
+                    syncCache.storeLastSuccessfulSyncTime(timeHelper.now())
+                }
                 success()
             } else {
                 throw IllegalArgumentException("SyncId missed")

infra/event-sync/src/main/java/com/simprints/infra/eventsync/sync/up/tasks/EventUpSyncTask.kt

@@ -25,6 +25,7 @@
 import com.simprints.infra.events.event.domain.models.scope.EventScopeType
 import com.simprints.infra.eventsync.event.remote.ApiUploadEventsBody
 import com.simprints.infra.eventsync.event.remote.EventRemoteDataSource
+import com.simprints.infra.eventsync.event.remote.exceptions.TooManyRequestsException
 import com.simprints.infra.eventsync.event.remote.models.session.ApiEventScope
 import com.simprints.infra.eventsync.event.usecases.MapDomainEventScopeToApiUseCase
 import com.simprints.infra.eventsync.exceptions.TryToUploadEventsForNotSignedProject
@@ -37,7 +38,9 @@
 import com.simprints.infra.eventsync.sync.up.EventUpSyncProgress
 import com.simprints.infra.logging.LoggingConstants.CrashReportTag.SYNC
 import com.simprints.infra.logging.Simber
+import com.simprints.infra.network.exceptions.BackendMaintenanceException
 import com.simprints.infra.network.exceptions.NetworkConnectionException
+import com.simprints.infra.network.exceptions.SyncCloudIntegrationException
 import com.simprints.infra.serialization.SimJson
 import kotlinx.coroutines.currentCoroutineContext
 import kotlinx.coroutines.flow.Flow
@@ -173,17 +176,16 @@
 
             emitProgress(lastOperation, 0)
         } catch (t: Throwable) {
-            if (t is RemoteDbNotSignedInException) {
-                throw t
-            }
-
             Simber.e("Failed to upload event scopes", t, tag = SYNC)
             lastOperation = lastOperation.copy(
                 lastState = FAILED,
                 lastSyncTime = timeHelper.now().ms,
             )
 
             emitProgress(lastOperation, 0)
+            if (isTerminalException(t)) {
+                throw t
+            }
         }
     }
 
@@ -195,6 +197,11 @@
         this.emit(EventUpSyncProgress(lastOperation, count))
     }
 
+    private fun isTerminalException(t: Throwable): Boolean = t is RemoteDbNotSignedInException ||
+        t is BackendMaintenanceException ||
+        t is SyncCloudIntegrationException ||
+        t is TooManyRequestsException
+
     private fun uploadEventScopeType(
         eventScope: EventScope,
         project: Project,
@@ -350,9 +357,9 @@
         config.canSyncAllDataToSimprints() -> events
 
         config.canSyncBiometricDataToSimprints() -> events.filter {
             it is EnrolmentEventV2 ||
                 it is EnrolmentEventV4 ||
                 it is PersonCreationEvent ||
                 it is FingerprintCaptureBiometricsEvent ||
                 it is FaceCaptureBiometricsEvent ||
                 it is BiometricReferenceCreationEvent ||