improve multipart part retries with exponential backoff & preserve root errors

Sources/InternxtSwiftCore/Services/Network/NetworkFacade.swift

@@ -215,6 +215,9 @@ public struct NetworkFacade {
             iv: iv
         ) { encryptedChunk in
             guard await !uploadState.isAborted() else {
+                if let abortError = await uploadState.getAbortError() {
+                    throw abortError
+                }
                 throw UploadError.UploadNotSuccessful
             }
 
@@ -541,12 +544,61 @@ public struct NetworkFacade {
             }
         }
 
+        static func isRetryableChunkError(_ error: Error) -> Bool {
+            var unwrappedError = error
+            if case UploadError.PartUploadFailed(_, let innerError) = error {
+                unwrappedError = innerError
+            }
+
+            if let urlError = unwrappedError as? URLError {
+                switch urlError.code {
+                case .timedOut, .notConnectedToInternet, .networkConnectionLost, .cannotConnectToHost, .cannotFindHost:
+                    return true
+                default:
+                    return false
+                }
+            }
+
+            if let apiClientError = unwrappedError as? APIClientError {
+                if apiClientError.statusCode >= 500 || apiClientError.statusCode == 429 || apiClientError.statusCode == -1 {
+                    return true
+                }
+                if apiClientError.statusCode >= 400 && apiClientError.statusCode < 500 {
+                    return false
+                }
+            }
+
+            if let uploadError = unwrappedError as? UploadError {
+                switch uploadError {
+                case .MissingChunk, .MissingUploadUrl, .InvalidIndex, .CannotGenerateFileHash:
+                    return false
+                default:
+                    return true
+                }
+            }
+
+            let nsError = unwrappedError as NSError
+            if nsError.domain == NSURLErrorDomain {
+                switch nsError.code {
+                case NSURLErrorTimedOut, NSURLErrorNotConnectedToInternet, NSURLErrorNetworkConnectionLost, NSURLErrorCannotConnectToHost, NSURLErrorCannotFindHost:
+                    return true
+                default:
+                    break
+                }
+            }
+
+            return true
+        }
+
         private func uploadPartWithRetry() async throws {
             var attempt = 0
             let maxRetries = 3
 
             while attempt < maxRetries {
                 if await uploadState.isAborted() {
+                    if let abortError = await uploadState.getAbortError() {
+                        throw abortError
+                    }
                     throw UploadError.UploadNotSuccessful
                 }
 
@@ -558,8 +610,9 @@ public struct NetworkFacade {
                         while !NetworkMonitor.shared.isConnected {
                             try await Task.sleep(nanoseconds: 20 * 1_000_000_000) // Check every 20 seconds
                             if Date().timeIntervalSince(startTime) > MAX_WAIT_TIME {
-                                await uploadState.setAborted()
-                                throw UploadError.UploadNotSuccessful
+                                let abortErr = UploadError.PartUploadFailed(partIndex: partIndex, error: APIClientError(statusCode: -1, message: "Network connection wait timeout"))
+                                await uploadState.setAborted(error: abortErr)
+                                throw abortErr
                             }
                         }
                     }
@@ -579,17 +632,23 @@ public struct NetworkFacade {
                     return
                 }
                 catch {
-                    if let urlError = error as? URLError,
-                           urlError.code == .notConnectedToInternet || urlError.code == .networkConnectionLost  {
-                    }else {
-
-                        attempt += 1
-                        if attempt >= maxRetries {
-                            await uploadState.setAborted()
-                            throw UploadError.PartUploadFailed(partIndex: partIndex, error: error)
-                        }
+                    let partError = (error as? UploadError) ?? UploadError.PartUploadFailed(partIndex: partIndex, error: error)
+
+                    if !Self.isRetryableChunkError(partError) {
+                        await uploadState.setAborted(error: partError)
+                        throw partError
+                    }
+
+                    attempt += 1
+                    if attempt >= maxRetries {
+                        await uploadState.setAborted(error: partError)
+                        throw partError
                     }
 
+                    let baseDelay = pow(2.0, Double(attempt))
+                    let jitter = Double.random(in: 0.0...(baseDelay * 0.25))
+                    let totalDelay = baseDelay + jitter
+                    try await Task.sleep(nanoseconds: UInt64(totalDelay * 1_000_000_000))
                 }
             }
         }

Sources/InternxtSwiftCore/Services/Network/UploadMultipart.swift

@@ -81,7 +81,7 @@ public class UploadMultipart: NSObject {
     func uploadPart(encryptedChunk: Data, uploadUrl: String, partIndex: Int, progressHandler: @escaping ProgressHandler) async throws -> String {
 
         // Upload the chunk to the given URL
-        let uploadEtag = try await self.uploadEncryptedChunk(encryptedChunk: encryptedChunk, uploadUrl: uploadUrl, progressHandler: progressHandler)
+        let uploadEtag = try await self.uploadEncryptedChunk(encryptedChunk: encryptedChunk, uploadUrl: uploadUrl, partIndex: partIndex, progressHandler: progressHandler)
 
 
         return uploadEtag        
@@ -128,7 +128,7 @@ public class UploadMultipart: NSObject {
 
 
 
-    private func uploadEncryptedChunk(encryptedChunk: Data, uploadUrl: String, progressHandler: ProgressHandler?) async throws -> String {
+    private func uploadEncryptedChunk(encryptedChunk: Data, uploadUrl: String, partIndex: Int, progressHandler: ProgressHandler?) async throws -> String {
         return try await withCheckedThrowingContinuation { (continuation) in
             var request = URLRequest(
                 url: URL(string: uploadUrl)!,
@@ -144,8 +144,10 @@ public class UploadMultipart: NSObject {
                 completionHandler: { data, res, error in
                     guard let error = error else {
                         let response = res as? HTTPURLResponse
-                        if response?.statusCode != 200 {
-                            return continuation.resume(with: .failure(UploadError.UploadNotSuccessful))
+                        let statusCode = response?.statusCode ?? -999
+                        if statusCode != 200 {
+                            let apiError = APIClientError(statusCode: statusCode, message: "[PUT S3 Chunk] Returned HTTP \(statusCode)")
+                            return continuation.resume(with: .failure(UploadError.PartUploadFailed(partIndex: partIndex, error: apiError)))
                         } else {
                             guard let etagValue = response?.value(forHTTPHeaderField: "Etag") else {
                                 return continuation.resume(with: .failure(UploadError.MissingEtag))
@@ -155,7 +157,7 @@ public class UploadMultipart: NSObject {
 
                     }
 
-                    continuation.resume(throwing: error)
+                    continuation.resume(throwing: UploadError.PartUploadFailed(partIndex: partIndex, error: error))
                 }
             )
 

Sources/InternxtSwiftCore/Utils/Errors.swift

@@ -181,6 +181,31 @@ public enum UploadError: Error, Equatable {
     }
 }
 
+extension UploadError: LocalizedError {
+    public var errorDescription: String? {
+        switch self {
+        case .PartUploadFailed(let partIndex, let innerError):
+            return "PartUploadFailed(partIndex: \(partIndex), error: \(innerError.localizedDescription))"
+        case .InvalidIndex:
+            return "InvalidIndex"
+        case .CannotGenerateFileHash:
+            return "CannotGenerateFileHash"
+        case .FailedToFinishUpload:
+            return "FailedToFinishUpload"
+        case .MissingUploadUrl:
+            return "MissingUploadUrl"
+        case .UploadNotSuccessful:
+            return "UploadNotSuccessful"
+        case .UploadedSizeNotMatching:
+            return "UploadedSizeNotMatching"
+        case .MissingEtag:
+            return "MissingEtag"
+        case .MissingChunk:
+            return "MissingChunk"
+        }
+    }
+}
+
 
 public class StartUploadError: Error {
     public var apiError: APIClientError? = nil

Tests/InternxtSwiftCoreTests/Services/Network/NetworkFacadeTests.swift

@@ -254,4 +254,56 @@ final class NetworkFacadeTests: XCTestCase {
             XCTFail("Expected EnrichedError but got \(type(of: error))")
         }
     }
+
+    func testUploadErrorLocalizedDescription() {
+        let innerErr = URLError(.timedOut)
+        let partError = UploadError.PartUploadFailed(partIndex: 0, error: innerErr)
+
+        XCTAssertTrue(partError.localizedDescription.contains("PartUploadFailed"))
+        XCTAssertTrue(partError.localizedDescription.contains("partIndex: 0"))
+        XCTAssertEqual(UploadError.UploadNotSuccessful.localizedDescription, "UploadNotSuccessful")
+    }
+
+    func testUploadStatePreservesAbortError() async throws {
+        let uploadState = NetworkFacade.UploadState()
+        let expectedError = UploadError.PartUploadFailed(partIndex: 2, error: URLError(.timedOut))
+
+        await uploadState.setAborted(error: expectedError)
+
+        let isAborted = await uploadState.isAborted()
+        XCTAssertTrue(isAborted)
+
+        let rawError = await uploadState.getAbortError()
+        let savedError = try XCTUnwrap(rawError, "Expected a non-nil abort error")
+        let uploadErr = try XCTUnwrap(savedError as? UploadError, "Expected abort error to be of type UploadError")
+        guard case let .PartUploadFailed(partIndex, _) = uploadErr else {
+            XCTFail("Expected UploadError.PartUploadFailed  \(uploadErr)")
+            return
+        }
+        XCTAssertEqual(partIndex, 2)
+    }
+
+    func testPartUploadFailedEqualityAndMatching() {
+        let err1 = UploadError.PartUploadFailed(partIndex: 1, error: URLError(.timedOut))
+        let err2 = UploadError.PartUploadFailed(partIndex: 1, error: URLError(.timedOut))
+        let err3 = UploadError.PartUploadFailed(partIndex: 2, error: URLError(.timedOut))
+
+        XCTAssertEqual(err1, err2)
+        XCTAssertNotEqual(err1, err3)
+        XCTAssertNotEqual(err1, UploadError.UploadNotSuccessful)
+    }
+
+    func testIsRetryableChunkError() {
+        let timeoutError = UploadError.PartUploadFailed(partIndex: 0, error: URLError(.timedOut))
+        XCTAssertTrue(NetworkFacade.UploadPartOperation.isRetryableChunkError(timeoutError))
+
+        let serverError = UploadError.PartUploadFailed(partIndex: 0, error: APIClientError(statusCode: 500, message: "Server Error"))
+        XCTAssertTrue(NetworkFacade.UploadPartOperation.isRetryableChunkError(serverError))
+
+        let forbiddenError = UploadError.PartUploadFailed(partIndex: 0, error: APIClientError(statusCode: 403, message: "Forbidden"))
+        XCTAssertFalse(NetworkFacade.UploadPartOperation.isRetryableChunkError(forbiddenError))
+
+        let missingChunkError = UploadError.MissingChunk
+        XCTAssertFalse(NetworkFacade.UploadPartOperation.isRetryableChunkError(missingChunkError))
+    }
 }