diff --git a/contracts/predictify-hybrid/src/config.rs b/contracts/predictify-hybrid/src/config.rs index f01095e0..62a13444 100644 --- a/contracts/predictify-hybrid/src/config.rs +++ b/contracts/predictify-hybrid/src/config.rs @@ -360,6 +360,18 @@ pub const ORACLE_TIMEOUT_SECONDS: u64 = 30; // ===== MONITORING CONSTANTS ===== +/// Oracle health degradation threshold — consecutive bad samples needed to flip from Working to Degraded. +/// +/// Higher values reduce flapping but delay detection of real issues. +/// Safe range: 2–10. Defaults trade off promptness vs stability. +pub const ORACLE_HEALTH_DEGRADED_THRESHOLD: u32 = 3; + +/// Oracle health recovery threshold — consecutive good samples needed to flip from Degraded back to Working. +/// +/// Higher values prevent premature recovery after a transient blip. +/// Safe range: 2–10. Should be >= the degraded threshold for stability. +pub const ORACLE_HEALTH_RECOVERY_THRESHOLD: u32 = 3; + /// Maximum number of alerts retained in the monitoring queue (FIFO, oldest dropped first). /// /// When the queue is full and a new alert arrives, the oldest entry is evicted and diff --git a/contracts/predictify-hybrid/src/graceful_degradation.rs b/contracts/predictify-hybrid/src/graceful_degradation.rs index 7dd09a7c..44edb354 100644 --- a/contracts/predictify-hybrid/src/graceful_degradation.rs +++ b/contracts/predictify-hybrid/src/graceful_degradation.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] +use crate::config::{ORACLE_HEALTH_DEGRADED_THRESHOLD, ORACLE_HEALTH_RECOVERY_THRESHOLD}; use crate::err::Error; use crate::events::EventEmitter; // use crate::oracles::{OracleInterface, ReflectorOracle}; @@ -19,6 +20,7 @@ enum DegradationStorageKey { struct OracleDegradationState { health: OracleHealth, consecutive_failures: u32, + consecutive_good: u32, last_reason: String, updated_at: u64, } @@ -31,26 +33,93 @@ fn load_degradation_state(env: &Env, oracle: &OracleProvider) -> Option 0, - OracleHealth::Degraded | OracleHealth::Broken => load_degradation_state(env, oracle) - .map(|state| state.consecutive_failures.saturating_add(1)) - .unwrap_or(1), + let previous = load_degradation_state(env, oracle); + let prev_health = previous + .as_ref() + .map(|s| s.health.clone()) + .unwrap_or(OracleHealth::Working); + let prev_failures = previous + .as_ref() + .map(|s| s.consecutive_failures) + .unwrap_or(0); + let prev_good = previous.as_ref().map(|s| s.consecutive_good).unwrap_or(0); + let (new_health, new_failures, new_good, changed) = match sample { + // --- good sample --- + OracleHealth::Working => { + let good = prev_good.saturating_add(1); + let failures = 0; + if prev_health == OracleHealth::Working { + // Already healthy — just update counters. + (OracleHealth::Working, failures, good, false) + } else if good >= ORACLE_HEALTH_RECOVERY_THRESHOLD { + // Enough consecutive good samples to recover. + (OracleHealth::Working, failures, good, true) + } else { + // Still below threshold — stay in current state. + (prev_health.clone(), failures, good, false) + } + } + // --- bad sample --- + OracleHealth::Degraded | OracleHealth::Broken => { + let failures = prev_failures.saturating_add(1); + let good = 0; + match prev_health { + OracleHealth::Degraded | OracleHealth::Broken => { + // Already degraded / broken — stay there. + (prev_health.clone(), failures, good, false) + } + OracleHealth::Working => { + if failures >= ORACLE_HEALTH_DEGRADED_THRESHOLD { + // Enough consecutive failures to mark degraded. + (OracleHealth::Degraded, failures, good, true) + } else { + // Below threshold — still Working on paper. + (OracleHealth::Working, failures, good, false) + } + } + } + } }; + // Emit event only on actual state change. + if changed { + let provider_str = String::from_str(env, oracle.as_str()); + // Generate a valid placeholder address since this code path only has + // an OracleProvider enum (not a real contract address). The `provider` + // field in the event carries the oracle identity. Callers that have + // the real address (e.g., OracleBackup methods) can emit separately. + EventEmitter::emit_oracle_health_status( + env, + &Address::generate(env), + &provider_str, + prev_health == OracleHealth::Working, + new_health == OracleHealth::Working, + new_failures, + ); + } + let state = OracleDegradationState { - health, - consecutive_failures, + health: new_health, + consecutive_failures: new_failures, + consecutive_good: new_good, last_reason: reason.clone(), updated_at: env.ledger().timestamp(), }; - env.storage().persistent().set(°radation_key(oracle), &state); + env.storage() + .persistent() + .set(°radation_key(oracle), &state); } // Basic oracle backup system @@ -189,14 +258,27 @@ pub fn monitor_oracle_health( oracle_address: &Address, ) -> OracleHealth { let backup = OracleBackup::new(oracle.clone(), oracle); - let stored_health = load_degradation_state(env, &backup.primary).map(|state| state.health); - // Probe live first so a recovered oracle clears any prior degraded state. - if backup.is_working(env, oracle_address).unwrap_or(false) { + // Probe the oracle. + let working = backup.is_working(env, oracle_address).unwrap_or(false); + + // Record the sample through the hysteresis gate. + let sample = if working { OracleHealth::Working } else { - stored_health.unwrap_or(OracleHealth::Broken) - } + OracleHealth::Degraded + }; + let reason = if working { + String::from_str(env, "Oracle probe succeeded") + } else { + String::from_str(env, "Oracle probe failed") + }; + record_oracle_health(env, &oracle, sample, &reason); + + // Return the *stored* health (after hysteresis), not the raw sample. + load_degradation_state(env, &oracle) + .map(|s| s.health) + .unwrap_or(OracleHealth::Working) } pub fn get_degradation_status( @@ -242,6 +324,10 @@ mod tests { use soroban_sdk::testutils::Events; use soroban_sdk::Env; + // ------------------------------------------------------------------ + // Legacy / existing behaviour tests (adapted for hysteresis) + // ------------------------------------------------------------------ + #[test] fn can_create_backup() { let backup = OracleBackup::new(OracleProvider::reflector(), OracleProvider::pyth()); @@ -252,11 +338,9 @@ mod tests { #[test] fn can_check_health() { let env = Env::default(); - //1. register the contract so we have a context let contract_id = env.register(crate::PredictifyHybrid, ()); let addr = Address::generate(&env); - //2. wrap the execution in the contract context env.as_contract(&contract_id, || { let health = monitor_oracle_health(&env, OracleProvider::reflector(), &addr); assert!(matches!( @@ -288,19 +372,16 @@ mod tests { #[test] fn test_is_working_propagates_error_and_emits_event() { let env = Env::default(); - // 1. Register the contract let contract_id = env.register(crate::PredictifyHybrid, ()); let backup = OracleBackup::new(OracleProvider::pyth(), OracleProvider::dia()); let oracle_address = Address::generate(&env); - // 2. Wrap the execution in the contract context env.as_contract(&contract_id, || { let result = backup.is_working(&env, &oracle_address); - assert!(result.is_err()); // No longer fails silently + assert!(result.is_err()); }); - // 3. Verify event emission let events = env.events().all(); assert!( events.events().len() > 0, @@ -328,23 +409,234 @@ mod tests { ); } + // ------------------------------------------------------------------ + // Hysteresis tests + // ------------------------------------------------------------------ + + /// Helper: directly record a health sample (bypassing probe). + fn record_sample(env: &Env, oracle: &OracleProvider, working: bool) { + let sample = if working { + OracleHealth::Working + } else { + OracleHealth::Degraded + }; + let reason = String::from_str(env, if working { "ok" } else { "fail" }); + record_oracle_health(env, oracle, sample, &reason); + } + + /// Helper: load the stored health for an oracle. + fn stored_health(env: &Env, oracle: &OracleProvider) -> OracleHealth { + load_degradation_state(env, oracle) + .map(|s| s.health) + .unwrap_or(OracleHealth::Working) + } + #[test] - fn test_oracle_fallback_timeout_marks_oracle_degraded() { + fn hysteresis_single_bad_sample_does_not_transition() { + let env = Env::default(); + let contract_id = env.register(crate::PredictifyHybrid, ()); + let oracle = OracleProvider::reflector(); + + env.as_contract(&contract_id, || { + // Start from default (Working). + assert_eq!(stored_health(&env, &oracle), OracleHealth::Working); + + // 1 bad sample — should NOT flip to Degraded. + record_sample(&env, &oracle, false); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Working); + + // 2 bad samples — still below threshold. + record_sample(&env, &oracle, false); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Working); + }); + } + + #[test] + fn hysteresis_three_consecutive_bad_triggers_degraded() { + let env = Env::default(); + let contract_id = env.register(crate::PredictifyHybrid, ()); + let oracle = OracleProvider::reflector(); + + env.as_contract(&contract_id, || { + // 3 consecutive bad samples = transition to Degraded. + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, false); + + assert_eq!(stored_health(&env, &oracle), OracleHealth::Degraded); + + // Verify consecutive_failures counter. + let state = load_degradation_state(&env, &oracle).unwrap(); + assert_eq!(state.consecutive_failures, 3); + }); + } + + #[test] + fn hysteresis_good_sample_resets_bad_counter() { + let env = Env::default(); + let contract_id = env.register(crate::PredictifyHybrid, ()); + let oracle = OracleProvider::reflector(); + + env.as_contract(&contract_id, || { + // 2 bad, then 1 good — should reset counter and stay Working. + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, true); + + assert_eq!(stored_health(&env, &oracle), OracleHealth::Working); + let state = load_degradation_state(&env, &oracle).unwrap(); + assert_eq!(state.consecutive_failures, 0); + assert_eq!(state.consecutive_good, 1); + }); + } + + #[test] + fn hysteresis_recovery_requires_three_consecutive_good() { + let env = Env::default(); + let contract_id = env.register(crate::PredictifyHybrid, ()); + let oracle = OracleProvider::reflector(); + + env.as_contract(&contract_id, || { + // Push to Degraded. + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, false); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Degraded); + + // 1 good — still Degraded. + record_sample(&env, &oracle, true); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Degraded); + + // 2 good — still Degraded. + record_sample(&env, &oracle, true); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Degraded); + + // 3 good — recovery! + record_sample(&env, &oracle, true); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Working); + }); + } + + #[test] + fn hysteresis_event_emitted_only_on_state_transition() { + let env = Env::default(); + let contract_id = env.register(crate::PredictifyHybrid, ()); + let oracle = OracleProvider::reflector(); + + env.as_contract(&contract_id, || { + // Bad samples #1 and #2 — no transition, so no "orc_hlth" event. + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, false); + + // No state change yet — confirm via state assertion. + assert_eq!(stored_health(&env, &oracle), OracleHealth::Working); + }); + + // Check that the health-status event ("orc_hlth") was NOT emitted + // (no transition happened). The oracle_degradation event is still + // emitted by is_working, so we don't check total event count. + let events = env.events().all(); + let health_events: soroban_sdk::Vec<_> = events + .events() + .iter() + .filter(|e| e.0 == (soroban_sdk::symbol_short!("orc_hlth"),)) + .collect(); + assert!( + health_events.is_empty(), + "No OracleHealthStatusEvent should be emitted before transition" + ); + + // Now trigger the transition with a 3rd bad sample. + env.as_contract(&contract_id, || { + record_sample(&env, &oracle, false); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Degraded); + }); + + let events = env.events().all(); + let health_events: soroban_sdk::Vec<_> = events + .events() + .iter() + .filter(|e| e.0 == (soroban_sdk::symbol_short!("orc_hlth"),)) + .collect(); + assert!( + health_events.len() >= 1, + "OracleHealthStatusEvent should be emitted on transition to Degraded" + ); + } + + #[test] + fn hysteresis_timeout_requires_three_calls_to_degrade() { let env = Env::default(); let contract_id = env.register(crate::PredictifyHybrid, ()); let oracle = OracleProvider::reflector(); let oracle_address = Address::generate(&env); env.as_contract(&contract_id, || { + // 1st timeout — should NOT flip immediately (needs 3). + handle_oracle_timeout(oracle.clone(), 61, &env); + let health = get_degradation_status(oracle.clone(), &env, &oracle_address); + assert_eq!(health, OracleHealth::Working); + + // 2nd timeout — still not enough. + handle_oracle_timeout(oracle.clone(), 61, &env); + let health = get_degradation_status(oracle.clone(), &env, &oracle_address); + assert_eq!(health, OracleHealth::Working); + + // 3rd timeout — now it degrades. handle_oracle_timeout(oracle.clone(), 61, &env); let health = get_degradation_status(oracle.clone(), &env, &oracle_address); assert_eq!(health, OracleHealth::Degraded); }); + } - let events = env.events().all(); - assert!( - events.events().len() > 0, - "Expected timeout handling to emit a degradation event" - ); + #[test] + fn hysteresis_no_event_on_noop_transition() { + // When already Degraded, another bad sample should NOT re-emit the event. + // We verify by checking that the health status stays Degraded and the + // consecutive_failures counter increments. + let env = Env::default(); + let contract_id = env.register(crate::PredictifyHybrid, ()); + let oracle = OracleProvider::reflector(); + + env.as_contract(&contract_id, || { + // Push to Degraded. + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, false); + record_sample(&env, &oracle, false); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Degraded); + + // Additional bad sample — stays Degraded, failure counter increments. + record_sample(&env, &oracle, false); + assert_eq!(stored_health(&env, &oracle), OracleHealth::Degraded); + let state = load_degradation_state(&env, &oracle).unwrap(); + assert_eq!(state.consecutive_failures, 4); + }); + } + + #[test] + fn hysteresis_get_degradation_status_uses_hysteresis() { + // `get_degradation_status` calls `monitor_oracle_health` which probes + // the oracle. Since the oracle is unregistered, probes fail, but + // the hysteresis gate should require 3 consecutive failures before + // returning Degraded. + let env = Env::default(); + let contract_id = env.register(crate::PredictifyHybrid, ()); + let oracle = OracleProvider::reflector(); + let oracle_address = Address::generate(&env); + + env.as_contract(&contract_id, || { + // First probe — should still return Working (hysteresis gate). + let health = get_degradation_status(oracle.clone(), &env, &oracle_address); + assert_eq!(health, OracleHealth::Working); + + // Second probe — still Working. + let health = get_degradation_status(oracle.clone(), &env, &oracle_address); + assert_eq!(health, OracleHealth::Working); + + // Third probe — now Degraded. + let health = get_degradation_status(oracle.clone(), &env, &oracle_address); + assert_eq!(health, OracleHealth::Degraded); + }); } } +