diff --git a/services/orchestrator/driver/src/driver.rs b/services/orchestrator/driver/src/driver.rs index ad8dc9ef..b21b3121 100644 --- a/services/orchestrator/driver/src/driver.rs +++ b/services/orchestrator/driver/src/driver.rs @@ -15,8 +15,9 @@ use crate::board::{ Board, BoardCapabilities, ImageSource, Report, ReportSink, SvnFloorBinding, Verdict, Verifier, }; use orchestrator_capabilities::{ - BootControl, BootWatch, FailureCause, Progress, Recovery, RestoreOutcome, StageProgress, Svn, - SvnFloor, Updatable, WalkVerdict, + BootControl, BootWatch, FailureCause, Progress, Recovery, RestoreOutcome, RunningImage, + SelfUpdate, SelfUpdateState, StageProgress, Svn, SvnFloor, TrialOutcome, Updatable, + WalkVerdict, }; use util_io::{ByteSource, ByteWindow}; @@ -621,6 +622,33 @@ impl PlatformDriver { } } +/// What [`settle_self_update`] found in the eRoT's last self-update. +/// +/// What to do about it is the caller's, the same split the capability seams +/// use. +#[must_use] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum SelfUpdateSettlement { + /// Nothing is left to do, so the boot carries on and a new self-update + /// may start: no session, a session nothing would confirm and that was + /// reverted, or a confirmed session whose floor had already taken its + /// SVN. + Settled, + /// A session is still being judged, either a trial run waiting on the + /// update agent or a confirmed one whose floor is still below `svn`. No + /// new self-update may start: recording one would overwrite the session. + AwaitingUpdateAgent { + /// The SVN the session recorded, and the floor's target. + svn: Svn, + }, + /// A trial image is running that no session claims, so the eRoT must not + /// keep running it. The session is reverted, which clears the pending + /// mark, so the confirmed image runs after a reset. An implementation + /// whose `revert` leaves the mark in place would run the same image + /// again. + UnclaimedImageRunning, +} + /// One [`PlatformDriver::pump_update`] round. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct UpdatePoll { @@ -743,6 +771,127 @@ pub fn bring_up( ) } +/// Finishes off the eRoT's last self-update, at the start of the next boot. +/// +/// An update writes the new image to the secondary slot and marks the session +/// pending, so the next reset runs that image once. That trial run is then +/// either confirmed or reverted. The eRoT loses all of RAM across the reset, +/// so the session in durable storage plus the image this boot is running is +/// everything it has to go on. This reads both and says what it found: +/// +/// - No session: nothing happened, carry on. +/// - The session is pending and the trial image booted: this boot is the trial +/// run. Nothing here judges it. The update agent does, with +/// UpdateSecurityRevision, so the session is left alone. +/// - The session is prepared or pending but the confirmed image booted: the +/// update never ran, or its trial failed and the platform fell back. The +/// session is reverted, so the update is done again rather than counted as +/// finished. +/// - A trial image is running that no session claims: the session is reverted, +/// which also clears the pending mark, so the confirmed image runs after a +/// reset. The eRoT must not keep running an image nothing vouched for. +/// - The session is confirmed and the floor is still below its SVN: the floor +/// advance is what is left, and the update agent asks for it, so this boot +/// must not do it. +/// - The session is confirmed and the floor already reads that SVN or higher: +/// the advance landed before a crash, so the session is closed here. +/// +/// Running it twice lands in the same place, which is what lets a boot that +/// died partway through simply repeat it. A storage fault leaves the session +/// as it is and comes back as [`SettleError`] with the storage's own error +/// inside, rather than guessing. +/// +/// Must run before the machine can grant a new update: `prepare` overwrites +/// whatever session is there, so a new update recorded over one still being +/// judged would lose what it owes. +/// +/// The trial gets one boot. A platform whose pending mark survives a reset has +/// to clear it before this runs, otherwise an unplanned reset runs the trial +/// image again. +/// +/// Takes the session and the eRoT's own floor directly. Neither is board +/// wiring: nothing calls this yet, and a seam joins [`BoardCapabilities`] when +/// an executor needs it. +pub fn settle_self_update( + session: &mut S, + floor: &F, +) -> Result> { + let state = session.state().map_err(SettleError::Session)?; + let running = session.running().map_err(SettleError::Session)?; + match orchestrator_capabilities::trial_outcome(state, running) { + TrialOutcome::NoSession => Ok(SelfUpdateSettlement::Settled), + TrialOutcome::InProgress => { + // Nothing here judges the trial. The update agent does. + let SelfUpdateState::TrialPending { svn } = state else { + // trial_outcome answers InProgress for TrialPending alone. + // Fail secure rather than guess which SVN was recorded. + return Err(SettleError::InconsistentSession(state)); + }; + Ok(SelfUpdateSettlement::AwaitingUpdateAgent { svn }) + } + TrialOutcome::Unconfirmed => { + session.revert().map_err(SettleError::Session)?; + if running == RunningImage::Trial { + return Ok(SelfUpdateSettlement::UnclaimedImageRunning); + } + Ok(SelfUpdateSettlement::Settled) + } + TrialOutcome::ConfirmedUncommitted { svn } => { + let reached = floor.floor().map_err(SettleError::Floor)?; + if reached >= svn { + session.complete().map_err(SettleError::Session)?; + return Ok(SelfUpdateSettlement::Settled); + } + Ok(SelfUpdateSettlement::AwaitingUpdateAgent { svn }) + } + } +} + +/// Why [`settle_self_update`] could not finish. +/// +/// Carries the storage error rather than flattening it, which is what the +/// `core::error::Error` bound on both seams is for. A caller that only needs +/// to fail secure collapses it to one [`DriverError`] in a line. +#[derive(Debug, PartialEq, Eq)] +pub enum SettleError { + /// The session could not be read or written. + Session(S), + /// The eRoT's own anti-rollback floor could not be read. + Floor(F), + /// `trial_outcome` said a trial is in progress for a state that is not + /// `TrialPending`. It cannot today, since that is the only state it + /// answers `InProgress` for. Kept so a change there fails secure rather + /// than guessing which SVN was recorded. + InconsistentSession(SelfUpdateState), +} + +impl core::fmt::Display for SettleError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + SettleError::Session(err) => write!(f, "self-update session: {err}"), + SettleError::Floor(err) => write!(f, "self-update floor: {err}"), + SettleError::InconsistentSession(state) => { + write!( + f, + "self-update session reads {state:?} and cannot be settled" + ) + } + } + } +} + +impl core::error::Error + for SettleError +{ + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + match self { + SettleError::Session(err) => Some(err), + SettleError::Floor(err) => Some(err), + SettleError::InconsistentSession(_) => None, + } + } +} + /// The connection between an update frontend and the SM: called (by the /// event loop, on the frontend's behalf) once a complete candidate for /// `target` sits in the staging region. Records the job first, then injects diff --git a/services/orchestrator/driver/src/lib.rs b/services/orchestrator/driver/src/lib.rs index 7279dc0a..21b53951 100644 --- a/services/orchestrator/driver/src/lib.rs +++ b/services/orchestrator/driver/src/lib.rs @@ -38,4 +38,7 @@ mod tests; pub use board::{ Board, BoardCapabilities, ImageSource, Report, ReportSink, SvnFloorBinding, Verdict, Verifier, }; -pub use driver::{bring_up, request_update, BootWalkPoll, DriverError, PlatformDriver, UpdatePoll}; +pub use driver::{ + bring_up, request_update, settle_self_update, BootWalkPoll, DriverError, PlatformDriver, + SelfUpdateSettlement, SettleError, UpdatePoll, +}; diff --git a/services/orchestrator/driver/src/tests.rs b/services/orchestrator/driver/src/tests.rs index 6c34ea09..85d2e869 100644 --- a/services/orchestrator/driver/src/tests.rs +++ b/services/orchestrator/driver/src/tests.rs @@ -9,7 +9,8 @@ use openprot_orchestrator_sm::{ Platform, PowerOnResult, State, }; use orchestrator_capabilities::{ - BootWatch, FailureCause, Progress, Recovery, RestoreOutcome, Svn, SvnFloor, WalkVerdict, + BootWatch, FailureCause, Progress, Recovery, RestoreOutcome, RunningImage, SelfUpdate, + SelfUpdateState, Svn, SvnFloor, WalkVerdict, }; use util_io::{ByteReadError, ByteSource}; @@ -269,11 +270,22 @@ struct MockFloor { impl MockFloor { fn new() -> Self { + Self::with_floor(Svn(0)) + } + + fn with_floor(floor: Svn) -> Self { Self { - floor: 0, + floor: floor.0, fail: false, } } + + fn faulting() -> Self { + Self { + floor: 0, + fail: true, + } + } } /// Chain entries for an all-passive chain of `N` components, which is what @@ -298,9 +310,9 @@ fn entries_with_kinds( }) } -/// Update adapter without a HAL. Wiring-only for now: it stages the whole -/// payload in one step. The update pump replaces it with a stepping mock -/// when the executors land. +/// Update adapter without a HAL. Stages the whole payload in one step by +/// default; `stepping`, `stalling` and `faulting` give the pump the other +/// shapes it has to handle. struct MockUpdatable { ready: bool, active: bool, @@ -460,6 +472,115 @@ impl orchestrator_capabilities::Updatable for MockUpdatable { } } +/// The eRoT's own update session, in RAM. `running` says which image this +/// boot runs, the trial one or the confirmed one, as a test sets it. +struct MockSelfUpdate { + state: SelfUpdateState, + running: RunningImage, + fail: bool, +} + +impl MockSelfUpdate { + fn idle() -> Self { + Self { + state: SelfUpdateState::Idle, + running: RunningImage::Confirmed, + fail: false, + } + } + + fn with_session(state: SelfUpdateState, running: RunningImage) -> Self { + Self { + state, + running, + fail: false, + } + } +} + +impl SelfUpdate for MockSelfUpdate { + type Error = SelfSessionFault; + + fn state(&self) -> Result { + self.checked().map(|_| self.state) + } + + fn running(&self) -> Result { + self.checked().map(|_| self.running) + } + + fn prepare(&mut self, svn: Svn) -> Result<(), SelfSessionFault> { + self.checked()?; + self.state = SelfUpdateState::Prepared { svn }; + Ok(()) + } + + fn set_trial_pending(&mut self) -> Result<(), SelfSessionFault> { + self.checked()?; + // Marking the trial pending twice is a retry, not a fault. + let (SelfUpdateState::Prepared { svn } | SelfUpdateState::TrialPending { svn }) = + self.state + else { + return Err(SelfSessionFault); + }; + self.state = SelfUpdateState::TrialPending { svn }; + Ok(()) + } + + fn confirm(&mut self) -> Result<(), SelfSessionFault> { + self.checked()?; + // Confirming a confirmed session is the same retry. + let (SelfUpdateState::TrialPending { svn } | SelfUpdateState::Committed { svn }) = + self.state + else { + return Err(SelfSessionFault); + }; + self.state = SelfUpdateState::Committed { svn }; + Ok(()) + } + + fn complete(&mut self) -> Result<(), SelfSessionFault> { + self.checked()?; + // A session still being judged is dropped with revert, never + // completed, so complete faults rather than discard it. + if !matches!( + self.state, + SelfUpdateState::Committed { .. } | SelfUpdateState::Idle + ) { + return Err(SelfSessionFault); + } + self.state = SelfUpdateState::Idle; + Ok(()) + } + + fn revert(&mut self) -> Result<(), SelfSessionFault> { + self.checked()?; + self.state = SelfUpdateState::Idle; + Ok(()) + } +} + +impl MockSelfUpdate { + fn checked(&self) -> Result<(), SelfSessionFault> { + if self.fail { + Err(SelfSessionFault) + } else { + Ok(()) + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct SelfSessionFault; + +impl core::fmt::Display for SelfSessionFault { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("self-update session fault") + } +} + +impl core::error::Error for SelfSessionFault {} + /// The test board's type choices. struct MockBoard; @@ -1952,3 +2073,150 @@ fn a_second_request_while_an_update_runs_is_refused_as_busy() { "the running job survives" ); } + +// --------------------------------------------------------------------------- +// Settling the eRoT's own last update at boot +// --------------------------------------------------------------------------- + +// No session: the ordinary boot changes nothing. +#[test] +fn settle_with_no_session_changes_nothing() { + let mut session = MockSelfUpdate::idle(); + + assert_eq!( + settle_self_update(&mut session, &MockFloor::new()), + Ok(SelfUpdateSettlement::Settled) + ); + + assert_eq!(session.state, SelfUpdateState::Idle); +} + +// The trial image is running. The update agent judges it, so the session +// stays as it is. +#[test] +fn settle_leaves_a_running_trial_for_the_update_agent() { + let pending = SelfUpdateState::TrialPending { svn: Svn(7) }; + let mut session = MockSelfUpdate::with_session(pending, RunningImage::Trial); + + assert_eq!( + settle_self_update(&mut session, &MockFloor::new()), + Ok(SelfUpdateSettlement::AwaitingUpdateAgent { svn: Svn(7) }) + ); + + assert_eq!(session.state, pending); +} + +// The trial ran and the platform fell back, so nothing will confirm it. +// The session is reverted and the update has to be done again. +#[test] +fn settle_reverts_a_trial_that_fell_back() { + let mut session = MockSelfUpdate::with_session( + SelfUpdateState::TrialPending { svn: Svn(7) }, + RunningImage::Confirmed, + ); + + assert_eq!( + settle_self_update(&mut session, &MockFloor::new()), + Ok(SelfUpdateSettlement::Settled), + "the confirmed image booted, so there is nothing to judge" + ); + + assert_eq!(session.state, SelfUpdateState::Idle); +} + +// Confirmed, floor still below the session's SVN. The advance is the +// update agent's to ask for, so this boot keeps the session and moves +// nothing. +#[test] +fn settle_keeps_a_confirmed_session_the_floor_has_not_taken() { + let committed = SelfUpdateState::Committed { svn: Svn(7) }; + let mut session = MockSelfUpdate::with_session(committed, RunningImage::Trial); + let floor = MockFloor::with_floor(Svn(3)); + + assert_eq!( + settle_self_update(&mut session, &floor), + Ok(SelfUpdateSettlement::AwaitingUpdateAgent { svn: Svn(7) }) + ); + + assert_eq!(session.state, committed); + assert_eq!(floor.floor(), Ok(Svn(3))); +} + +// The crash point between advancing the floor and closing the session. +// The floor already reads the session's SVN, so the advance landed and the +// session is closed here. +#[test] +fn settle_completes_a_session_whose_floor_already_moved() { + let mut session = MockSelfUpdate::with_session( + SelfUpdateState::Committed { svn: Svn(7) }, + RunningImage::Trial, + ); + + assert_eq!( + settle_self_update(&mut session, &MockFloor::with_floor(Svn(7))), + Ok(SelfUpdateSettlement::Settled) + ); + + assert_eq!(session.state, SelfUpdateState::Idle); +} + +// A floor above the session's SVN closes it too: something moved the floor +// further after the advance landed. +#[test] +fn settle_completes_a_session_whose_floor_moved_past_it() { + let mut session = MockSelfUpdate::with_session( + SelfUpdateState::Committed { svn: Svn(7) }, + RunningImage::Trial, + ); + + assert_eq!( + settle_self_update(&mut session, &MockFloor::with_floor(Svn(9))), + Ok(SelfUpdateSettlement::Settled) + ); + + assert_eq!(session.state, SelfUpdateState::Idle); +} + +// An unreadable session fails secure rather than guessing: without it the +// eRoT cannot tell a confirmed update from a reverted one. The storage +// error comes back with it. +#[test] +fn settle_with_an_unreadable_session_fails_secure() { + let mut session = MockSelfUpdate::idle(); + session.fail = true; + + assert_eq!( + settle_self_update(&mut session, &MockFloor::new()), + Err(SettleError::Session(SelfSessionFault)) + ); +} + +// A floor that does not answer fails secure in the same way. The session +// stays confirmed and the next boot tries again. +#[test] +fn settle_with_an_unreadable_floor_fails_secure() { + let committed = SelfUpdateState::Committed { svn: Svn(7) }; + let mut session = MockSelfUpdate::with_session(committed, RunningImage::Trial); + + assert_eq!( + settle_self_update(&mut session, &MockFloor::faulting()), + Err(SettleError::Floor(FloorFaultInjected)) + ); + + assert_eq!(session.state, committed); +} + +// A trial image is running that no session claims. The session is +// reverted, which clears the pending mark, so the confirmed image runs +// after a reset. +#[test] +fn settle_reports_an_unclaimed_image_and_reverts_the_session() { + let mut session = MockSelfUpdate::with_session(SelfUpdateState::Idle, RunningImage::Trial); + + assert_eq!( + settle_self_update(&mut session, &MockFloor::new()), + Ok(SelfUpdateSettlement::UnclaimedImageRunning) + ); + + assert_eq!(session.state, SelfUpdateState::Idle); +}