From 56831645073d2760ed3d2965c28d78d32febc6d1 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Sat, 24 Oct 2026 17:16:30 -0700 Subject: [PATCH] nvx-rs: give a clone's failures documented statuses Add nvx::protocol::status, the exit statuses with which a clone's guest ends its VM once it fails fast, and from which the host classifies a clone's end: the time ABI's 193, 194 and 195, which keep their meanings, one status for each other class of a clone's repair's failures, 40 to 45, and one reserved for each class of a clone's activation's failures, 50 to 58. Add the time ABI's strict mode, TimeAbi::strict, for a caller that owns the VM's exit status and its shutdown: a violation that stops the VM in enforcing mode fails the call instead, after its event, with the Violation, whose status is the one with which enforcing mode stops the VM. The command tools keep enforcing mode, and so do the time daemon and a capture's preparation. nvx::guest::clone takes the time ABI in strict mode for the capture and the repair. A RepairError reports the repair's RepairProgress, the memory blocks that it added and onlined, its reseed, and its finish, and selects its status, RepairError::status. The runtime adapter's guest repairs a clone that a restore of its template's snapshot started, behind its closed fence, and ends with 31 once repaired, since it does not activate a clone yet, or with the status of the repair's failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: fbebd4aa-2ad7-427d-80ea-49358a645e9a --- nvx-rs/README.md | 5 +- nvx-rs/src/guest/clone.rs | 75 +++-- nvx-rs/src/guest/clone/repair.rs | 303 +++++++++++++---- nvx-rs/src/guest/clone/repair/tests.rs | 234 +++++++++++++ nvx-rs/src/guest/clone/tests.rs | 8 + nvx-rs/src/guest/time/api.rs | 313 +++++++++++++++--- nvx-rs/src/guest/time/mod.rs | 7 + nvx-rs/src/guest/time/system.rs | 142 +++++++- nvx-rs/src/lib.rs | 9 +- nvx-rs/src/protocol/mod.rs | 3 + nvx-rs/src/protocol/status.rs | 184 ++++++++++ tests/runtime-adapter/README.md | 54 ++- .../src/bin/runtime-adapter-guest.rs | 15 +- tests/runtime-adapter/src/template.rs | 22 +- tests/runtime-adapter/src/worker/capture.rs | 131 ++++++-- 15 files changed, 1310 insertions(+), 195 deletions(-) create mode 100644 nvx-rs/src/protocol/status.rs diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 9ff1e576c..1a6ef325b 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -85,6 +85,7 @@ on Linux and Windows. | `nvx::protocol::nvxs` | The NVXS session, independent of `NVXC` and of any other application protocol: `RecordType`, `Record`, `encode_record`, and `RecordParser`, which parses records incrementally across partial reads with buffers bounded by one record; `HostSession` and `GuestSession`, the state machines of the host's and the guest's roles, which validate every record's type, identity, and sequence number, and the guest's receive window and credits, without I/O. An application message may span DATA records, and the application's own framing reassembles it | | `nvx::protocol::root` | The boot tokens of a layered root, which the host renders and guests parse by the same rules: a `RootLayout` is a `RootImage` on the `distro` block device, an EROFS filesystem or a GPT disk with the `nvx::uuid::Uuid` that the guest expects of it, if any, over a `RootUpper`, the ext4 scratch disk or RAM. `RootLayout::tokens` renders `nvx_layer=distro,0xd0003000,FORMAT[:IDENTITY]` and `nvx_scratch=0xd0006000,ext4` or `nvx_overlay_upper=ramfs`, and `RootLayout::parse` reads them from a kernel command line, and refuses malformed, repeated, and conflicting tokens and the earlier form without a format, with no fallback that probes the devices. It reads the virtio-mmio devices of the command line with `nvx::protocol::cmdline`, to refuse a RAM upper beside one at the scratch base. `tests/fixtures/root-tokens.json` holds its cases, which the tests of the guest's shell init replay too | | `nvx::protocol::share` | The guest targets that a sandbox's live shares may not take, which the host's `sandbox::mount` and the guest's `root` both refuse: `RESERVED_MOUNT_TARGETS`, the trees where the container runtime mounts or binds, `RESERVED_EXACT_MOUNT_TARGETS`, the directories that it creates, `RESERVED_TARGET_CHARACTERS`, and `reservation`, which tells why a target is reserved | +| `nvx::protocol::status` | The exit statuses with which a clone's guest ends its VM once it fails fast, from which the host classifies a clone's end, with the diagnostic that precedes each: the time ABI's own, `TIME_CONFORMANCE`, 193, `TIME_VIOLATION`, 194, and `TIME_REPAIR`, 195, which keep their meanings; one for each other class of a clone's repair's failures, `REPAIR_RESTORE` to `REPAIR_FINISH`, 40 to 45, which `nvx::guest::clone::RepairError::status` selects; and one reserved for each class of a clone's activation's failures, `ACTIVATION_DEADLINE` to `ACTIVATION_ORDER`, 50 to 58. `describe` tells what a status means | | `nvx::activation` | The activation record of a layered root: one bounded, versioned, non-secret record of the settings that its activation applies, which a clone restored from a template reads in place of its captured configuration. An `ActivationRecord` holds the schema's `VERSION`, a clone's `RestoreSection`, the restore's `TransactionId` and the `TemplateReference` of its template, the template's `StoreGeneration`, a nonzero publication ID that its store assigns in order, and the `Identity` of its key, and the `Configuration`: the `Image`, EROFS or GPT, with the `Identity` of its binding and the UUID or disk GUID that the guest expects; the `Upper`, a tmpfs or the scratch with its binding's `Identity` and its `Journal` policy; the `OverlayMode`; an `Export`, a directory or an aggregate with its plan's `Identity`, and its 1 to 4,096 mappings; the `Network`, its policy's `Identity`, a DNS address, the listeners' `Publication`, and a proxy endpoint without credentials; the `WorkloadView`, its hostname, `MachineIdMode`, and `WorkloadIdentity`, whose IDs are neither 0 nor `u32::MAX`; and the `ExpectedResources`. The other inputs of the root's assembly are invariants of the template. `ActivationRecord::new` refuses a configuration that `Configuration::check` finds past a public limit, `MAX_MAPPINGS`, `MAX_HOSTNAME_BYTES`, or `MAX_PROXY_ENDPOINT_BYTES`, which keep every record within `MAX_RECORD_BYTES`, or with an export without mappings, mappings without an export, a UID or GID of 0 or `u32::MAX`, a proxy endpoint with credentials, or a zero size. `ActivationRecord::encode` writes a record's canonical encoding, its version, then a tagged, length-prefixed field for each part in the order of their tags, 602 bytes at most, and `ActivationRecord::decode` reads exactly those bytes back and refuses any others, naming the `Field` at fault. `ReadyActivation` is what Ready reports once the activation applied: a clone's transaction ID, which `ReadyActivation::of` takes from a record, and nothing more for a cold boot, in a canonical encoding of `COLD_LEN` or `CLONE_LEN` bytes, 1 or 17, that `ReadyActivation::encode` writes and `ReadyActivation::decode` reads. The records and both encodings need no features | | `nvx::contract` | The OpenVMM microVM ABI, boot contract, broker protocol, and control-contract revision that the crate speaks, and when each changes, and `manifest_is_compatible` for a release's `SOURCE-MANIFEST.json`; and the versions of OpenVMM's snapshot format, saved device state, time ABI, and restore packet, which a template's key holds and which an ignored test of `tests/openvmm_vectors.rs` checks against the OpenVMM submodule | | `nvx::net` | `Cidr`: IPv4 and IPv6 networks and their subtraction, which `Cidr::parse` reads without host bits and `Cidr::parse_masked` as OpenVMM reads the destination of an egress rule; `is_valid_hostname` and `is_valid_guest_network`, the forms of a VM's hostname and guest address, and `guest_gateway`, the gateway of a guest address | @@ -118,8 +119,8 @@ on Linux and Windows. | `nvx::guest::cgroup` | `Layout`: the agent's cgroup and the workload cgroup, which the caller names, and which may not hold the agent's. `ExecutionCgroup`: the cgroup of one execution, the workload cgroup itself or a child of it, which it prepares with the caller's resource values, places processes in, checks for occupancy through `pids.current`, kills with `cgroup.kill`, and settles, over the execution's cgroup or the whole workload subtree, as the caller's `Scope` chooses. It holds its cgroups' directories open, and the process's removals and preparations of cgroups take turns, so that it never reaches another cgroup that takes one of their paths. Settling calls the caller's status collector between its checks, and collects no status itself | | `nvx::guest::exec` | `Execution`: drives one program that the caller spawned with the program's ends of its `Pipes`, or that the caller started itself and adopts with its own ends of the program's pipes and a `Child` that names it, without blocking, from the caller's own loop or reactor, and reports each step as an `Event`: the launch report, which an adopted program has at once, the output of each stream, kill failures, and the `Outcome`. It reads a stream only as far as the caller's `grant` allows, so a stalled sink stalls its own execution alone, writes the program's standard input without `SIGPIPE`, and ends the program on a cancellation, a timeout, or a disconnection, with `SIGKILL` through the program's pidfd to the program and, if the caller asks, its process group, while no collector has taken its status, and, through the opened directory of a cgroup that the caller names, to that cgroup. It closes the program's input once the program has ended. It waits for its program alone, by a pidfd, or takes the status from the caller's collector with `observe`; with `Options::caller_reaps`, that collector is the program's only one. NVX's managed agent is one caller | | `nvx::guest::exec::binding` | `Binding`: an `nvx::exec::Runner` bound to an `Execution`, the glue that a guest adapter would otherwise write. The runner's read allowances become grants, its discards discards, the input that it accepted writes and then a close, its terminations a cancellation, a disconnection, or a termination, and its settlement a kill of the execution's cgroup and checks until no process occupies it; without a cgroup, nothing proves that the program's descendants are gone, so the execution is fenced. It feeds the execution's events back to the runner, statuses that a caller who reaps hands over with `observe` included, and never blocks: the caller polls its descriptors and its input writer until its deadline, and each call takes a bounded number of steps, so that no execution holds the caller's loop | -| `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. The request's mode selects how a restore hands over its fresh entropy: the file mode, which `nvx-time` keeps, writes it to a file, and the in-memory mode returns it as the `Restore`'s owned `Seed`, and writes no file. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | -| `nvx::guest::clone` | x86_64: the preparation of a template's guest for a capture before its root's assembly, and the repair of each clone that a restore of its snapshot starts, for a guest init or agent, whether or not it is PID 1. `CapturePreparation`, of the caller's `AdmissionFence` and the `Devices` that the template's clones replace, the `distro` and scratch block devices, which their MMIO windows name, and the host filesystem device's virtio-fs tag: `prepare` closes the fence, so that no execution starts, refuses with a `Mounted` the first mount from those devices, a distro partition included, or from an overlay, bind mounts included, in a bounded read of `/proc/self/mountinfo` of at most `MOUNT_TABLE_LIMIT` bytes, then waits for the time ABI's boot checks, runs its capture checks, and suppresses the stall detectors, as `nvx-time pre-capture` does, and returns an owned `CaptureBarrier`. `CaptureBarrier::capture` requests the capture with the platform tier's request, in the in-memory mode, through `TimeAbi::capture`, and returns `Captured::Continued` once it opened the fence and restored the detectors' settings, or, in a clone, `Captured::Restored`, with the fence still closed, since nothing else may run before repair acknowledges the restore, and the detectors suppressed until the time daemon restores their settings: `Restored::into_parts` hands the `Restore` that repair takes next, which holds the restore's `Seed`, and the closed fence to the caller, which keeps it closed through the repair and opens it once the clone may admit executions, such as once it is activated. `CaptureBarrier::cancel`, a dropped barrier, and a failure to read the VM generation ID open the fence and restore the detectors' settings; a capture request that fails restores the settings but leaves the fence closed, since it may fail in a clone whose restore it could not repair. Each failure is a `CaptureError`, with its `CaptureStage` and `CaptureCause`. `repair`, or a `Repair` that the caller may retry, repairs the clone of a `Restore` next: it checks that the online CPUs are a prefix that meets the restore's processor target, since it brings no CPU online; with a memory target, it checks, before it adds any memory block, that the kernel's block size is 128 MiB and that the restore's ranges continue the captured RAM in logical order, with no gap, overlap, or range in the MMIO aperture, while the host bounds the target by the snapshot's capacity, then adds and onlines the blocks through sysfs, as `nvx-snapshot` does, and verifies them; it reseeds the kernel's random number generator with the `Seed`, having drawn no randomness before, and drops it; and it finishes the restore through `TimeAbi::restore_finish`, with the acknowledgement when the host gates its input until then, which hands the deferred time work to the time daemon that the caller supervises. It returns the `Repaired` clone, with its generation ID, its online CPUs and memory, and when it acknowledged the restore, and never touches the fence. It adds no memory block twice, and reseeds and finishes at most once, so a failure at or after the reseed is final. Each failure is a `RepairError`, with its `RepairStage` and `RepairCause` | +| `nvx::guest::time` | x86_64: `TimeAbi`, the steps of NVX time ABI v1 as typed calls in the caller's process, for any guest init or agent: the cold-boot time step and, separately, the start of the time daemon in a child process that the caller supervises, the VM generation ID, a `CaptureRequest` that repairs a `Restore`, and a `RestoreFinish`. The request's mode selects how a restore hands over its fresh entropy: the file mode, which `nvx-time` keeps, writes it to a file, and the in-memory mode returns it as the `Restore`'s owned `Seed`, and writes no file. A violation of the ABI stops the VM in enforcing mode, with the time ABI's status, as `nvx-time` does; in strict mode, `TimeAbi::strict`, which a caller that owns the VM's exit status and shutdown selects, the call fails with the `Violation`, whose `status` is the one with which enforcing mode stops the VM, and the time daemon and a capture's preparation enforce the ABI all the same. A call takes the permission for OpenVMM's ports on the calling thread for each port transaction alone, and fails with `TimeError::PortRelease` if the thread may still hold some of it, which a process that it started would inherit, and from which `nvx::guest::spawn` starts none. No call waits for a child that it did not start. `nvx-time` runs its steps through them | +| `nvx::guest::clone` | x86_64: the preparation of a template's guest for a capture before its root's assembly, and the repair of each clone that a restore of its snapshot starts, for a guest init or agent, whether or not it is PID 1. `CapturePreparation`, of the caller's `AdmissionFence` and the `Devices` that the template's clones replace, the `distro` and scratch block devices, which their MMIO windows name, and the host filesystem device's virtio-fs tag: `prepare` closes the fence, so that no execution starts, refuses with a `Mounted` the first mount from those devices, a distro partition included, or from an overlay, bind mounts included, in a bounded read of `/proc/self/mountinfo` of at most `MOUNT_TABLE_LIMIT` bytes, then waits for the time ABI's boot checks, runs its capture checks, and suppresses the stall detectors, as `nvx-time pre-capture` does, and returns an owned `CaptureBarrier`. `CaptureBarrier::capture` requests the capture with the platform tier's request, in the in-memory mode, through `TimeAbi::capture`, and returns `Captured::Continued` once it opened the fence and restored the detectors' settings, or, in a clone, `Captured::Restored`, with the fence still closed, since nothing else may run before repair acknowledges the restore, and the detectors suppressed until the time daemon restores their settings: `Restored::into_parts` hands the `Restore` that repair takes next, which holds the restore's `Seed`, and the closed fence to the caller, which keeps it closed through the repair and opens it once the clone may admit executions, such as once it is activated. `CaptureBarrier::cancel`, a dropped barrier, and a failure to read the VM generation ID open the fence and restore the detectors' settings; a capture request that fails restores the settings but leaves the fence closed, since it may fail in a clone whose restore it could not repair. Each failure is a `CaptureError`, with its `CaptureStage` and `CaptureCause`. `repair`, or a `Repair` that the caller may retry, repairs the clone of a `Restore` next: it checks that the online CPUs are a prefix that meets the restore's processor target, since it brings no CPU online; with a memory target, it checks, before it adds any memory block, that the kernel's block size is 128 MiB and that the restore's ranges continue the captured RAM in logical order, with no gap, overlap, or range in the MMIO aperture, while the host bounds the target by the snapshot's capacity, then adds and onlines the blocks through sysfs, as `nvx-snapshot` does, and verifies them; it reseeds the kernel's random number generator with the `Seed`, having drawn no randomness before, and drops it; and it finishes the restore through `TimeAbi::restore_finish`, with the acknowledgement when the host gates its input until then, which hands the deferred time work to the time daemon that the caller supervises. It returns the `Repaired` clone, with its generation ID, its online CPUs and memory, and when it acknowledged the restore, and never touches the fence. It adds no memory block twice, and reseeds and finishes at most once, so a failure at or after the reseed is final. Each failure is a `RepairError`, with its `RepairStage`, its `RepairCause`, and the repair's `RepairProgress`, and `RepairError::status` selects the status, of `nvx::protocol::status`, with which the caller ends the VM. The capture and the repair take the time ABI in strict mode, so that a violation of it fails them, with its status, rather than stopping the VM, and only the capture's preparation and the time daemon enforce it | | `nvx::guest::reseed` | `Seed`, `Seed::read`, and `reseed`: a restore's fresh entropy, which starts with its `GenerationId`, and the reseed of the kernel's random number generator with it, in the caller's process. A `Seed` keeps its bytes in one allocation, which no move copies, zeroes them with volatile writes when it is dropped, can't be cloned, and formats none of them. `nvx-reseed` runs them | | `nvx::guest::root` | The operations of a layered root, for a guest init or agent, whether or not it is PID 1. `bootstrap`: the agent's and the workload's sibling cgroups of a `Layout`, with every process of their parent moved into the agent's, the agent's `memory.low`, and the workload cgroup prepared through `ExecutionCgroup::prepare`, with its `memory.max` and `pids.max`, and user namespaces kept unavailable. `assemble`: the `Assembly`'s block devices, found by their MMIO windows, checked against their roles' access modes and their expected sizes, and flushed, and the `Root` that it mounts, at a runtime directory, a canonical path that no symbolic link leads through, that it holds exclusively until its teardown, where nothing is mounted already, once it has checked everything: the read-only layers of the `Image` that the caller names, which it never probes: an EROFS image, whose expected `nvx::uuid::Uuid` it checks in its superblock, or a GPT disk whose partition table it reads and checks with `nvx::guest::gpt::read_disk`, against the caller's `nvx::gpt::Expected` values, whose first partition it reads, up to the caller's bound, as opaque bytes that it never mounts, and whose later partitions, at least one, hold ext4 layers that it mounts `ro,nosuid,nodev` with `errors=remount-ro`, so that an error that the kernel finds in a layer fails the operation that found it, rather than panic the guest, as the layer's superblock may ask, from the block devices that the kernel created when it found the disk, once their superblocks show that no journal needs a recovery, which their read-only device can't take, partition 2's at the overlay's base and the last partition's at its top, under an overlay mounted `nosuid,nodev` with the one option set of the caller's `OverlayMode`, `metacopy=on,xino=on` or, for lowers that may carry untrusted overlay xattrs, `metacopy=off,redirect_dir=nofollow`, with no fallback, whose upper and work directories are on the `Upper`: an ext4 scratch mounted `nosuid,nodev`, whose superblock it checks for a journal that the caller's `Journal` policy forbids, or a tmpfs mounted `nosuid,nodev` at the kernel's default size. It refuses overlay options longer than `MOUNT_OPTIONS_LIMIT`, the most that a mount passes, which, below `/run/nvx`, fit up to 195 layers of a GPT disk, and an overlay that the kernel's mount table shows read-only or without the mode's options in effect, since the kernel mounts one so, rather than fail, when its upper can't hold its work directory or its extended attributes. The `Root` reports the first partition's bytes, its `Lower` layers, its `UpperKind`, and its `OverlayMode`. `Root::mount_share` and `Root::mount_aggregate`: virtio-fs live shares at targets in the root that it checks, against the targets that `protocol::share` reserves among others, with one aggregate per root, which it attaches through its `mappings::Mapper` beneath the overlay, with NVX's container root's reserved targets: a share's device mounted out of the root, then moved onto its target, and a copy of each child's entry of the aggregate. `Root::mount_export`: the host export, the aggregate device of `protocol::export`, one per root, mounted `nosuid,nodev` with the access of its `ExportMode` at `hostfs/root` in the runtime directory, out of the root, below a directory that only root may enter, whose root it returns as the `SourceRoot` of the caller's mappings. `Root::map`: a caller's host mappings, through the same mapper. A share, a child, the export, or a mapping that fails latches the root. `Root::teardown`: the shares and mappings in reverse order, then the mounts of an aggregate and of the export, the later first, the overlay, the layers from the top down, and the upper, with every failed unmount reported. Each failure is a typed `Error`. The operations collect no child's status, change neither the caller's root nor its credentials, start no async runtime, and never power the VM off | | `nvx::guest::gpt` | `read_disk`: the partition table of a GPT disk, read and checked with `nvx::gpt`'s `Disk::read` through the disk's block device, which the caller opened, at the logical block size and size that `BLKSSZGET` and `BLKGETSIZE64` report, with the caller's `Expected` values, such as the disk GUID that a boot parameter names. It only reads the device, at explicit offsets, so it never moves the file's position. It neither compares the partitions that the kernel created with the table nor asks the kernel to reread the table. Each failure is a typed `DeviceError` | diff --git a/nvx-rs/src/guest/clone.rs b/nvx-rs/src/guest/clone.rs index bc78dbce0..ea27f73a8 100644 --- a/nvx-rs/src/guest/clone.rs +++ b/nvx-rs/src/guest/clone.rs @@ -71,18 +71,34 @@ //! the host gates its input until then, which hands the deferred time work to the time //! daemon: the restore checks, the stall detectors' grace period and release, and the //! wall-clock discipline's fast cadence. The caller starts the daemon with -//! [`TimeAbi::start_daemon`] at boot, before the capture, and supervises it. In enforcing -//! mode, a finish without a running daemon stops the VM, and in report-only mode, it runs the -//! work itself before it returns. The work may end after the clone is ready, and its failures -//! are fatal repair failures: the daemon reports them as violations of the time ABI, which -//! stop the VM in enforcing mode. +//! [`TimeAbi::start_daemon`] at boot, before the capture, and supervises it. A finish without +//! a running daemon fails with a violation of the time ABI's check C10, and in report-only +//! mode, it runs the work itself before it returns. The work may end after the clone is +//! ready, and its failures are fatal repair failures: the daemon reports them as violations +//! of the time ABI, which stop the VM, with the time ABI's status, unless the ABI is +//! report-only, since the daemon enforces it. //! //! It returns the [`Repaired`] clone: its new VM generation ID, its number of online CPUs, the //! size of its online memory blocks, and when it acknowledged the restore, which an activation //! checks against the resources that it expects. The repair records how far it got, so that it //! never adds a memory block twice, and runs the reseed and the finish at most once, whoever //! retries it: a failure at or after the reseed is final. It never touches the fence. A failure is -//! a [`RepairError`], which names the [`RepairStage`] that failed and its [`RepairCause`]. +//! a [`RepairError`], which names the [`RepairStage`] that failed, its [`RepairCause`], and the +//! repair's [`RepairProgress`], and selects the status with which the caller ends the VM, +//! [`RepairError::status`], which `nvx::protocol::status` documents. +//! +//! # Strict mode +//! +//! The capture and the repair take the time ABI in strict mode, [`TimeAbi::strict`], unless the +//! kernel command line makes it report-only: a violation of the ABI fails them, after its +//! violation event, with [`TimeError::Violation`], whose +//! [`status`](crate::guest::time::Violation::status) is the one with which the ABI would have +//! stopped the VM, rather than stopping the VM, so that the caller owns the VM's exit status and +//! its shutdown. A capture request fails so in a clone whose restore the time ABI could not +//! repair, with 195, as a [`CaptureStage::Capture`] error, and a repair, with 195 or 193, as +//! [`RepairError::status`] tells. The capture's preparation enforces the ABI, as +//! `nvx-time pre-capture` does, and so does the time daemon: a violation of either stops the VM, +//! unless the ABI is report-only. //! //! # Example //! @@ -128,7 +144,14 @@ //! // A clone continues here, with its wall clock repaired. Its repair activates its //! // memory, reseeds it, and acknowledges the restore, while its fence stays closed. //! let (restore, fence) = restored.into_parts(); -//! let repaired = clone::repair(restore)?; +//! let repaired = match clone::repair(restore) { +//! Ok(repaired) => repaired, +//! Err(error) => { +//! // The agent ends the VM with the failure's status, after an error line. +//! eprintln!("{error} ({})", error.progress()); +//! std::process::exit(error.status().into()); +//! } +//! }; //! println!("{} CPUs, {} bytes of memory", repaired.cpus(), repaired.memory()); //! // The clone opens the fence once it may admit executions, such as once activated. //! fence.open(); @@ -147,9 +170,10 @@ //! its memory interface. They mount nothing and start no thread, and they start no process and //! collect no child's status, except a report-only finish without a running time daemon, which //! runs the deferred time work itself, in a child that it waits for, as -//! [`TimeAbi::restore_finish`] tells. Apart from a violation of the time ABI in enforcing mode, -//! which stops the VM as [`TimeAbi`] tells, they never power the VM off, so a caller that is not -//! PID 1 keeps its own reaping, transport, and shutdown. +//! [`TimeAbi::restore_finish`] tells. Apart from a violation of the time ABI in the capture's +//! preparation, which stops the VM unless the ABI is report-only, as [strict +//! mode](#strict-mode) tells, they never power the VM off, so a caller that is not PID 1 keeps +//! its own reaping, transport, and shutdown. mod mounts; mod repair; @@ -172,7 +196,7 @@ use super::time::{ pub use self::mounts::{Mounted, Origin}; pub use self::repair::{ - RangeFault, Repair, RepairCause, RepairError, RepairStage, Repaired, repair, + RangeFault, Repair, RepairCause, RepairError, RepairProgress, RepairStage, Repaired, repair, }; /// The most bytes of the mount table that a [`CapturePreparation`] reads: 1 MiB, the lines of @@ -257,8 +281,9 @@ impl CapturePreparation { /// suppresses the stall detectors, then returns the barrier that holds them. /// /// The wait for the boot checks lasts 30 seconds at most. A capture check that fails reports - /// a violation of the time ABI, which stops the VM in enforcing mode; in report-only mode, - /// the preparation goes on, as `nvx-time pre-capture` does. A preparation that fails after it + /// a violation of the time ABI, which stops the VM unless the ABI is report-only, since the + /// preparation enforces it even in [strict mode](self#strict-mode); in report-only mode, the + /// preparation goes on, as `nvx-time pre-capture` does. A preparation that fails after it /// closed the fence opens it again, and leaves the detectors' settings as they were: when it /// suppressed some already, it restores their settings, and a failure to restore them is a /// [`CaptureStage::Release`] error. One after which the calling thread may still hold some of @@ -267,7 +292,7 @@ impl CapturePreparation { pub fn prepare(self) -> Result, CaptureError> { let system = LiveSystem { paths: Paths::default(), - abi: TimeAbi::new(), + abi: live_abi(), }; self.prepare_with(Box::new(system)) } @@ -365,10 +390,12 @@ impl CaptureBarrier { /// A failure to read the VM generation ID, before the request, opens the fence and restores /// the detectors' settings, as [`cancel`](Self::cancel) does. A request that fails restores /// the detectors' settings, but leaves the fence closed, since it may fail in a clone whose - /// restore it could not repair, where no execution may start. Either way, the caller ends - /// the VM, and a failure to restore the settings is a [`CaptureStage::Release`] error, unless - /// the calling thread may still hold some of the permission for the time ABI's ports, which - /// [`TimeError::PortRelease`] reports instead. + /// restore it could not repair, where no execution may start: there, the time ABI, in + /// [strict mode](self#strict-mode), fails it with the violation, [`TimeError::Violation`], + /// rather than stopping the VM. Either way, the caller ends the VM, with the violation's + /// status if it has one, and a failure to restore the settings is a [`CaptureStage::Release`] + /// error, unless the calling thread may still hold some of the permission for the time ABI's + /// ports, which [`TimeError::PortRelease`] reports instead. pub fn capture(mut self) -> Result, CaptureError> { let generation_id = match self.system.generation_id() { Ok(generation_id) => generation_id, @@ -578,8 +605,8 @@ pub enum CaptureCause { /// The stall detectors' settings are saved already, by a capture that is still pending, which /// restores them. Pending, - /// A call of the time ABI failed: a violation that a report-only call reports, or a failure of - /// its ports, its time state, or its entropy file. + /// A call of the time ABI failed: a violation that a strict or report-only call reports, or a + /// failure of its ports, its time state, or its entropy file. Time(TimeError), } @@ -675,10 +702,16 @@ trait System: Send { struct LiveSystem { /// The directories of the kernel's interfaces. paths: Paths, - /// The time ABI, in the mode that the kernel command line selects. + /// The time ABI of [`live_abi`], whose preparation of a capture enforces the ABI all the same. abi: TimeAbi, } +/// Returns the time ABI of the capture and of the repair: in strict mode, unless the kernel +/// command line makes it report-only. +fn live_abi() -> TimeAbi { + TimeAbi::new().strict() +} + impl System for LiveSystem { fn mount_table(&mut self) -> io::Result> { let path = self.paths.proc.join(MOUNT_TABLE); diff --git a/nvx-rs/src/guest/clone/repair.rs b/nvx-rs/src/guest/clone/repair.rs index 006071d5f..8b32eb67d 100644 --- a/nvx-rs/src/guest/clone/repair.rs +++ b/nvx-rs/src/guest/clone/repair.rs @@ -4,9 +4,11 @@ #[cfg(test)] mod tests; +use std::collections::BTreeMap; use std::fmt; use std::fs::{self, OpenOptions}; use std::io::{self, Write}; +use std::mem; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -19,6 +21,7 @@ use crate::guest::reseed::{self, ReseedError, Seed}; use crate::guest::root::Paths; use crate::guest::sys; use crate::guest::time::{MemoryRange, Restore, RestoreFinish, TimeAbi, TimeError}; +use crate::protocol::status; /// The memory block size of the snapshot ABI, 128 MiB, to which a restore's memory ranges are /// aligned. @@ -63,21 +66,28 @@ pub fn repair(restore: Restore) -> Result { /// /// - Until the memory target is active, a failed attempt may be retried, and the next one /// repeats the checks and adds or onlines only the memory blocks that are not online yet. The -/// first attempt derives the end of the RAM that the clone captured, before it adds any block, -/// and every later one keeps it. +/// first attempt derives the end of the RAM that the clone captured, before it adds any memory +/// block, and every later one keeps it. /// - The reseed and the finish run at most once. A failure of either is final: every later call /// returns its error again. /// - Once the repair completed, every later call returns the same [`Repaired`]. /// -/// The repair never touches the caller's fence of workload admission, which stays closed: the -/// caller opens it when the clone may admit executions, such as once it is activated. +/// The repair takes the time ABI in strict mode, [`TimeAbi::strict`], so that a violation of the +/// ABI fails it, with the violation's status, instead of stopping the VM: it never powers the VM +/// off, and the caller ends the VM with [`RepairError::status`]. The repair never touches the +/// caller's fence of workload admission, which stays closed: the caller opens it when the clone +/// may admit executions, such as once it is activated. pub struct Repair { /// The restore that the clone's capture returned, without its seed. restore: Restore, /// The restore's fresh entropy, until the reseed drops it. seed: Option, - /// How far the repair got. - progress: Progress, + /// Where the repair resumes. + state: State, + /// The memory blocks that the repair added and onlined, across its attempts. + activations: Activations, + /// How far the repair got, which its errors report. + progress: RepairProgress, /// The kernel and the time ABI. system: Box, } @@ -90,7 +100,7 @@ impl Repair { pub fn new(restore: Restore) -> Self { let system = LiveSystem { paths: Paths::default(), - abi: TimeAbi::new(), + abi: super::live_abi(), }; Self::with_system(restore, Box::new(system)) } @@ -101,7 +111,9 @@ impl Repair { Self { restore, seed, - progress: Progress::Memory { captured: None }, + state: State::Memory { captured: None }, + activations: Activations::default(), + progress: RepairProgress::default(), system, } } @@ -109,32 +121,41 @@ impl Repair { /// Runs the repair from where it got, as the [module](super) describes, and returns the /// repaired clone. /// - /// A failure names its [`RepairStage`] and its [`RepairCause`]. One before the reseed may be - /// retried, and changes nothing that a retry undoes: a refused restore or memory target adds - /// no memory block, and a failed activation leaves the blocks that it added and onlined, - /// which a retry counts as done. The reseed and the finish run at most once, so a failure of - /// either, or of a finish that never returned, such as after a panic, is final, and every - /// later call returns that error again. A failed finish may have acknowledged the restore - /// already. Once the repair completed, every later call returns the same [`Repaired`], - /// without another step. + /// A failure names its [`RepairStage`], its [`RepairCause`], and the repair's + /// [`RepairProgress`]. One before the reseed may be retried, and changes nothing that a retry + /// undoes: a refused restore or memory target adds no memory block, and a failed activation + /// leaves the blocks that it added and onlined, which a retry counts as done. The reseed and + /// the finish run at most once, so a failure of either, or of a finish that never returned, + /// such as after a panic, is final, and every later call returns that error again. A failed + /// finish may have acknowledged the restore already. Once the repair completed, every later + /// call returns the same [`Repaired`], without another step. pub fn run(&mut self) -> Result { + self.resume().map_err(|error| RepairError { + progress: self.progress, + ..error + }) + } + + /// Runs the repair from where it got, as [`run`](Self::run) does, with errors that do not + /// tell its progress yet. + fn resume(&mut self) -> Result { loop { - match &self.progress { - Progress::Memory { .. } => { + match &self.state { + State::Memory { .. } => { let resources = self.activate()?; - self.progress = Progress::Reseed(resources); + self.state = State::Reseed(resources); } - Progress::Reseed(resources) => { + State::Reseed(resources) => { let resources = *resources; if let Err(error) = self.reseed() { - self.progress = Progress::Failed(error.clone()); + self.state = State::Failed(error.clone()); return Err(error); } - self.progress = Progress::Finish(resources); + self.state = State::Finish(resources); } - Progress::Finish(resources) => return self.finish(*resources), - Progress::Repaired(repaired) => return Ok(*repaired), - Progress::Failed(error) => return Err(error.clone()), + State::Finish(resources) => return self.finish(*resources), + State::Repaired(repaired) => return Ok(*repaired), + State::Failed(error) => return Err(error.clone()), } } } @@ -194,14 +215,14 @@ impl Repair { /// any memory block, then adds each block of the ranges that the kernel lacks, onlines each /// that is not online, and verifies that it is. fn add_ranges(&mut self) -> Result<(), RepairCause> { - let captured = match self.progress { - Progress::Memory { + let captured = match self.state { + State::Memory { captured: Some(captured), } => captured, _ => { let blocks = self.system.memory_blocks().map_err(RepairCause::Io)?; let captured = captured_end(&blocks)?; - self.progress = Progress::Memory { + self.state = State::Memory { captured: Some(captured), }; captured @@ -211,7 +232,12 @@ impl Repair { for range in self.restore.memory_ranges() { let mut address = range.start(); while address < range.start() + range.length() { - activate_block(self.system.as_mut(), address)?; + activate_block( + self.system.as_mut(), + &mut self.activations, + &mut self.progress, + address, + )?; address += BLOCK_SIZE; } } @@ -240,7 +266,9 @@ impl Repair { let seed = self.seed.take().ok_or_else(|| fail(RepairCause::NoSeed))?; let result = self.system.reseed(&seed); drop(seed); - result.map_err(|error| fail(RepairCause::Reseed(error))) + result.map_err(|error| fail(RepairCause::Reseed(error)))?; + self.progress.reseeded = true; + Ok(()) } /// Finishes the restore, with the acknowledgement if the host gates its input until then, @@ -252,7 +280,8 @@ impl Repair { finish = finish.acknowledge(); } // The finish runs at most once, even when it never returns, such as after a panic. - self.progress = Progress::Failed(RepairError::new( + self.progress.finish_started = true; + self.state = State::Failed(RepairError::new( RepairStage::Finish, RepairCause::Incomplete, )); @@ -260,7 +289,7 @@ impl Repair { let acknowledged_at = acknowledge.then(Instant::now); if let Err(error) = self.system.restore_finish(finish) { let error = RepairError::new(RepairStage::Finish, RepairCause::Time(error)); - self.progress = Progress::Failed(error.clone()); + self.state = State::Failed(error.clone()); return Err(error); } let repaired = Repaired { @@ -269,7 +298,7 @@ impl Repair { memory: resources.memory, acknowledged_at, }; - self.progress = Progress::Repaired(repaired); + self.state = State::Repaired(repaired); Ok(repaired) } } @@ -279,14 +308,15 @@ impl fmt::Debug for Repair { formatter .debug_struct("Repair") .field("restore", &self.restore) + .field("state", &self.state) .field("progress", &self.progress) .finish_non_exhaustive() } } -/// How far a repair got, from which its next attempt resumes. +/// Where a repair resumes, which its next attempt starts from. #[derive(Debug)] -enum Progress { +enum State { /// The memory target is not active yet, which the next attempt activates. Memory { /// The logical end of the RAM that the clone captured, which the first attempt that @@ -312,6 +342,26 @@ struct Resources { memory: u64, } +/// The memory blocks that a repair added and onlined, across its attempts, each with whether the +/// repair found it since, present or online, so that its progress counts each block once, once it +/// found it: an attempt may fail between an addition or an onlining that the kernel accepted and +/// the read that finds the block, which a later attempt then makes. +#[derive(Debug, Default)] +struct Activations { + /// The blocks whose addition the kernel accepted, each with whether the repair found it since. + added: BTreeMap, + /// The blocks whose onlining the kernel accepted, each with whether the repair found it online + /// since. + onlined: BTreeMap, +} + +/// Records that the repair found `block`, if `log` holds it, and returns whether it had not +/// found it before. +fn first_found(log: &mut BTreeMap, block: u64) -> bool { + log.get_mut(&block) + .is_some_and(|found| !mem::replace(found, true)) +} + /// A clone that its [`Repair`] repaired: its new VM generation ID, the resources that it has, /// which an activation checks against the resources that it expects, and when its repair /// acknowledged its restore. @@ -356,6 +406,71 @@ impl Repaired { } } +/// How far a clone's repair got, across its attempts, which its [`RepairError`] reports: the +/// memory blocks that it added and brought online, whether it reseeded the kernel's random number +/// generator, and whether it started the finish, which may have acknowledged the restore. +/// +/// It counts each memory block once, once the repair found it present after its addition, or +/// online after its onlining. A block whose addition or onlining the kernel accepted, but that an +/// attempt could not find, such as when a read of its state failed, counts once a later attempt +/// finds it. +/// +/// Its `Display` form lists them, such as `added 2 memory blocks, onlined 2, reseeded, started +/// the finish`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct RepairProgress { + /// The memory blocks that the repair added, and found since. + added: u64, + /// The memory blocks that the repair onlined, and found online since. + onlined: u64, + /// Whether the repair reseeded the kernel's random number generator. + reseeded: bool, + /// Whether the repair started the finish. + finish_started: bool, +} + +impl RepairProgress { + /// Returns the number of memory blocks that the repair added through the kernel's memory + /// probe interface, and found present since. + pub fn added(&self) -> u64 { + self.added + } + + /// Returns the number of memory blocks that the repair brought online, and found online + /// since, besides those that the kernel brought online itself. + pub fn onlined(&self) -> u64 { + self.onlined + } + + /// Returns whether the repair reseeded the kernel's random number generator. + pub fn reseeded(&self) -> bool { + self.reseeded + } + + /// Returns whether the repair started the restore's finish, which may have acknowledged the + /// restore. + pub fn finish_started(&self) -> bool { + self.finish_started + } +} + +impl fmt::Display for RepairProgress { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "added {} memory blocks, onlined {}", + self.added, self.onlined + )?; + if self.reseeded { + formatter.write_str(", reseeded")?; + } + if self.finish_started { + formatter.write_str(", started the finish")?; + } + Ok(()) + } +} + /// A stage of a clone's repair, which a [`RepairError`] names. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] @@ -466,8 +581,8 @@ pub enum RepairCause { }, /// The kernel refused the reseed. Reseed(ReseedError), - /// The time ABI's finish failed: a violation that a report-only call reports, or a failure of - /// its ports or its time state. + /// The time ABI's finish failed: a violation that a strict or report-only call reports, whose + /// status the error takes, or a failure of its ports or its time state. Time(TimeError), /// An earlier attempt at the stage, which runs at most once, never returned, such as after a /// panic. @@ -512,25 +627,29 @@ impl fmt::Display for RepairCause { } } -/// Why a clone's repair failed: the stage that failed, and its cause. +/// Why a clone's repair failed: the stage that failed, its cause, and how far the repair got. /// -/// Its `Display` form names both, such as `failed to activate the memory target: memory range 0 -/// overlaps the RAM before it`. A clone keeps one error of a final failure, which every later -/// [`Repair::run`] returns again. +/// Its `Display` form names the stage and the cause, such as `failed to activate the memory +/// target: memory range 0 overlaps the RAM before it`. A clone keeps one error of a final +/// failure, which every later [`Repair::run`] returns again. The caller ends the clone's VM with +/// its [`status`](Self::status), after an error line that names it. #[derive(Clone, Debug)] pub struct RepairError { /// The stage that failed. stage: RepairStage, /// Why it failed. cause: Arc, + /// How far the repair got. + progress: RepairProgress, } impl RepairError { - /// Returns the failure of `stage`, because of `cause`. + /// Returns the failure of `stage`, because of `cause`, of a repair that did nothing yet. fn new(stage: RepairStage, cause: RepairCause) -> Self { Self { stage, cause: Arc::new(cause), + progress: RepairProgress::default(), } } @@ -543,6 +662,42 @@ impl RepairError { pub fn cause(&self) -> &RepairCause { &self.cause } + + /// Returns how far the repair got, across its attempts, when it failed. + pub fn progress(&self) -> RepairProgress { + self.progress + } + + /// Returns the exit status with which the clone's guest ends its VM for the failure, as + /// `nvx::protocol::status` documents it. A violation of the time ABI, a [`RepairCause::Time`] + /// of [`TimeError::Violation`], has the violation's + /// [`status`](crate::guest::time::Violation::status): 193 for a conformance check, or 195 + /// for the time ABI's part of the repair. Any other failure has its stage's status, from 40 + /// to 45: [`REPAIR_RESTORE`](status::REPAIR_RESTORE), + /// [`REPAIR_PROCESSORS`](status::REPAIR_PROCESSORS), + /// [`REPAIR_MEMORY_TARGET`](status::REPAIR_MEMORY_TARGET) for a memory target that the memory + /// stage refused before it added any memory block, [`REPAIR_MEMORY`](status::REPAIR_MEMORY) + /// for the memory stage's other failures, [`REPAIR_RESEED`](status::REPAIR_RESEED), and + /// [`REPAIR_FINISH`](status::REPAIR_FINISH). + pub fn status(&self) -> u8 { + if let RepairCause::Time(TimeError::Violation(violation)) = self.cause.as_ref() { + return violation.status(); + } + match (self.stage, self.cause.as_ref()) { + (RepairStage::Restore, _) => status::REPAIR_RESTORE, + (RepairStage::Processors, _) => status::REPAIR_PROCESSORS, + ( + RepairStage::Memory, + RepairCause::BlockSize(_) + | RepairCause::CapturedRam + | RepairCause::UntargetedRanges + | RepairCause::Range { .. }, + ) => status::REPAIR_MEMORY_TARGET, + (RepairStage::Memory, _) => status::REPAIR_MEMORY, + (RepairStage::Reseed, _) => status::REPAIR_RESEED, + (RepairStage::Finish, _) => status::REPAIR_FINISH, + } + } } impl fmt::Display for RepairError { @@ -642,35 +797,45 @@ fn check_ranges(ranges: &[MemoryRange], captured: u64) -> Result<(), RepairCause } /// Adds the memory block that starts at `address` through `system`, unless the kernel has it -/// already, onlines it, unless it is online, and verifies that it is. -fn activate_block(system: &mut dyn RepairSystem, address: u64) -> Result<(), RepairCause> { +/// already, onlines it, unless it is online, and verifies that it is. `activations` records the +/// additions and onlinings that the kernel accepted, across the repair's attempts, and `progress` +/// counts the block once, once the repair found it present after its own addition, and once, once +/// it found it online after its own onlining. +fn activate_block( + system: &mut dyn RepairSystem, + activations: &mut Activations, + progress: &mut RepairProgress, + address: u64, +) -> Result<(), RepairCause> { let block = address / BLOCK_SIZE; - let state = match system.block_state(block).map_err(RepairCause::Io)? { - Some(state) => state, - None => { - system.add_block(address).map_err(RepairCause::Io)?; - system - .block_state(block) - .map_err(RepairCause::Io)? - .ok_or(RepairCause::Missing(block))? + let mut state = system.block_state(block).map_err(RepairCause::Io)?; + if state.is_none() { + system.add_block(address).map_err(RepairCause::Io)?; + activations.added.entry(block).or_insert(false); + state = system.block_state(block).map_err(RepairCause::Io)?; + } + let state = state.ok_or(RepairCause::Missing(block))?; + if first_found(&mut activations.added, block) { + progress.added += 1; + } + if trimmed(&state) != ONLINE { + system.online_block(block).map_err(RepairCause::Io)?; + activations.onlined.entry(block).or_insert(false); + let state = system + .block_state(block) + .map_err(RepairCause::Io)? + .ok_or(RepairCause::Missing(block))?; + if trimmed(&state) != ONLINE { + return Err(RepairCause::Offline { + block, + state: lossy(&state), + }); } - }; - if trimmed(&state) == ONLINE { - return Ok(()); - } - system.online_block(block).map_err(RepairCause::Io)?; - let state = system - .block_state(block) - .map_err(RepairCause::Io)? - .ok_or(RepairCause::Missing(block))?; - if trimmed(&state) == ONLINE { - Ok(()) - } else { - Err(RepairCause::Offline { - block, - state: lossy(&state), - }) } + if first_found(&mut activations.onlined, block) { + progress.onlined += 1; + } + Ok(()) } /// Returns `text`, a value that the kernel writes in sysfs, without the newline that ends it. @@ -716,7 +881,7 @@ trait RepairSystem: Send { struct LiveSystem { /// The directories of the kernel's interfaces. paths: Paths, - /// The time ABI, in the mode that the kernel command line selects. + /// The time ABI, in strict mode unless the kernel command line makes it report-only. abi: TimeAbi, } diff --git a/nvx-rs/src/guest/clone/repair/tests.rs b/nvx-rs/src/guest/clone/repair/tests.rs index ee3217fba..ec3c3412e 100644 --- a/nvx-rs/src/guest/clone/repair/tests.rs +++ b/nvx-rs/src/guest/clone/repair/tests.rs @@ -12,6 +12,7 @@ use std::time::Duration; use super::*; use crate::guest::reseed::tests::take_dropped; use crate::guest::sys::Errno; +use crate::guest::time::Violation; /// The new VM generation ID of the clone. const RESTORED: GenerationId = GenerationId::from_bytes([8; GenerationId::LENGTH]); @@ -84,6 +85,11 @@ struct FakeSystem { add_failures: BTreeMap, /// The error numbers with which the next onlining of each block fails, once. online_failures: BTreeMap, + /// The error numbers with which a read of a block's state fails, once, by the block's number + /// and the read's index among the reads of its state, from 0. + read_failures: BTreeMap<(u64, usize), Errno>, + /// The number of reads of each block's state so far. + reads: BTreeMap, /// Whether an added block comes online by itself, as the kernel onlines added memory when its /// policy says so. auto_online: bool, @@ -93,6 +99,8 @@ struct FakeSystem { reseed_failure: Option, /// Whether the finish fails. finish_fails: bool, + /// The violation with which the finish fails, as the strict time ABI's finish does, if any. + finish_violation: Option, /// Whether the finish panics, as one that never returns would leave the repair. finish_panics: bool, } @@ -107,10 +115,13 @@ impl FakeSystem { blocks: (0..4).map(|block| (block, ONLINE_STATE)).collect(), add_failures: BTreeMap::new(), online_failures: BTreeMap::new(), + read_failures: BTreeMap::new(), + reads: BTreeMap::new(), auto_online: false, stuck: None, reseed_failure: None, finish_fails: false, + finish_violation: None, finish_panics: false, } } @@ -142,6 +153,12 @@ impl RepairSystem for FakeSystem { } fn block_state(&mut self, block: u64) -> io::Result>> { + let reads = self.reads.entry(block).or_default(); + let read = *reads; + *reads += 1; + if let Some(errno) = self.read_failures.remove(&(block, read)) { + return Err(errno.into()); + } Ok(self.blocks.get(&block).map(|state| state.to_vec())) } @@ -190,6 +207,9 @@ impl RepairSystem for FakeSystem { if self.finish_panics { panic!("the finish never returns"); } + if let Some(violation) = &self.finish_violation { + return Err(TimeError::Violation(violation.clone())); + } if self.finish_fails { Err(TimeError::StateUnavailable) } else { @@ -653,6 +673,220 @@ fn a_finish_that_never_returned_never_runs_again() { assert_eq!(calls.take(), Vec::::new()); } +/// Returns the error with which the repair of `restore` on `system` fails. +fn failure(restore: Restore, system: FakeSystem) -> RepairError { + repairing(restore, system).run().unwrap_err() +} + +#[test] +fn each_failure_class_ends_the_clone_with_a_status_of_its_own() { + let calls = Calls::default(); + let system = || FakeSystem::new(&calls); + let target = || restore().with_memory_target(&[(512 * MIB, 256 * MIB)]); + let mut cases = Vec::new(); + // The restore. + let unseeded = Restore::to(RESTORED).gated(); + cases.push(( + "no seed", + failure(unseeded, system()), + status::REPAIR_RESTORE, + )); + let finished = restore().finished_by_capture(); + cases.push(( + "finished", + failure(finished, system()), + status::REPAIR_RESTORE, + )); + // The processors. + let mut unlisted = system(); + unlisted.cpus = b"1-3\n"; + let error = failure(restore(), unlisted); + cases.push(("online CPUs", error, status::REPAIR_PROCESSORS)); + let error = failure(restore().with_processor_target(4), system()); + cases.push(("a processor target", error, status::REPAIR_PROCESSORS)); + // The memory target, which the repair refuses before it adds any memory block. + let mut wide = system(); + wide.block_size = b"40000000\n"; + let error = failure(target(), wide); + cases.push(("the block size", error, status::REPAIR_MEMORY_TARGET)); + let untargeted = restore().with_memory_ranges(&[(512 * MIB, 128 * MIB)]); + let error = failure(untargeted, system()); + cases.push(("untargeted ranges", error, status::REPAIR_MEMORY_TARGET)); + let mut unordered = system(); + unordered.blocks.remove(&2); + let error = failure(target(), unordered); + cases.push(("the captured RAM", error, status::REPAIR_MEMORY_TARGET)); + let overlap = restore().with_memory_target(&[(384 * MIB, 256 * MIB)]); + let error = failure(overlap, system()); + cases.push(("a range", error, status::REPAIR_MEMORY_TARGET)); + // The memory target's activation. + let mut refusing = system(); + refusing.add_failures.insert(512 * MIB, Errno::EIO); + let error = failure(target(), refusing); + cases.push(("an addition", error, status::REPAIR_MEMORY)); + let mut stuck = system(); + stuck.stuck = Some(5); + let error = failure(target(), stuck); + cases.push(("an offline block", error, status::REPAIR_MEMORY)); + // The reseed. + let mut refused = system(); + refused.reseed_failure = Some(Errno::EPERM); + let error = failure(restore(), refused); + cases.push(("the reseed", error, status::REPAIR_RESEED)); + // The finish, and one that never returned. + let mut unfinished = system(); + unfinished.finish_fails = true; + let error = failure(restore(), unfinished); + cases.push(("the finish", error, status::REPAIR_FINISH)); + let mut panicking = system(); + panicking.finish_panics = true; + let mut repair = repairing(restore(), panicking); + let unwound = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| repair.run())); + assert!(unwound.is_err()); + let error = repair.run().unwrap_err(); + cases.push(("an incomplete finish", error, status::REPAIR_FINISH)); + // The violations of the time ABI, whose statuses they keep. + let mut unrepaired = system(); + unrepaired.finish_violation = Some(Violation::of_repair("the time state is unreadable")); + let error = failure(restore(), unrepaired); + cases.push(("the time ABI's repair", error, status::TIME_REPAIR)); + let mut undaemoned = system(); + undaemoned.finish_violation = Some(Violation::of_missing_daemon()); + let error = failure(restore(), undaemoned); + cases.push(("the time daemon", error, status::TIME_CONFORMANCE)); + + let mut classes = Vec::new(); + for (case, error, class) in cases { + assert_eq!(error.status(), class, "{case}: {error}"); + classes.push(class); + } + // The cases cover every status of the repair, and the time ABI's statuses of a strict call. + classes.sort_unstable(); + classes.dedup(); + let mut expected: Vec<_> = status::REPAIR_STATUSES.collect(); + expected.extend([status::TIME_CONFORMANCE, status::TIME_REPAIR]); + assert_eq!(classes, expected); +} + +#[test] +fn a_violation_of_the_strict_finish_is_final() { + let calls = Calls::default(); + let mut system = FakeSystem::new(&calls); + system.finish_violation = Some(Violation::of_missing_daemon()); + let mut repair = repairing(restore(), system); + let error = repair.run().unwrap_err(); + assert_eq!(error.stage(), RepairStage::Finish); + assert_eq!( + error.to_string(), + "failed to finish the restore: G_CONFORMANCE_C10: the time daemon is not running" + ); + let mut expected = vec![Call::OnlineCpus, Call::BlockSize]; + expected.extend(finishing()); + assert_eq!(calls.take(), expected); + // Every retry returns the same error, with its status, without a second acknowledgement. + for _ in 0..2 { + let again = repair.run().unwrap_err(); + assert!(Arc::ptr_eq(&again.cause, &error.cause)); + assert_eq!(again.status(), status::TIME_CONFORMANCE); + } + assert_eq!(calls.take(), Vec::::new()); +} + +#[test] +fn errors_report_how_far_the_repair_got() { + let calls = Calls::default(); + let mut system = FakeSystem::new(&calls); + system.add_failures.insert(768 * MIB, Errno::EIO); + system.online_failures.insert(7, Errno::EBUSY); + let target = restore().with_memory_target(&[(512 * MIB, 512 * MIB)]); + let mut repair = repairing(target, system); + // The first attempt added and onlined two blocks before the third's addition failed. + let progress = repair.run().unwrap_err().progress(); + assert_eq!((progress.added(), progress.onlined()), (2, 2), "{progress}"); + assert!(!progress.reseeded() && !progress.finish_started()); + assert_eq!(progress.to_string(), "added 2 memory blocks, onlined 2"); + // The retry added the third and the fourth, and onlined the third, before the fourth's + // onlining failed: the progress counts the attempts' work together. + let progress = repair.run().unwrap_err().progress(); + assert_eq!((progress.added(), progress.onlined()), (4, 3), "{progress}"); + repair.run().unwrap(); + + // A failed finish follows the reseed, and may have acknowledged the restore. The kernel + // brought the blocks that the repair added online itself. + let mut system = FakeSystem::new(&calls); + system.auto_online = true; + system.finish_fails = true; + let target = restore().with_memory_target(&[(512 * MIB, 256 * MIB)]); + let mut repair = repairing(target, system); + let progress = repair.run().unwrap_err().progress(); + assert_eq!((progress.added(), progress.onlined()), (2, 0), "{progress}"); + assert!(progress.reseeded() && progress.finish_started()); + assert_eq!( + progress.to_string(), + "added 2 memory blocks, onlined 0, reseeded, started the finish" + ); + // A final failure keeps its progress. + assert_eq!(repair.run().unwrap_err().progress(), progress); + + // A failed reseed reseeded nothing, and started no finish. + let mut system = FakeSystem::new(&calls); + system.reseed_failure = Some(Errno::EPERM); + let error = repairing(restore(), system).run().unwrap_err(); + assert_eq!(error.progress(), RepairProgress::default()); + assert_eq!( + error.progress().to_string(), + "added 0 memory blocks, onlined 0" + ); +} + +#[test] +fn a_block_counts_once_the_repair_finds_it_added_or_online() { + let calls = Calls::default(); + let mut system = FakeSystem::new(&calls); + // The reads of a block's state come before its addition, then before its onlining, then + // after it: block 4's state cannot be read once it was added, nor block 5's once it was + // onlined. + system.read_failures.insert((4, 1), Errno::EIO); + system.read_failures.insert((5, 2), Errno::EIO); + system.reseed_failure = Some(Errno::EPERM); + let target = restore().with_memory_target(&[(512 * MIB, 256 * MIB)]); + let mut repair = repairing(target, system); + let expected = |added, onlined| RepairProgress { + added, + onlined, + ..RepairProgress::default() + }; + // The kernel added block 4, which the attempt could not find. + let error = repair.run().unwrap_err(); + assert!(matches!( + error.cause(), + RepairCause::Io(error) if error.raw_os_error() == Some(Errno::EIO as i32) + )); + assert_eq!(error.progress(), expected(0, 0)); + assert_eq!( + changes(calls.take()), + [Call::MemoryBlocks, Call::Add(512 * MIB)] + ); + // The retry finds block 4 added, and onlines it, then adds and onlines block 5, which it + // could not find online. + let error = repair.run().unwrap_err(); + assert_eq!(error.stage(), RepairStage::Memory); + assert_eq!(error.progress(), expected(2, 1)); + let mut activated = vec![Call::Online(4)]; + activated.extend(activations([5])); + assert_eq!(changes(calls.take()), activated); + // The next finds block 5 online, which counts once, and adds and onlines nothing. + let error = repair.run().unwrap_err(); + assert_eq!(error.stage(), RepairStage::Reseed); + assert_eq!(error.progress(), expected(2, 2)); + assert_eq!( + changes(calls.take()), + [Call::MemoryBlocks, Call::Reseed(RESTORED)] + ); + // A final failure keeps the count. + assert_eq!(repair.run().unwrap_err().progress(), expected(2, 2)); +} + /// Checks that the repair of a clone whose kernel lists the online CPUs `cpus`, and whose restore /// has the processor target `target`, if any, counts `online` CPUs. fn counts(cpus: &'static [u8], target: Option, online: u32) { diff --git a/nvx-rs/src/guest/clone/tests.rs b/nvx-rs/src/guest/clone/tests.rs index 8fc75afe9..0b0bb029c 100644 --- a/nvx-rs/src/guest/clone/tests.rs +++ b/nvx-rs/src/guest/clone/tests.rs @@ -1050,3 +1050,11 @@ fn the_live_system_finds_windows_and_block_devices_in_procfs_and_sysfs() { io::ErrorKind::NotFound ); } + +#[test] +fn the_capture_and_the_repair_take_the_time_abi_in_strict_mode() { + let abi = live_abi(); + // Strict, unless the kernel command line of the test's host makes the time ABI report-only. + assert_ne!(abi.is_strict(), abi.is_report_only()); + assert_eq!(abi, TimeAbi::new().strict()); +} diff --git a/nvx-rs/src/guest/time/api.rs b/nvx-rs/src/guest/time/api.rs index e425a3e8d..082b5ad3a 100644 --- a/nvx-rs/src/guest/time/api.rs +++ b/nvx-rs/src/guest/time/api.rs @@ -15,10 +15,10 @@ use super::state::{ CheckState, CheckStatus, Discontinuity, DowntimeSource, SampleRejection, TimeState, }; use super::system::{ - self, PORTB_GENERATION_ID_AVAILABLE, PORTB_PACKET_AVAILABLE, PRIVATE_FILE_MODE, RESTORE_PATH, - RESTORE_TEMP_PATH, RealOps, Runtime, SELECT_GENERATION_ID, SELECT_PACKET, SampleError, - SaveError, SpawnError, clock_ns, cpu_time_ns, current_frequency, fork_process, initial_sample, - portb_read, set_frequency, state_apply, state_load, state_store, step_clock, + self, PORTB_GENERATION_ID_AVAILABLE, PORTB_PACKET_AVAILABLE, PRIVATE_FILE_MODE, PowerOffStatus, + RESTORE_PATH, RESTORE_TEMP_PATH, RealOps, Runtime, SELECT_GENERATION_ID, SELECT_PACKET, + SampleError, SaveError, SpawnError, clock_ns, cpu_time_ns, current_frequency, fork_process, + initial_sample, portb_read, set_frequency, state_apply, state_load, state_store, step_clock, }; use super::{NSEC_PER_USEC, Phase}; use crate::guest::GenerationId; @@ -47,13 +47,19 @@ const REPORT_ONLY_TOKEN: &[u8] = b"nvx_time_abi=report-only"; /// /// A violation of the time ABI is reported by a violation event on the console, in the kernel /// log, and on OpenVMM's debug console. In enforcing mode, the event stops the VM, so the call -/// does not return. In report-only mode, the call fails with [`TimeError::Violation`] when the -/// violation keeps it from doing its work. Otherwise, it goes on as `nvx-time` does, and the -/// event is the only report of the violation, as each call tells. +/// does not return. In strict mode, which a caller selects with [`strict`](Self::strict), so +/// that it owns the VM's exit status and its shutdown, such as a guest init or agent that runs a +/// clone's capture and repair in its own process, the call fails with [`TimeError::Violation`] +/// instead, whose [`status`](Violation::status) is the one with which enforcing mode would have +/// stopped the VM, and never stops the VM. In report-only mode, the call fails with +/// [`TimeError::Violation`] when the violation keeps it from doing its work. Otherwise, it goes +/// on as `nvx-time` does, and the event is the only report of the violation, as each call tells. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct TimeAbi { /// Whether violations are reported instead of stopping the VM. report_only: bool, + /// Whether a violation that stops the VM in enforcing mode fails the call instead. + strict: bool, } impl TimeAbi { @@ -62,12 +68,32 @@ impl TimeAbi { pub fn new() -> Self { Self { report_only: cmdline_requests_report_only(), + strict: false, } } /// Returns the time ABI in report-only mode, whatever the kernel command line selects. pub fn report_only() -> Self { - Self { report_only: true } + Self { + report_only: true, + strict: false, + } + } + + /// Returns the time ABI in strict mode, unless it is in report-only mode, which it keeps: a + /// violation that stops the VM in enforcing mode fails the call instead, after its event, + /// with [`TimeError::Violation`], the call's first, so that the caller ends the VM, with the + /// violation's [`status`](Violation::status). The command tools, such as `nvx-time`, keep + /// enforcing mode. + /// + /// The time daemon that [`start_daemon`](Self::start_daemon) starts enforces the ABI even + /// then, since no caller owns its failures, and so does the preparation of a capture that + /// `nvx::guest::clone` runs, as `nvx-time pre-capture` does. + pub fn strict(self) -> Self { + Self { + strict: !self.report_only, + ..self + } } /// Returns whether violations are reported instead of stopping the VM. @@ -75,8 +101,24 @@ impl TimeAbi { self.report_only } + /// Returns whether a violation that stops the VM in enforcing mode fails the call instead. + pub fn is_strict(&self) -> bool { + self.strict + } + /// Returns the runtime of one call. pub(crate) fn runtime(self) -> Runtime<'static> { + let runtime = self.enforcing_runtime(); + if self.strict { + runtime.strict() + } else { + runtime + } + } + + /// Returns the runtime of a call that enforces the ABI even in strict mode: in enforcing or + /// report-only mode, as the kernel command line selects. + fn enforcing_runtime(self) -> Runtime<'static> { Runtime::new(&RealOps, self.report_only) } @@ -89,7 +131,8 @@ impl TimeAbi { /// [`start_daemon`](Self::start_daemon) starts the time daemon from its outcome, /// separately, which `nvx-time boot` does next. pub fn cold_boot(&self) -> Result { - released(cold_boot(&self.runtime())) + let runtime = self.runtime(); + settled(&runtime, cold_boot(&runtime)) } /// Starts the time daemon of `boot`, the outcome of [`cold_boot`](Self::cold_boot), in a @@ -102,7 +145,9 @@ impl TimeAbi { /// its standard streams on the null device, until the VM stops or `SIGTERM` stops it, and /// waits only for the children that it starts itself. It takes the command name `nvx-time`, /// by which the calls of the time ABI recognize it. The caller supervises it and collects - /// its status: no call of the time ABI waits for it. + /// its status: no call of the time ABI waits for it. The child enforces the ABI even when it + /// is [strict](Self::strict), since no caller owns its failures: a violation that it reports + /// stops the VM, unless the ABI is report-only. /// /// The call returns when the child starts, so the boot checks' failures, which the child /// reports by their events, do not fail it. @@ -116,7 +161,7 @@ impl TimeAbi { match fork_process() { Ok(ForkResult::Child) => { sys::reset_child_signal(); - daemon::boot_checker(&self.runtime(), &boot) + daemon::boot_checker(&self.enforcing_runtime(), &boot) } Ok(ForkResult::Parent(child)) => Ok(child.raw().unsigned_abs()), Err(SpawnError::Ports(error)) => Err(error.into()), @@ -126,7 +171,8 @@ impl TimeAbi { /// Reads the VM generation ID, which a [capture](Self::capture) needs. pub fn generation_id(&self) -> Result { - released(read_generation_id(&self.runtime())) + let runtime = self.runtime(); + settled(&runtime, read_generation_id(&runtime)) } /// Requests a capture through the snapshot port, preparation step 5, and, in a VM that a @@ -144,7 +190,8 @@ impl TimeAbi { /// [`Restore::finished`] tells whether the call finished the restore, and how it reports a /// failed handoff of repair step 7. pub fn capture(&self, request: CaptureRequest) -> Result { - released(capture(&self.runtime(), request)) + let runtime = self.runtime(); + settled(&runtime, capture(&runtime, request)) } /// Finishes a restore that [`capture`](Self::capture) left unfinished, after the caller @@ -152,16 +199,17 @@ impl TimeAbi { /// CPUs that it brought online, acknowledges it when asked, repair step 6, and hands repair /// step 7, the restore checks and the release of the stall detectors, to the time daemon. /// - /// Without a running daemon, an enforcing call reports a violation of check C10, and a - /// report-only call runs step 7 itself, which needs a process that has no other thread. Step - /// 7 forces an RCU grace period in a grandchild, which a child of the call waits for until - /// the RCU stall timeout at the latest. The call waits for that child alone, whatever the - /// process's disposition of `SIGCHLD`, so a release that outlives the timeout passes to - /// init, or to the nearest subreaper, without the process's descriptors. The call returns - /// when step 7 ends, which reports its failures by their events only, as + /// Without a running daemon, an enforcing or strict call reports a violation of check C10, + /// and a report-only call runs step 7 itself, which needs a process that has no other + /// thread. Step 7 forces an RCU grace period in a grandchild, which a child of the call waits + /// for until the RCU stall timeout at the latest. The call waits for that child alone, + /// whatever the process's disposition of `SIGCHLD`, so a release that outlives the timeout + /// passes to init, or to the nearest subreaper, without the process's descriptors. The call + /// returns when step 7 ends, which reports its failures by their events only, as /// `nvx-time restore-finish` does. pub fn restore_finish(&self, request: RestoreFinish) -> Result<(), TimeError> { - released(restore_finish(&self.runtime(), request)) + let runtime = self.runtime(); + settled(&runtime, restore_finish(&runtime, request)) } /// Prepares the guest for a capture, as `nvx-time pre-capture` does: waits until the boot @@ -169,9 +217,10 @@ impl TimeAbi { /// capture checks require, runs the capture checks, saves the stall detectors' settings and /// suppresses the detectors, and records the capture checks. /// - /// A capture check that fails reports a violation, which stops the VM in enforcing mode; a - /// report-only call goes on, as `nvx-time` does. A call that fails leaves the detectors as - /// they were, and never touches those of a capture that is pending already, + /// A capture check that fails reports a violation, which stops the VM in enforcing mode, and + /// in strict mode too, in which the preparation enforces the ABI, as `nvx-time pre-capture` + /// does; a report-only call goes on, as `nvx-time` does. A call that fails leaves the + /// detectors as they were, and never touches those of a capture that is pending already, /// [`PreCaptureError::Pending`], unless it /// [holds their suppression](PreCaptureError::holds_suppression), which the caller then /// releases with [`cancel_capture`](Self::cancel_capture), as after a call that succeeded. @@ -179,7 +228,10 @@ impl TimeAbi { /// [`PreCaptureError::PortRelease`], if the thread may still hold some of the permission for /// the ports. pub(crate) fn pre_capture(&self) -> Result<(), PreCaptureError> { - released_preparation(pre_capture(&self.runtime()), sys::check_released()) + released_preparation( + pre_capture(&self.enforcing_runtime()), + sys::check_released(), + ) } /// Restores the stall detectors' settings that [`pre_capture`](Self::pre_capture) saved, and @@ -214,6 +266,16 @@ fn released(result: Result) -> Result { result } +/// Ends a call of `runtime` with `result`, as `released` does, unless the call reported a +/// violation in strict mode, with which it fails instead: the first that it reported, with which +/// enforcing mode would have stopped the VM. +fn settled(runtime: &Runtime<'_>, result: Result) -> Result { + released(match runtime.take_violation() { + Some(violation) => Err(TimeError::Violation(violation)), + None => result, + }) +} + /// Ends a preparation of a capture with `result`, unless the calling thread may still hold port /// permission, as `released` tells, which the preparation reports instead, with whether it holds /// the stall detectors' suppression. @@ -265,6 +327,10 @@ fn cold_boot(runtime: &Runtime<'_>) -> Result { Ok(initial) => Some((initial, clock_ns(sys::Clock::Monotonic))), Err(detail) => { checks.fail(runtime, CheckId::C12, detail); + // A strict call stops where enforcing mode stops the VM. + if let Some(violation) = runtime.take_violation() { + return Err(TimeError::Violation(violation)); + } None } }; @@ -1162,19 +1228,23 @@ fn restore_finish(runtime: &Runtime<'_>, request: RestoreFinish) -> Result<(), T }) } -/// Reports that restore-time repair failed, which stops the VM unless report-only, and returns -/// the violation. +/// Reports that restore-time repair failed, which stops the VM unless report-only or strict, +/// and returns the violation. fn repair_failed( runtime: &Runtime<'_>, code: ViolationCode, detail: impl AsRef<[u8]>, ) -> TimeError { runtime.repair_failure(code, detail.as_ref()); - TimeError::Violation(Violation::new(code, detail.as_ref())) + TimeError::Violation(Violation::new( + PowerOffStatus::Repair, + code, + detail.as_ref(), + )) } -/// Reports that check `id` failed in `phase`, which stops the VM unless report-only, and -/// returns the violation. +/// Reports that check `id` failed in `phase`, which stops the VM unless report-only or strict, +/// and returns the violation. fn check_failed( runtime: &Runtime<'_>, phase: Phase, @@ -1183,6 +1253,7 @@ fn check_failed( ) -> TimeError { Checks::empty(phase).fail(runtime, id, detail.as_ref()); TimeError::Violation(Violation::new( + PowerOffStatus::Conformance, ViolationCode::Conformance(id), detail.as_ref(), )) @@ -1191,6 +1262,8 @@ fn check_failed( /// A violation of the time ABI, which a violation event reported. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Violation { + /// The status with which enforcing mode stops the VM for the violation. + status: PowerOffStatus, /// The ABI's code of the violation. code: String, /// The event's detail. @@ -1198,14 +1271,26 @@ pub struct Violation { } impl Violation { - /// Returns the violation of `code`, with `detail`. - fn new(code: ViolationCode, detail: &[u8]) -> Self { + /// Returns the violation of `code`, with `detail`, for which enforcing mode stops the VM + /// with `status`. + pub(crate) fn new(status: PowerOffStatus, code: ViolationCode, detail: &[u8]) -> Self { Self { + status, code: code.to_string(), detail: String::from_utf8_lossy(detail).into_owned(), } } + /// Returns the exit status with which enforcing mode stops the VM for the violation, and + /// with which a [strict](TimeAbi::strict) caller ends it: 193, + /// [`TIME_CONFORMANCE`](crate::protocol::status::TIME_CONFORMANCE), for a conformance check + /// that failed, or 195, [`TIME_REPAIR`](crate::protocol::status::TIME_REPAIR), for the + /// repair of a restore that failed. Only the time daemon stops the VM with 194, + /// [`TIME_VIOLATION`](crate::protocol::status::TIME_VIOLATION). + pub fn status(&self) -> u8 { + self.status.code() + } + /// Returns the time ABI's code of the violation, such as `G_REPAIR_PACKET`. pub fn code(&self) -> &str { &self.code @@ -1224,6 +1309,29 @@ impl Display for Violation { } } +#[cfg(test)] +impl Violation { + /// Returns the violation with which a strict call fails once the time ABI's part of a + /// restore's repair failed, with `detail`, as the tests of the ABI's callers need one. + pub(crate) fn of_repair(detail: &str) -> Self { + Self::new( + PowerOffStatus::Repair, + ViolationCode::RepairClock, + detail.as_bytes(), + ) + } + + /// Returns the violation with which a strict finish fails without a running time daemon, a + /// failure of check C10, as the tests of the ABI's callers need one. + pub(crate) fn of_missing_daemon() -> Self { + Self::new( + PowerOffStatus::Conformance, + ViolationCode::Conformance(CheckId::C10), + b"the time daemon is not running", + ) + } +} + /// Why a call of the time ABI failed. /// /// Its `Display` form is a one-line diagnostic, such as `ioperm: Operation not permitted`. @@ -1252,8 +1360,9 @@ pub enum TimeError { GenerationIdUnavailable, /// The time daemon's process could not be started. Spawn(io::Error), - /// The guest violates the time ABI, which a report-only call reports with a violation - /// event before it fails. + /// The guest violates the time ABI, which a report-only or [strict](TimeAbi::strict) call + /// reports with a violation event before it fails. Its + /// [`status`](Violation::status) is the one with which enforcing mode stops the VM. Violation(Violation), } @@ -1311,9 +1420,11 @@ mod tests { use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::Path; + use std::sync::atomic::Ordering; use super::*; use crate::guest::time::system::tests::FakeOps; + use crate::protocol::status; /// Returns whether `error` holds the error number `errno`. fn has_errno(error: &io::Error, errno: Errno) -> bool { @@ -1360,6 +1471,7 @@ mod tests { ), ( TimeError::Violation(Violation::new( + PowerOffStatus::Repair, ViolationCode::RepairPacket, b"restore packet version is 3", )), @@ -1369,13 +1481,79 @@ mod tests { for (error, text) in cases { assert_eq!(error.to_string(), text, "{error:?}"); } - let violation = Violation::new(ViolationCode::Conformance(CheckId::C10), b"detail"); + let violation = Violation::new( + PowerOffStatus::Conformance, + ViolationCode::Conformance(CheckId::C10), + b"detail", + ); assert_eq!( - (violation.code(), violation.detail()), - ("G_CONFORMANCE_C10", "detail") + (violation.status(), violation.code(), violation.detail()), + (193, "G_CONFORMANCE_C10", "detail") ); } + #[test] + fn strict_mode_keeps_report_only_mode_and_spares_the_daemon_and_the_preparation() { + let enforcing = TimeAbi { + report_only: false, + strict: false, + }; + assert!(!enforcing.is_strict() && !enforcing.runtime().is_strict()); + let strict = enforcing.strict(); + assert!(strict.is_strict() && !strict.is_report_only()); + assert!(strict.runtime().is_strict()); + // The daemon and the preparation of a capture enforce the ABI. + let spared = strict.enforcing_runtime(); + assert!(!spared.is_strict() && !spared.report_only()); + // Report-only mode stays report-only. + let report_only = TimeAbi::report_only().strict(); + assert!(report_only.is_report_only() && !report_only.is_strict()); + assert!(report_only.runtime().report_only() && !report_only.runtime().is_strict()); + assert_eq!(TimeAbi::report_only(), report_only); + } + + #[test] + fn a_strict_call_fails_with_the_violation_that_stops_an_enforcing_one() { + for (report_only, strict, power) in [ + (false, false, status::TIME_REPAIR), + (false, true, 0), + (true, false, 0), + (true, true, 0), + ] { + let ops = FakeOps::new([]); + let runtime = Runtime::new(&ops, report_only); + let runtime = if strict { runtime.strict() } else { runtime }; + for read in [ + PacketRead::InvalidHeader(PacketError::Version(3)), + PacketRead::Generation(3), + ] { + let target = RepairTarget { + previous_id: GenerationId::from_bytes([1; GenerationId::LENGTH]), + entropy: EntropySink::Seed, + finish: false, + daemon_pid: None, + }; + let error = settled(&runtime, repair_restore(&runtime, read, target, 0)); + let Err(TimeError::Violation(violation)) = error else { + panic!("{error:?}"); + }; + assert_eq!(violation.status(), status::TIME_REPAIR, "{violation}"); + assert!(violation.code().starts_with("G_REPAIR_"), "{violation}"); + // The call keeps no violation for the next one. + assert_eq!(runtime.take_violation(), None); + } + assert_eq!(ops.power.load(Ordering::Relaxed), usize::from(power)); + // Each violation has its event, in its mode's form. + let console = String::from_utf8(ops.console.lock().unwrap().clone()).unwrap(); + let prefix = if report_only { + "NVX-TIME-REPORT-VIOLATION: " + } else { + "NVX-TIME-ABI-VIOLATION: " + }; + assert_eq!(console.matches(prefix).count(), 2, "{console}"); + } + } + #[test] fn a_call_holds_the_port_permission_for_its_transaction_alone() { // Port B offers no VM generation ID. @@ -1776,4 +1954,65 @@ mod tests { } } } + + /// Runs, through the fakes, cold boots whose initial sample fails check C12, in strict mode + /// and then in report-only mode, on a tmpfs at `/run` in a mount namespace of the process's + /// own. + fn cold_boot_in_strict_and_report_only_mode() { + sys::enter_private_mount_namespace().unwrap(); + let _run = sys::TmpfsMount::mount(c"nvx-test", c"/run", c"mode=0755").unwrap(); + // Port B offers no time-sample window, so the initial sample fails check C12. + let ops = FakeOps::new([0]); + let runtime = Runtime::new(&ops, false).strict(); + let error = super::cold_boot(&runtime); + let Err(TimeError::Violation(violation)) = error else { + panic!("{error:?}"); + }; + assert_eq!( + (violation.status(), violation.code()), + (status::TIME_CONFORMANCE, "G_CONFORMANCE_C12") + ); + // A strict cold boot stops where enforcing mode stops the VM, without stopping it: it + // publishes no time state. + assert_eq!(ops.power.load(Ordering::Relaxed), 0); + assert!(state_load().is_err()); + assert_eq!(runtime.take_violation(), None); + let console = String::from_utf8(ops.console.lock().unwrap().clone()).unwrap(); + assert!( + console.starts_with( + "NVX-TIME-ABI-VIOLATION: v=1 code=G_CONFORMANCE_C12 source=conformance phase=boot " + ), + "{console}" + ); + // A report-only cold boot goes on, and leaves the wall clock unsynchronized. + let ops = FakeOps::new([0]); + let boot = super::cold_boot(&Runtime::new(&ops, true)).unwrap(); + assert!(!boot.synchronized()); + assert_eq!(boot.failures, 1); + assert!(!state_load().unwrap().synchronized); + } + + #[test] + fn a_privileged_strict_cold_boot_stops_at_its_violation() { + if crate::guest::managed_agent::isolate_process_test( + "NVX_TIME_COLD_BOOT_PRIVILEGED_TEST_CHILD", + "guest::time::api::tests::a_privileged_strict_cold_boot_stops_at_its_violation", + ) { + return; + } + if !sys::holds_capability(sys::CAP_SYS_ADMIN).unwrap() { + return; + } + // A cold boot keeps its state and its locks in `/run/nvx`, which a child of the + // harness's process holds alone in a mount namespace of its own. + match sys::fork_isolated_test().unwrap() { + ForkResult::Child => { + let outcome = std::panic::catch_unwind(cold_boot_in_strict_and_report_only_mode); + sys::exit_immediately(u8::from(outcome.is_err())); + } + ForkResult::Parent(child) => { + assert_eq!(sys::wait(child), Some(sys::ChildStatus::Exited(0))); + } + } + } } diff --git a/nvx-rs/src/guest/time/mod.rs b/nvx-rs/src/guest/time/mod.rs index 57be2ae87..3a3e00910 100644 --- a/nvx-rs/src/guest/time/mod.rs +++ b/nvx-rs/src/guest/time/mod.rs @@ -41,6 +41,13 @@ //! # } //! ``` //! +//! A violation of the ABI stops the VM in enforcing mode, the mode of the command tools, with a +//! status that `nvx::protocol::status` publishes: 193 for a conformance check, 194 for the time +//! daemon's watcher, and 195 for the repair of a restore. A caller that owns the VM's exit +//! status and its shutdown takes the ABI in strict mode, with [`TimeAbi::strict`], in which a +//! call fails with the violation instead, whose [`status`](Violation::status) the caller ends the +//! VM with. +//! //! `nvx-time` runs the conformance checks, the violation watcher and the wall-clock discipline //! in one daemon, a separate process, and the snapshot agent's time steps: the guest preparation //! steps of the architecture guide's section 5.3.2 and the guest repair steps of its section diff --git a/nvx-rs/src/guest/time/system.rs b/nvx-rs/src/guest/time/system.rs index ba1a6994d..0a25a0180 100644 --- a/nvx-rs/src/guest/time/system.rs +++ b/nvx-rs/src/guest/time/system.rs @@ -3,9 +3,10 @@ use std::ffi::{CStr, CString}; use std::fmt::{self, Display}; -use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Mutex, PoisonError}; +use super::api::Violation; use super::decoders::{self, SAMPLE_SIZE, TimeSample, TimeSampleError}; use super::events::{EventSource, ViolationCode, format_event}; use super::state::TimeState; @@ -15,17 +16,19 @@ use crate::guest::sys::{ ForkResult, Mode, MsrAccess, MsrFile, OpenFlags, Port, PortAccess, PortError, TimeValLike, kernel, }; +use crate::protocol::status; -/// Statuses with which a fatal time-ABI failure stops the VM through the control port. +/// Statuses with which a fatal time-ABI failure stops the VM through the control port, which +/// `nvx::protocol::status` publishes for the host. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub(crate) enum PowerOffStatus { /// A time-ABI conformance check failed. - Conformance = 193, + Conformance = status::TIME_CONFORMANCE, /// The runtime watcher observed a time-ABI violation. - Violation = 194, + Violation = status::TIME_VIOLATION, /// Restore-time repair could not complete safely. - Repair = 195, + Repair = status::TIME_REPAIR, } impl PowerOffStatus { @@ -226,6 +229,11 @@ pub(crate) struct Runtime<'a> { pub(crate) ops: &'a dyn PrivilegedOps, /// Whether violations are reported instead of powering off the VM. report_only: bool, + /// Whether a violation that powers off the VM in enforcing mode fails the call instead, which + /// the runtime keeps the violation for. + strict: bool, + /// The first violation that the call reported in strict mode, which fails the call. + violation: Mutex>, /// Current VM generation. generation: AtomicU32, /// Whether an asynchronous child may have interrupted console output. @@ -240,17 +248,42 @@ impl<'a> Runtime<'a> { Self { ops, report_only, + strict: false, + violation: Mutex::new(None), generation: AtomicU32::new(0), async_output: AtomicBool::new(false), report_lock: Mutex::new(()), } } + /// Returns the runtime in strict mode, unless it is report-only: a violation that powers off + /// the VM in enforcing mode is reported by its event alone, and the runtime keeps the first + /// such violation, with which the call fails, as [`take_violation`](Self::take_violation) + /// tells. The runtime then never powers off the VM. + pub(crate) fn strict(mut self) -> Self { + self.strict = !self.report_only; + self + } + /// Returns whether conformance failures are report-only. pub(crate) fn report_only(&self) -> bool { self.report_only } + /// Returns whether a violation fails the call instead of powering off the VM. + pub(crate) fn is_strict(&self) -> bool { + self.strict + } + + /// Takes the first violation that the call reported in strict mode, which fails the call, + /// if any. + pub(crate) fn take_violation(&self) -> Option { + self.violation + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + } + /// Returns the current VM generation. pub(crate) fn generation(&self) -> u32 { self.generation.load(Ordering::Relaxed) @@ -336,16 +369,15 @@ impl<'a> Runtime<'a> { let _ = count_violation(); } - /// Reports a conformance failure and powers off unless report-only. + /// Reports a conformance failure, and powers off unless report-only or strict. pub(crate) fn conformance_failure(&self, code: ViolationCode, phase: Phase, detail: &[u8]) { let _guard = self.report_lock.lock().ok(); self.emit_event_unlocked(code, EventSource::Conformance, phase, detail); - if !self.report_only() { - self.ops.power_off(PowerOffStatus::Conformance); - } + self.stop(PowerOffStatus::Conformance, code, detail); } - /// Emits a fatal event and requests its typed power-off status. + /// Emits a fatal event and requests its typed power-off status, unless report-only or + /// strict. pub(crate) fn fail_fatal( &self, status: PowerOffStatus, @@ -355,13 +387,11 @@ impl<'a> Runtime<'a> { detail: &[u8], ) { self.emit_event(code, source, phase, detail); - if !self.report_only() { - self.ops.power_off(status); - } + self.stop(status, code, detail); } /// Reports that restore-time repair failed, a fatal event of the restore phase, and powers off - /// with the repair status unless report-only. + /// with the repair status unless report-only or strict. pub(crate) fn repair_failure(&self, code: ViolationCode, detail: &[u8]) { self.fail_fatal( PowerOffStatus::Repair, @@ -371,6 +401,20 @@ impl<'a> Runtime<'a> { detail, ); } + + /// Stops the VM with `status` once its event reported the violation of `code`, with `detail`: + /// powers it off in enforcing mode, keeps the call's first violation in strict mode, and does + /// neither in report-only mode. + fn stop(&self, status: PowerOffStatus, code: ViolationCode, detail: &[u8]) { + if self.is_strict() { + self.violation + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get_or_insert_with(|| Violation::new(status, code, detail)); + } else if !self.report_only { + self.ops.power_off(status); + } + } } /// Reads one Linux clock in nanoseconds, or zero when it cannot be read. @@ -837,6 +881,7 @@ pub(crate) mod tests { use super::*; use crate::guest::sys::PortBackend; use crate::guest::sys::x86::msr; + use crate::guest::time::events::CheckId; /// Privileged operations that record their requests and replay canned port input. pub(crate) struct FakeOps { @@ -1135,13 +1180,17 @@ pub(crate) mod tests { } #[test] - fn repair_failures_report_restore_events_and_power_off_unless_report_only() { - for (report_only, prefix, power) in [ - (false, "NVX-TIME-ABI", PowerOffStatus::Repair.code()), - (true, "NVX-TIME-REPORT", 0), + fn repair_failures_report_restore_events_and_power_off_only_when_enforcing() { + for (report_only, strict, prefix, power) in [ + (false, false, "NVX-TIME-ABI", PowerOffStatus::Repair.code()), + (false, true, "NVX-TIME-ABI", 0), + (true, false, "NVX-TIME-REPORT", 0), + (true, true, "NVX-TIME-REPORT", 0), ] { let ops = FakeOps::new([]); let runtime = Runtime::new(&ops, report_only); + let runtime = if strict { runtime.strict() } else { runtime }; + assert_eq!(runtime.is_strict(), strict && !report_only); runtime.set_generation(4); runtime.repair_failure(ViolationCode::RepairSuppression, b"cannot restore"); let line = String::from_utf8(ops.console.lock().unwrap().clone()).unwrap(); @@ -1158,9 +1207,66 @@ pub(crate) mod tests { let port: Vec<_> = line.bytes().map(|byte| (Port::PortbData, byte)).collect(); assert_eq!(*ops.output.lock().unwrap(), port); assert_eq!(ops.power.load(Ordering::Relaxed), usize::from(power)); + // A strict runtime keeps the violation instead, with enforcing mode's status. + let kept = runtime.take_violation(); + if runtime.is_strict() { + let violation = kept.unwrap(); + assert_eq!( + (violation.status(), violation.code(), violation.detail()), + (195, "G_REPAIR_SUPPRESSION", "cannot restore") + ); + } else { + assert_eq!(kept, None); + } } } + #[test] + fn a_strict_runtime_keeps_its_first_violation_and_never_powers_off() { + let ops = FakeOps::new([]); + let runtime = Runtime::new(&ops, false).strict(); + runtime.conformance_failure( + ViolationCode::Conformance(CheckId::C12), + Phase::Boot, + b"first", + ); + runtime.repair_failure(ViolationCode::RepairSuppression, b"second"); + runtime.fail_fatal( + PowerOffStatus::Violation, + ViolationCode::RcuStall, + EventSource::Watcher, + Phase::Runtime, + b"third", + ); + assert_eq!(ops.power.load(Ordering::Relaxed), 0); + let console = String::from_utf8(ops.console.lock().unwrap().clone()).unwrap(); + assert_eq!( + console.matches("NVX-TIME-ABI-VIOLATION: ").count(), + 3, + "{console}" + ); + let violation = runtime.take_violation().unwrap(); + assert_eq!( + (violation.status(), violation.code(), violation.detail()), + (193, "G_CONFORMANCE_C12", "first") + ); + assert_eq!(runtime.take_violation(), None); + // The statuses are those that the protocol publishes for the host. + assert_eq!( + [ + PowerOffStatus::Conformance, + PowerOffStatus::Violation, + PowerOffStatus::Repair + ] + .map(PowerOffStatus::code), + [ + status::TIME_CONFORMANCE, + status::TIME_VIOLATION, + status::TIME_REPAIR + ] + ); + } + #[test] fn modes_of_the_runtime_files_match_the_c_program() { assert_eq!(DIRECTORY_MODE.bits(), 0o755); diff --git a/nvx-rs/src/lib.rs b/nvx-rs/src/lib.rs index c03b3e88b..9873f135e 100644 --- a/nvx-rs/src/lib.rs +++ b/nvx-rs/src/lib.rs @@ -6,8 +6,9 @@ //! control console. This crate holds everything it takes to drive one: //! //! - [`protocol`]: the codec of the control console, the `NVXS` session that OpenVMM brokers and -//! the `NVXC` frames of the guest agent, and the guest targets that a sandbox's live shares may -//! not take. It has no dependencies, so the host, the guest, and test doubles share it. +//! the `NVXC` frames of the guest agent, the guest targets that a sandbox's live shares may not +//! take, and the exit statuses with which a clone's guest fails fast. It has no dependencies, +//! so the host, the guest, and test doubles share it. //! - [`activation`]: the activation record of a layered root, the bounded, versioned, non-secret //! record of the settings that its activation applies, which a clone restored from a template //! reads in place of its captured configuration, and what Ready reports about the activation. @@ -51,8 +52,8 @@ //! (`guest::root`), and reads the partition table of a GPT disk's block device (`guest::gpt`), //! the typed calls through which it runs the time ABI's steps (`guest::time`) and reseeds after //! a restore (`guest::reseed`) in its own process, and the preparation of a template's guest -//! for a capture before its root's assembly (`guest::clone`). The crate's `nvx-*` guest -//! binaries run the programs. +//! for a capture before its root's assembly, and the repair of its clones (`guest::clone`). The +//! crate's `nvx-*` guest binaries run the programs. //! //! # Example //! diff --git a/nvx-rs/src/protocol/mod.rs b/nvx-rs/src/protocol/mod.rs index 2710670eb..e38062744 100644 --- a/nvx-rs/src/protocol/mod.rs +++ b/nvx-rs/src/protocol/mod.rs @@ -28,6 +28,8 @@ //! - [`nvxs`]: the NVXS session without I/O: validated records, incremental parsing, and the //! state machines of the host's and the guest's roles, which carry application bytes of any //! protocol. +//! - [`status`]: the exit statuses with which a clone's guest ends its VM once its repair or its +//! activation failed, the time ABI's included, from which the host classifies a clone's end. //! //! The module depends on nothing outside the standard library, so the host, the guest, and test //! doubles share it. Tests pin its wire constants as literals, and its records, frames, and @@ -47,6 +49,7 @@ pub mod nvxs; mod outer; pub mod root; pub mod share; +pub mod status; #[cfg(test)] mod tests; diff --git a/nvx-rs/src/protocol/status.rs b/nvx-rs/src/protocol/status.rs new file mode 100644 index 000000000..9e2c4a25e --- /dev/null +++ b/nvx-rs/src/protocol/status.rs @@ -0,0 +1,184 @@ +//! The exit statuses with which a clone's guest ends its VM once it fails fast, and from which +//! the host classifies a clone's end. The guest library's errors select them: the time ABI's +//! violations, `nvx::guest::time::Violation::status`, and a clone's repair errors, +//! `nvx::guest::clone::RepairError::status`. +//! +//! - [`TIME_CONFORMANCE`], 193, [`TIME_VIOLATION`], 194, and [`TIME_REPAIR`], 195: the time ABI's +//! own statuses, which keep their meanings. In enforcing mode, the time ABI stops the VM with +//! them itself. In strict mode, which a guest init or agent selects when it runs the time ABI's +//! calls in its own process, such as a clone's capture and repair, a call fails with the +//! violation instead, whose status is one of these, and the caller ends the VM with it. +//! - [`REPAIR_RESTORE`] to [`REPAIR_FINISH`], 40 to 45: the other failures of a clone's repair, one +//! status for each class. +//! - [`ACTIVATION_DEADLINE`] to [`ACTIVATION_ORDER`], 50 to 58: the failures of a clone's +//! activation, which follows its repair, one status reserved for each class: the deadline, the +//! host's session, the Activate's record and the resources that it expects, the activation of +//! the root, of the workload, of the host export, and of the network settings, and the guest's +//! own order. +//! +//! The guest ends the VM with a status after a diagnostic that names the failure, such as a +//! violation event of the time ABI, or an error line. A status alone does not tell the cause, nor +//! where it came from, since a guest whose workload ends it may exit with any status, so the host +//! classifies a clone's end by its status together with that diagnostic. + +use std::ops::RangeInclusive; + +/// A conformance check of the time ABI failed. +pub const TIME_CONFORMANCE: u8 = 193; +/// The time ABI's violation watcher observed a violation at run time, such as a clocksource +/// switch, a TSC warp, or an RCU stall. +pub const TIME_VIOLATION: u8 = 194; +/// The time ABI's part of a restore's repair failed: the restore packet's validation, the wall +/// clock's step, the record of the restore, or the release of the stall detectors. +pub const TIME_REPAIR: u8 = 195; + +/// The restore cannot be repaired: it holds no fresh entropy, as when its capture wrote the +/// entropy to a file, or its capture finished it already. +pub const REPAIR_RESTORE: u8 = 40; +/// The online CPUs cannot be read, are not a prefix of the CPUs, or do not meet the restore's +/// processor target, which repair does not activate. +pub const REPAIR_PROCESSORS: u8 = 41; +/// The restore's memory target is refused, before any memory block was added: the kernel's +/// memory block size is not the snapshot ABI's, the restore lists ranges without a target, or +/// they do not continue the RAM that the clone captured. +pub const REPAIR_MEMORY_TARGET: u8 = 42; +/// The activation of the restore's memory target failed: a memory block could not be added or +/// onlined, or did not come online, or the kernel's memory interface could not be read. +pub const REPAIR_MEMORY: u8 = 43; +/// The reseed of the kernel's random number generator with the restore's fresh entropy failed. +pub const REPAIR_RESEED: u8 = 44; +/// The restore's finish failed other than by a violation of the time ABI, which ends the clone +/// with the violation's status, or an earlier attempt at it never returned. It may have +/// acknowledged the restore. +pub const REPAIR_FINISH: u8 = 45; + +/// The clone was not activated before its deadline, which starts at its repair's +/// acknowledgement. +pub const ACTIVATION_DEADLINE: u8 = 50; +/// The host broke the restricted session of a repaired clone before the clone was ready: it +/// reset the session again, sent a request before the activation, or sent an Activate of +/// another transaction, or one that changes the accepted record. +pub const ACTIVATION_PROTOCOL: u8 = 51; +/// The Activate's record is refused: it does not decode, breaks a limit, or is a cold boot's, +/// without the restore section of a clone's. +pub const ACTIVATION_RECORD: u8 = 52; +/// The Activate's record expects other resources than the repaired clone has: another number of +/// online CPUs, or another memory size. +pub const ACTIVATION_RESOURCES: u8 = 53; +/// The clone's root cannot be activated with the record's settings: its image does not assemble +/// over its upper, a GPT image's partition table is refused, or its scratch is not fresh. +pub const ACTIVATION_ROOT: u8 = 54; +/// The clone's workload cannot be activated with the record's settings: its account is missing +/// or invalid in the image, or its hostname or its machine ID cannot be set. +pub const ACTIVATION_WORKLOAD: u8 = 55; +/// The host export cannot be mounted in the clone. +pub const ACTIVATION_EXPORT: u8 = 56; +/// The record's network settings cannot be applied in the clone: its resolver or its proxy +/// endpoint. +pub const ACTIVATION_NETWORK: u8 = 57; +/// The guest's own order misused the activation: it completed an activation that did not run, +/// or started the sandbox before the clone was ready. +pub const ACTIVATION_ORDER: u8 = 58; + +/// The time ABI's statuses, from [`TIME_CONFORMANCE`] to [`TIME_REPAIR`]. +pub const TIME_STATUSES: RangeInclusive = TIME_CONFORMANCE..=TIME_REPAIR; +/// The statuses of a clone's repair besides the time ABI's, from [`REPAIR_RESTORE`] to +/// [`REPAIR_FINISH`]. +pub const REPAIR_STATUSES: RangeInclusive = REPAIR_RESTORE..=REPAIR_FINISH; +/// The statuses of a clone's activation, from [`ACTIVATION_DEADLINE`] to [`ACTIVATION_ORDER`]. +pub const ACTIVATION_STATUSES: RangeInclusive = ACTIVATION_DEADLINE..=ACTIVATION_ORDER; + +/// Returns what `status` means, as the module tells, such as `a clone's memory activation failed`, +/// or `None` for a status that the module does not name. +pub fn describe(status: u8) -> Option<&'static str> { + Some(match status { + TIME_CONFORMANCE => "a time ABI conformance check failed", + TIME_VIOLATION => "the time ABI's violation watcher observed a violation", + TIME_REPAIR => "the time ABI's part of a restore's repair failed", + REPAIR_RESTORE => "a clone's restore cannot be repaired", + REPAIR_PROCESSORS => "a clone's online CPUs do not meet its restore", + REPAIR_MEMORY_TARGET => "a clone's memory target is refused", + REPAIR_MEMORY => "a clone's memory activation failed", + REPAIR_RESEED => "a clone's reseed failed", + REPAIR_FINISH => "a clone's restore finish failed", + ACTIVATION_DEADLINE => "a clone was not activated before its deadline", + ACTIVATION_PROTOCOL => "the host broke a repaired clone's restricted session", + ACTIVATION_RECORD => "a clone's Activate record is refused", + ACTIVATION_RESOURCES => "a clone's Activate record expects other resources", + ACTIVATION_ROOT => "a clone's root activation failed", + ACTIVATION_WORKLOAD => "a clone's workload activation failed", + ACTIVATION_EXPORT => "a clone's host export cannot be mounted", + ACTIVATION_NETWORK => "a clone's network settings cannot be applied", + ACTIVATION_ORDER => "a guest's order misused a clone's activation", + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every status that the module names, in its order. + const ALL: [u8; 18] = [ + TIME_CONFORMANCE, + TIME_VIOLATION, + TIME_REPAIR, + REPAIR_RESTORE, + REPAIR_PROCESSORS, + REPAIR_MEMORY_TARGET, + REPAIR_MEMORY, + REPAIR_RESEED, + REPAIR_FINISH, + ACTIVATION_DEADLINE, + ACTIVATION_PROTOCOL, + ACTIVATION_RECORD, + ACTIVATION_RESOURCES, + ACTIVATION_ROOT, + ACTIVATION_WORKLOAD, + ACTIVATION_EXPORT, + ACTIVATION_NETWORK, + ACTIVATION_ORDER, + ]; + + #[test] + fn the_statuses_are_pinned() { + assert_eq!( + ALL, + [ + 193, 194, 195, 40, 41, 42, 43, 44, 45, 50, 51, 52, 53, 54, 55, 56, 57, 58 + ] + ); + assert_eq!(TIME_STATUSES, 193..=195); + assert_eq!(REPAIR_STATUSES, 40..=45); + assert_eq!(ACTIVATION_STATUSES, 50..=58); + } + + #[test] + fn each_status_has_a_class_and_a_meaning_of_its_own() { + let mut meanings = Vec::new(); + for status in ALL { + let classes = [&TIME_STATUSES, &REPAIR_STATUSES, &ACTIVATION_STATUSES] + .into_iter() + .filter(|statuses| statuses.contains(&status)) + .count(); + assert_eq!(classes, 1, "{status}"); + meanings.push(describe(status).unwrap()); + } + meanings.sort_unstable(); + meanings.dedup(); + assert_eq!(meanings.len(), ALL.len()); + // The ranges hold no other status, and the module names no other. + for status in 0..=u8::MAX { + assert_eq!( + describe(status).is_some(), + ALL.contains(&status), + "{status}" + ); + } + // None is a success, the generic failure, a usage error, a sandbox setup failure, or a + // status that a shell gives a command that it cannot run or that a signal ended. + for reserved in [0, 1, 2, 125, 126, 127, 128, 129, 137, 143, 255] { + assert!(!ALL.contains(&reserved), "{reserved}"); + } + } +} diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index e641f76c0..3d878d95e 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -722,13 +722,24 @@ through `worker::capture::capture_on_terminal`: Closing the worker's fence, `worker::capture::Admission`, acquires it: a close fails while the fence is closed, whoever closed it, and leaves it closed. The -worker of a template's VM admits no execution before the capture, nor, in a -clone, before repair acknowledged the restore. The worker returns from the -capture only once the capture ended without a snapshot, or failed, or in a -clone, which a restore of the snapshot started. It does not repair a clone yet: -the clone ends with its fence still closed and its stall detectors still -suppressed. The worker, and with it the VM, ends with one of these statuses, -after a diagnostic on standard error: +worker of a template's VM admits no execution before the capture. The worker +returns from the capture only once the capture ended without a snapshot, or +failed, or in a clone, which a restore of the snapshot started. In a clone, the +worker repairs the restore next, with `nvx::guest::clone::repair`, behind its +fence, which stays closed: the repair checks the online CPUs, activates the +restore's memory target, reseeds the kernel's random number generator with the +restore's fresh entropy, and finishes the restore, with the acknowledgement +that the host awaits, which hands the deferred time work, the restore checks +and the release of the stall detectors, to the time daemon. NVX's `/init` +started the daemon in the template, with the time ABI's boot step, before the +guest agent, so every clone holds it, and the guest agent's PID 1 collects its +status. The repair takes the time ABI in strict mode, so that a violation of +the ABI fails the repair, rather than stopping the VM, and the worker ends the +VM with the failure's status. The worker does not activate a clone yet: a +clone that it repaired ends at once, with its fence still closed, which may cut +the deferred time work short. A failure of that work stops the VM with the time +ABI's own status, since the daemon enforces the ABI. The worker, and with it +the VM, ends with one of these statuses, after a diagnostic on standard error: | Status | When | | --- | --- | @@ -740,19 +751,34 @@ after a diagnostic on standard error: | 24 | The time ABI's capture checks failed | | 25 | The stall detectors cannot be suppressed | | 26 | The VM generation ID cannot be read | -| 27 | The capture request failed, which leaves the fence closed | +| 27 | The capture request failed, which leaves the fence closed, other than by a violation of the time ABI in a clone | | 28 | The stall detectors' settings cannot be restored once the capture ended without a snapshot | | 29 | A stage of the capture failed that this build of the worker does not know | | 30 | The capture ended without a snapshot, as without a destination, or once OpenVMM rejected it, and the template continued | -| 31 | A restore of the template's snapshot started this VM, a clone, which the worker does not repair yet | +| 31 | A restore of the template's snapshot started this VM, a clone, which the worker repaired, and does not activate yet | +| 40 | The clone's restore cannot be repaired: it holds no fresh entropy, or its capture finished it | +| 41 | The clone's online CPUs cannot be read, are not a prefix of the CPUs, or do not meet the restore's processor target | +| 42 | The restore's memory target is refused, before any memory block is added | +| 43 | The activation of the restore's memory target failed | +| 44 | The reseed of the kernel's random number generator failed | +| 45 | The restore's finish failed, other than by a violation of the time ABI | +| 193 | A conformance check of the time ABI failed in the clone's repair, such as the finish's, which needs a running time daemon | +| 195 | The time ABI's part of the clone's repair failed, such as the validation of the restore packet, or the record of the restore | The statuses from 21 to 29 are `template::CAPTURE_STATUS_BASE` plus the number of the stage that failed, in the order in which -`nvx::guest::clone::CaptureStage` declares the stages. A stage whose failure -may leave the worker some of the permission for the time ABI's ports fails with -the time ABI's port error, and the worker then starts no thread or process -before it ends. A violation of the time ABI in enforcing mode stops the VM with -the time ABI's own status, from 193 to 195, instead. +`nvx::guest::clone::CaptureStage` declares the stages. A capture request that +fails with a violation of the time ABI, which it does in a clone whose restore +the time ABI could not repair, ends with the violation's status instead, as +`worker::capture::capture_status` tells. `nvx::protocol::status` documents +the statuses of a clone's repair, from 40 to 45, which +`nvx::guest::clone::RepairError::status` selects, the time ABI's own, from 193 +to 195, and those that it reserves for a clone's activation, from 50 to 58. A +stage whose failure may leave the worker some of the permission for the time +ABI's ports fails with the time ABI's port error, and the worker then starts no +thread or process before it ends. A violation of the time ABI in enforcing +mode, in the capture's preparation or in the time daemon, stops the VM with the +time ABI's own status, from 193 to 195, instead. ## Host program diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs index 6a57e1b27..205748826 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs @@ -22,9 +22,11 @@ //! session failed or the broker closed the tty, each but the first after a diagnostic on //! standard error. In a template's VM, whose kernel command line names the platform snapshot //! tier, the worker captures the VM before it assembles any root instead, on x86_64, as -//! `runtime_adapter::worker::capture` describes: it exits with 20 to 31, or with 1 once its -//! session failed, each after a diagnostic on standard error, unless a committed capture ended -//! the VM; see `runtime_adapter::template`. +//! `runtime_adapter::worker::capture` describes: it exits with 20 to 30, or, in a clone, which +//! it repairs, with 31 once it repaired the clone, with 40 to 45, 193, or 195 once the repair +//! failed, as `nvx::protocol::status` documents, or with 1 once its session failed, each after a +//! diagnostic on standard error, unless a committed capture ended the VM; see +//! `runtime_adapter::template`. //! - `runtime-adapter-guest idle` sleeps until a signal ends it. //! - `runtime-adapter-guest entries ROOT CGROUP` runs the executions of the worker's `entries` //! probe in the root ROOT, each placed in the cgroup CGROUP, in a mount namespace of its own, and @@ -174,9 +176,10 @@ fn worker(tty: &std::path::Path, sibling: Option) -> ExitCode { } /// Reads the kernel command line through `operations`, before anything else, and captures a -/// template's VM on the tty `tty`, as `runtime_adapter::worker::capture` describes. Returns the -/// status with which the worker ends then, after a diagnostic on standard error, or `None` in a -/// VM that boots cold, whose worker serves the host. +/// template's VM on the tty `tty`, as `runtime_adapter::worker::capture` describes, then repairs +/// a clone that a restore of its snapshot started. Returns the status with which the worker ends +/// then, after a diagnostic on standard error, or `None` in a VM that boots cold, whose worker +/// serves the host. /// /// A command line that the worker cannot read, or that `runtime_adapter::template::boot` refuses, /// ends the worker with `COMMAND_LINE_STATUS`. diff --git a/tests/runtime-adapter/src/template.rs b/tests/runtime-adapter/src/template.rs index 14906297b..86931dc9e 100644 --- a/tests/runtime-adapter/src/template.rs +++ b/tests/runtime-adapter/src/template.rs @@ -20,8 +20,10 @@ //! The worker of a template's VM, and with it the VM, ends with [`COMMAND_LINE_STATUS`] once //! `boot` refused its command line, with [`CAPTURE_STATUS_BASE`] plus the number of the stage of //! the capture that failed, with [`CONTINUED_STATUS`] once the capture ended without a snapshot, -//! and with [`RESTORED_STATUS`] in a clone, which it does not repair yet. A committed capture ends -//! the template's VM before the worker returns. +//! and, in a clone, which it repairs, with [`RESTORED_STATUS`] once it repaired the clone, which it +//! does not activate yet, or with the status of the repair's failure, which +//! `nvx::protocol::status` documents. A committed capture ends the template's VM before the worker +//! returns. use std::{fmt, str}; @@ -87,7 +89,8 @@ pub const CAPTURE_STATUS_BASE: u8 = 20; pub const CONTINUED_STATUS: u8 = 30; /// The status with which the worker of a template's VM ends in a clone, which a restore of the -/// template's snapshot started, and which the worker does not repair yet. +/// template's snapshot started, once the worker repaired it: the worker does not activate a clone +/// yet, so the clone ends at once, with its fence of workload admission still closed. pub const RESTORED_STATUS: u8 = 31; /// How a VM boots, as its kernel command line tells the guest agent. @@ -178,6 +181,8 @@ fn token(parameter: &Parameter<'_>) -> Vec { #[cfg(test)] mod tests { + use nvx::protocol::status; + use super::*; use crate::root::RootStage; @@ -531,5 +536,16 @@ mod tests { assert!(!statuses.contains(&stage.exit_status()), "{stage}"); } assert!(!statuses.contains(&0) && !statuses.contains(&1) && !statuses.contains(&2)); + // Nor is one the time ABI's, or one of a clone's repair or activation. + for reserved in [ + status::TIME_STATUSES, + status::REPAIR_STATUSES, + status::ACTIVATION_STATUSES, + ] { + assert!( + !statuses.iter().any(|own| reserved.contains(own)), + "{reserved:?}" + ); + } } } diff --git a/tests/runtime-adapter/src/worker/capture.rs b/tests/runtime-adapter/src/worker/capture.rs index f85f106ef..3b528c1f5 100644 --- a/tests/runtime-adapter/src/worker/capture.rs +++ b/tests/runtime-adapter/src/worker/capture.rs @@ -19,20 +19,31 @@ //! //! A committed capture ends the template's VM. The worker returns only once the capture ended //! without a snapshot, or failed, or in a clone, which a restore of the snapshot started, as a -//! [`Templated`] tells, and with which the worker ends. The worker does not repair a clone yet: -//! the clone ends too, with its fence still closed. +//! [`Templated`] tells, and with which the worker ends. In a clone, the worker repairs the restore +//! next, with `nvx::guest::clone::repair`, behind its fence, which stays closed: the repair +//! activates the restore's memory target, reseeds the kernel's random number generator, and +//! finishes the restore, with the acknowledgement that the host awaits, which hands the deferred +//! time work to the time daemon. NVX's `/init` started the daemon in the template, with the time +//! ABI's boot step, before the worker, so the snapshot holds it, and PID 1 collects its status. The +//! worker does not activate a clone yet: a clone that it repaired ends at once, with +//! [`RESTORED_STATUS`] and its fence still closed, which may cut the deferred time work short. A +//! repair that failed ends the clone with the [status](nvx::guest::clone::RepairError::status) of +//! its failure, which `nvx::protocol::status` documents, and a capture request that failed in a +//! clone, since the time ABI could not repair its restore, with the status of the time ABI's +//! violation, as [`capture_status`] tells. use std::fmt; use std::io::{self, Read, Write}; use std::path::Path; use nvx::guest::clone::{ - AdmissionFence, CaptureBarrier, CaptureError, CapturePreparation, CaptureStage, Captured, - Devices, Restored, + self, AdmissionFence, CaptureBarrier, CaptureCause, CaptureError, CapturePreparation, + CaptureStage, Captured, Devices, RepairError, Repaired, }; use nvx::guest::nvxs::GuestConnection; use nvx::guest::root::Device; use nvx::guest::terminal::Terminal; +use nvx::guest::time::TimeError; use nvx::protocol::root::{DISTRO_MMIO_BASE, SCRATCH_MMIO_BASE}; use crate::guest::{self, ServeError}; @@ -46,10 +57,11 @@ pub const FILESYSTEM_TAG: &str = "microvm"; /// The worker's fence of workload admission in a template's VM, which it hands the capture's /// preparation. /// -/// The worker of a template's VM admits no execution before the capture, nor, in a clone, before -/// repair acknowledged the restore. Closing the fence acquires it, as `nvx::guest::clone` -/// requires: [`close`](AdmissionFence::close) fails while the fence is closed, whoever closed it, -/// and leaves it closed. +/// The worker of a template's VM admits no execution before the capture, nor in a clone, which it +/// repairs behind the fence, and whose activation would open it, which the worker does not run +/// yet. Closing the fence acquires it, as `nvx::guest::clone` requires: +/// [`close`](AdmissionFence::close) fails while the fence is closed, whoever closed it, and leaves +/// it closed. #[derive(Debug, Default)] pub struct Admission { /// Whether the fence is closed, so that no execution starts. @@ -116,6 +128,27 @@ pub fn stage_status(stage: CaptureStage) -> u8 { } } +/// Returns the status with which the worker of a template's VM ends once its capture failed with +/// `error`: the [status](nvx::guest::time::Violation::status) of the time ABI's violation, 195 or +/// 193, when the request failed with one, as in a clone whose restore the time ABI could not +/// repair, and the [status](stage_status) of the stage that failed otherwise. +pub fn capture_status(error: &CaptureError) -> u8 { + let violation = match error.cause() { + CaptureCause::Time(TimeError::Violation(violation)) => Some(violation.status()), + _ => None, + }; + failure_status(error.stage(), violation) +} + +/// Returns the status of a capture that failed at `stage`, with a violation of the time ABI of +/// the status `violation`, if any, as [`capture_status`] tells. +fn failure_status(stage: CaptureStage, violation: Option) -> u8 { + match (stage, violation) { + (CaptureStage::Capture, Some(status)) => status, + (stage, _) => stage_status(stage), + } +} + /// How the capture of a template's VM ended, once the worker attached its session. #[derive(Debug)] pub enum Templated { @@ -123,10 +156,17 @@ pub enum Templated { /// it, and the template continued: the barrier opened the fence, and restored the stall /// detectors' settings. Continued, - /// A restore of the template's snapshot started this VM, a clone, whose wall clock the time - /// ABI repaired, with the fence still closed and the stall detectors still suppressed, as - /// `Restored` holds them for repair, which the worker does not run yet. - Restored(Restored), + /// A restore of the template's snapshot started this VM, a clone, which the worker repaired, + /// and does not activate yet. + Repaired { + /// The repaired clone, with its resources. + repaired: Repaired, + /// The worker's fence of workload admission, still closed: opening it is the activation's. + fence: Admission, + }, + /// A restore of the template's snapshot started this VM, a clone, whose repair failed. The + /// fence stays closed. + Unrepaired(RepairError), /// A stage of the capture failed. The preparation released what it took, and a failed /// request left the fence closed. Failed(CaptureError), @@ -134,12 +174,15 @@ pub enum Templated { impl Templated { /// Returns the status with which the worker, and with it the VM, ends: [`CONTINUED_STATUS`], - /// [`RESTORED_STATUS`], or the [status](stage_status) of the stage that failed. + /// [`RESTORED_STATUS`] once it repaired a clone, the + /// [status](nvx::guest::clone::RepairError::status) of a repair that failed, or the + /// [status](capture_status) of a capture that failed. pub fn status(&self) -> u8 { match self { Self::Continued => CONTINUED_STATUS, - Self::Restored(_) => RESTORED_STATUS, - Self::Failed(error) => stage_status(error.stage()), + Self::Repaired { .. } => RESTORED_STATUS, + Self::Unrepaired(error) => error.status(), + Self::Failed(error) => capture_status(error), } } } @@ -150,9 +193,18 @@ impl fmt::Display for Templated { Self::Continued => formatter.write_str( "the template's capture ended without a snapshot, and the template continued", ), - Self::Restored(_) => formatter.write_str( - "a restore of the template's snapshot started this clone, which the worker does \ - not repair", + Self::Repaired { repaired, .. } => write!( + formatter, + "a restore of the template's snapshot started this clone, which the worker \ + repaired, with {} online CPUs and {} bytes of memory, and does not activate", + repaired.cpus(), + repaired.memory() + ), + Self::Unrepaired(error) => write!( + formatter, + "the repair of this clone, which a restore of the template's snapshot started, \ + {error} ({})", + error.progress() ), Self::Failed(error) => error.fmt(formatter), } @@ -160,7 +212,8 @@ impl fmt::Display for Templated { } /// Captures the template's VM on its control tty at `tty`, as the [module documentation](self) -/// describes, and returns how the capture ended. +/// describes, and returns how the capture ended, once the worker repaired a clone that a restore +/// of the snapshot started. /// /// It opens the tty through [`guest::on_terminal`], as the worker of a VM that boots cold does, /// and fails as that does if the tty cannot be opened or put in raw mode. It fails with @@ -178,7 +231,14 @@ pub fn capture_on_terminal(tty: &Path) -> Result { let preparation = CapturePreparation::new(Admission::default(), devices()); match preparation.prepare().and_then(CaptureBarrier::capture) { Ok(Captured::Continued) => Templated::Continued, - Ok(Captured::Restored(restored)) => Templated::Restored(restored), + Ok(Captured::Restored(restored)) => { + // The repair runs behind the fence, which stays closed. + let (restore, fence) = restored.into_parts(); + match clone::repair(restore) { + Ok(repaired) => Templated::Repaired { repaired, fence }, + Err(error) => Templated::Unrepaired(error), + } + } Err(error) => Templated::Failed(error), } }) @@ -208,6 +268,7 @@ mod tests { use nvx::guest::nvxs::GuestConnectionError; use nvx::protocol::nvxs::GuestSession; + use nvx::protocol::status; use super::*; use crate::root::RootStage; @@ -248,13 +309,15 @@ mod tests { ]; for (stage, status) in stages { assert_eq!(stage_status(stage), status, "{stage}"); + assert_eq!(failure_status(stage, None), status, "{stage}"); } assert_eq!(Templated::Continued.status(), CONTINUED_STATUS); assert_eq!( Templated::Continued.to_string(), "the template's capture ended without a snapshot, and the template continued" ); - // No two outcomes of the worker share a status. + // No two outcomes of the worker share a status, nor one of a clone's repair or + // activation. let mut statuses = stages.map(|(_, status)| status).to_vec(); statuses.extend([ COMMAND_LINE_STATUS, @@ -264,12 +327,38 @@ mod tests { 0, ]); statuses.extend(RootStage::ALL.map(RootStage::exit_status)); + statuses.extend(status::TIME_STATUSES); + statuses.extend(status::REPAIR_STATUSES); + statuses.extend(status::ACTIVATION_STATUSES); let count = statuses.len(); statuses.sort_unstable(); statuses.dedup(); assert_eq!(statuses.len(), count); } + #[test] + fn a_request_that_the_time_abi_could_not_repair_ends_with_the_violation_s_status() { + // In a clone, whose restore the time ABI's part of the repair refused. + assert_eq!( + failure_status(CaptureStage::Capture, Some(status::TIME_REPAIR)), + 195 + ); + assert_eq!( + failure_status(CaptureStage::Capture, Some(status::TIME_CONFORMANCE)), + 193 + ); + // A violation of the preparation, which only a report-only time ABI returns, keeps the + // stage's status. + assert_eq!( + failure_status(CaptureStage::BootChecks, Some(status::TIME_CONFORMANCE)), + 23 + ); + assert_eq!( + failure_status(CaptureStage::CaptureChecks, Some(status::TIME_CONFORMANCE)), + 24 + ); + } + #[test] fn a_template_s_devices_are_those_of_a_root_and_the_host_filesystem() { let devices = devices();