diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index 3a380cccd..e27bf8e03 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -36,9 +36,9 @@ Each configuration enables one side of `nvx`: | Configuration | Cargo options | `nvx` features | Modules | Binary | | --- | --- | --- | --- | --- | -| Protocol-only | none | none | `frame`, `root`, `probe`, `exec` | none | -| Host-only | `--features host` | `host` | `frame`, `root`, `probe`, `exec`, `host`, `runtime`, `initramfs`, `staged`, `sharing`, `layered`, `hook` | `runtime-adapter-host` | -| Guest-only | `--features guest --target x86_64-unknown-linux-musl` | `guest` | `frame`, `root`, `probe`, `exec`, `guest`, `worker` | `runtime-adapter-guest`, a static executable | +| Protocol-only | none | none | `frame`, `root`, `probe`, `exec`, `template` | none | +| Host-only | `--features host` | `host` | `frame`, `root`, `probe`, `exec`, `template`, `host`, `runtime`, `initramfs`, `staged`, `sharing`, `layered`, `hook` | `runtime-adapter-host` | +| Guest-only | `--features guest --target x86_64-unknown-linux-musl` | `guest` | `frame`, `root`, `probe`, `exec`, `template`, `guest`, `worker` | `runtime-adapter-guest`, a static executable | ## Framing @@ -664,6 +664,90 @@ or with 1 after a diagnostic on standard error; and from inside the root, and which reports what the execution sees and may do; see [Probes](#probes). +### Template mode + +A template's VM captures a snapshot of its guest before the guest assembles any +root, so that every clone that a restore of the snapshot starts attaches a root +of its own. The host opens no application session with it, and waits for no +`Ready`: a committed capture ends the VM. Its kernel command line has +`nvx_sandbox=1`, so NVX's `/init` runs the time ABI's boot step, then hands the +VM to the guest agent's PID 1 of a layered root, which starts the worker, as in +every VM with a layered root. OpenVMM adds `nvx_snapshot_tier=platform` to the +kernel command line of a VM that captures a platform snapshot. + +On x86_64, whose time ABI a capture needs, the worker reads the kernel command +line before anything else, with `template::boot`, which splits it as the kernel +does, with `nvx::protocol::cmdline::parameters`. A command line in which the +kernel finds no parameter `nvx_snapshot_tier=platform` boots cold: the worker +goes on as above, whatever else the command line holds. With it, the VM is a +template's, whose snapshot holds the platform's invariant configuration alone, +which every clone shares. So the worker accepts only the parameters of the +platform tier's allowlist, `template::is_platform_token`, which OpenVMM's +capture of a platform snapshot allows too, byte for byte: + +- `earlycon=xe9`, `console=hvc0`, `console=hvc1`, `reboot=t`, `panic=-1`, + `nvx_sandbox=1`, `nvx_config=0xd0010000,65536`, + `nvx_snapshot_tier=platform`, and `nvx_control_tty=hvc2`; +- `nr_cpus=`, `maxcpus=`, `virtio_mmio.device=`, `virtnet_ip=`, + `virtnet_mask=`, `virtnet_gw=`, `virtnet_dns=`, `virtnet_ip6=`, and + `virtnet_gw6=`, with any value that the host chooses. + +It refuses any other parameter, and a double quote, which the kernel drops from +a parameter, or `--`, after which the kernel hands the rest of the command line +to init, though init may still read it from `/proc/cmdline`: OpenVMM refuses +both in the command line of a VM with a control console. Otherwise, the worker, +through `worker::capture::capture_on_terminal`: + +1. opens the control tty, in raw mode, as the worker of a VM that boots cold + does; +2. attaches the control session with `nvx::guest::nvxs::GuestConnection::attach`, + and waits until the tty has sent the attachment, so that the snapshot holds + the guest attached. It reads nothing from the session before the capture, + and serves no application session; +3. prepares the guest for the capture with + `nvx::guest::clone::CapturePreparation`, which closes the worker's fence of + workload admission, checks that nothing is mounted from the `distro` or the + scratch block device, at the MMIO windows from which the worker assembles a + root, from an overlay, or from OpenVMM's host filesystem device, whose tag + is `microvm`, then waits for the time ABI's boot checks, runs its capture + checks, and suppresses the stall detectors; +4. requests the capture with `CaptureBarrier::capture`, with the platform + tier's request, which keeps no scratch. + +Closing the worker's fence, `worker::capture::Admission`, acquires it: a close +fails while the fence is closed, whoever closed it, and leaves it closed. The +worker of a template's VM admits no execution before the capture, nor, in a +clone, before repair acknowledged the restore. The worker returns from the +capture only once the capture ended without a snapshot, or failed, or in a +clone, which a restore of the snapshot started. It does not repair a clone yet: +the clone ends with its fence still closed and its stall detectors still +suppressed. The worker, and with it the VM, ends with one of these statuses, +after a diagnostic on standard error: + +| Status | When | +| --- | --- | +| 1 | The control tty cannot be opened or put in raw mode, or the attachment cannot be sent | +| 20 | The kernel command line names the platform tier, but holds another parameter, a double quote, or `--`, or the worker cannot read it | +| 21 | The fence of workload admission cannot be closed | +| 22 | Something is mounted from the template's devices or from an overlay, or the mount table cannot be checked | +| 23 | The time ABI's boot checks did not complete | +| 24 | The time ABI's capture checks failed | +| 25 | The stall detectors cannot be suppressed | +| 26 | The VM generation ID cannot be read | +| 27 | The capture request failed, which leaves the fence closed | +| 28 | The stall detectors' settings cannot be restored once the capture ended without a snapshot | +| 29 | A stage of the capture failed that this build of the worker does not know | +| 30 | The capture ended without a snapshot, as without a destination, or once OpenVMM rejected it, and the template continued | +| 31 | A restore of the template's snapshot started this VM, a clone, which the worker does not repair yet | + +The statuses from 21 to 29 are `template::CAPTURE_STATUS_BASE` plus the number +of the stage that failed, in the order in which +`nvx::guest::clone::CaptureStage` declares the stages. A stage whose failure +may leave the worker some of the permission for the time ABI's ports fails with +the time ABI's port error, and the worker then starts no thread or process +before it ends. A violation of the time ABI in enforcing mode stops the VM with +the time ABI's own status, from 193 to 195, instead. + ## Host program `host::Client` is the host program's client of the guest agent. It runs over @@ -1214,7 +1298,7 @@ independent adapter's does: guest binary is a static executable: it has no program interpreter, needs no shared library, and keeps its stack non-executable. -The unit tests of `frame`, `root`, `probe`, and `exec` run in every +The unit tests of `frame`, `root`, `probe`, `exec`, and `template` run in every configuration. They cover each kind's encoding, messages in pieces of any size, a message of each kind split at every byte, the headers that the framing refuses before their body arrives, and the root configurations, readinesses, parts of @@ -1222,7 +1306,16 @@ mapping tables, failures, probes, facts, and messages of executions that decode, and those that don't, and the host's side of an execution, which still sends the cancellation of one that it gives up on in the step that ends it, without grace or settlement time, and which cancels one whose stream outgrows the 16 MiB that -it keeps. In the guest-only configuration, the unit tests of `worker::probes` +it keeps. Those of `template` check the platform tier's allowlist, token by +token, against the tokens that OpenVMM's capture of a platform snapshot allows +and refuses, and template mode: a command line in which the kernel finds no +platform tier boots cold, whatever else it holds, and one with it boots a +template's VM if it holds the tier's tokens alone, between any of the kernel's +whitespace, and is refused otherwise, for a parameter outside the allowlist at +its start, in its middle, or at its end, or within a platform token's value, +after a vertical tab or the byte 0xa0, which the kernel takes for whitespace +and a split at ASCII whitespace does not, for a double quote, and for `--`. In +the guest-only configuration, the unit tests of `worker::probes` cover what the probes read: the mount table with its escapes, the kernel's records of a process, `os-release`, and the guest's memory, and the probes and arguments that the worker refuses. Those of @@ -1237,7 +1330,12 @@ with a real-time signal, and check that PID 1 takes both stops and goes on until it collects the worker's status. Those of `worker::entries` cover the names of capabilities, the fields of a process's status, the outcomes of attempts, the entry's refusals, and the -probe's arguments. +probe's arguments. Those of `worker::capture` check that closing the worker's +fence acquires it, the status of each stage of a capture, which no other +outcome of the worker shares, a template's devices, and that the worker +attaches its session, and the stream sent the attachment, before the capture +runs, without reading anything of the session, and that no capture runs once +the attachment cannot be written or sent. `tests/guest_session.rs` checks, in the guest-only configuration, that the guest agent serves the control session with the adapter's own framing. A test diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs index 761c4c460..6a57e1b27 100644 --- a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs +++ b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs @@ -20,7 +20,11 @@ //! SIBLING is the process ID of PID 1's unrelated child, which its probes check. It exits with 0 //! once the host stopped it, with 11 to 16 once a stage of the root failed, and with 1 once its //! session failed or the broker closed the tty, each but the first after a diagnostic on -//! standard error. +//! standard error. In a template's VM, whose kernel command line names the platform snapshot +//! tier, the worker captures the VM before it assembles any root instead, on x86_64, as +//! `runtime_adapter::worker::capture` describes: it exits with 20 to 31, or with 1 once its +//! session failed, each after a diagnostic on standard error, unless a committed capture ended +//! the VM; see `runtime_adapter::template`. //! - `runtime-adapter-guest idle` sleeps until a signal ends it. //! - `runtime-adapter-guest entries ROOT CGROUP` runs the executions of the worker's `entries` //! probe in the root ROOT, each placed in the cgroup CGROUP, in a mount namespace of its own, and @@ -135,12 +139,17 @@ fn init(tty: &std::path::Path) -> ExitCode { } /// Runs the worker on the tty `tty`, with the process ID of PID 1's unrelated child, if any, and -/// ends with its status. +/// ends with its status. In a template's VM, the worker captures the VM instead; see +/// `capture_template`. #[cfg(target_os = "linux")] fn worker(tty: &std::path::Path, sibling: Option) -> ExitCode { use runtime_adapter::worker::{self, LibraryRoot, Worked}; let mut operations = LibraryRoot::new(sibling); + #[cfg(target_arch = "x86_64")] + if let Some(status) = capture_template(tty, &mut operations) { + return ExitCode::from(status); + } let worked = worker::work_on_terminal(tty, &mut operations); let status = match worked { Ok(worked) => { @@ -164,6 +173,45 @@ fn worker(tty: &std::path::Path, sibling: Option) -> ExitCode { ExitCode::from(status) } +/// Reads the kernel command line through `operations`, before anything else, and captures a +/// template's VM on the tty `tty`, as `runtime_adapter::worker::capture` describes. Returns the +/// status with which the worker ends then, after a diagnostic on standard error, or `None` in a +/// VM that boots cold, whose worker serves the host. +/// +/// A command line that the worker cannot read, or that `runtime_adapter::template::boot` refuses, +/// ends the worker with `COMMAND_LINE_STATUS`. +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +fn capture_template( + tty: &std::path::Path, + operations: &mut runtime_adapter::worker::LibraryRoot, +) -> Option { + use runtime_adapter::template::{self, Boot, COMMAND_LINE_STATUS}; + use runtime_adapter::worker::capture; + use runtime_adapter::worker::{RootOperations, SESSION_STATUS}; + + let boot = operations + .command_line() + .and_then(|command_line| template::boot(&command_line).map_err(|error| error.to_string())); + match boot { + Ok(Boot::Cold) => return None, + Ok(Boot::Template) => {} + Err(problem) => { + eprintln!("runtime-adapter-guest: {problem}"); + return Some(COMMAND_LINE_STATUS); + } + } + Some(match capture::capture_on_terminal(tty) { + Ok(templated) => { + eprintln!("runtime-adapter-guest: {templated}"); + templated.status() + } + Err(error) => { + eprintln!("runtime-adapter-guest: {error}"); + SESSION_STATUS + } + }) +} + /// Runs the executions of the worker's `entries` probe in the root `root`, each placed in the cgroup /// `workload`, and reports their facts. #[cfg(target_os = "linux")] diff --git a/tests/runtime-adapter/src/lib.rs b/tests/runtime-adapter/src/lib.rs index 885a65214..85d6c345f 100644 --- a/tests/runtime-adapter/src/lib.rs +++ b/tests/runtime-adapter/src/lib.rs @@ -5,8 +5,9 @@ //! Each configuration enables one side of `nvx`: //! //! - protocol-only, without features: the control protocol, the contract versions, and the other -//! items that `nvx` has without features, with the adapter's own [`frame`]s, and the bodies of -//! its messages about a layered [`root`], its [`probe`]s, and [`exec`]utions; +//! items that `nvx` has without features, with the adapter's own [`frame`]s, the bodies of its +//! messages about a layered [`root`], its [`probe`]s, and [`exec`]utions, and the kernel command +//! line and statuses of its [`template`] mode; //! - host-only, with the `host` feature: the host API too, and the host program, in the `host`, //! `runtime`, `initramfs`, `staged`, `sharing`, `layered`, and `hook` modules and the //! `runtime-adapter-host` binary; @@ -43,6 +44,10 @@ //! and its guest binding, from one loop that never blocks. The executions enter the root with the //! publication of listeners that the root configuration names, and the host gives them the URL of //! the network's proxy, if any, in their environment. +//! +//! A VM whose kernel command line names the platform snapshot tier is a template's, which the +//! worker captures before it assembles any root, so that every clone of the snapshot attaches a +//! root of its own, in [`template`] mode. pub mod exec; pub mod frame; @@ -64,6 +69,7 @@ pub mod runtime; pub mod sharing; #[cfg(feature = "host")] pub mod staged; +pub mod template; #[cfg(all(feature = "guest", target_os = "linux"))] pub mod worker; diff --git a/tests/runtime-adapter/src/template.rs b/tests/runtime-adapter/src/template.rs new file mode 100644 index 000000000..a60449c7e --- /dev/null +++ b/tests/runtime-adapter/src/template.rs @@ -0,0 +1,489 @@ +//! Template mode: the guest agent of a template's VM captures the VM before it assembles any root, +//! so that every clone that a restore of the snapshot starts attaches a root of its own. This +//! module holds what the host program and the guest agent share of it: how the guest agent tells +//! a template's VM by its kernel command line, the platform tier's allowlist of that command line, +//! and the statuses with which the guest agent of a template's VM ends. The worker captures the +//! VM, in `worker::capture`. +//! +//! OpenVMM adds [`PLATFORM_TIER`], `nvx_snapshot_tier=platform`, to the kernel command line of a +//! VM that captures a platform snapshot. [`boot`] reads the command line as the kernel splits it, +//! with `nvx::protocol::cmdline::parameters`: a command line without that parameter boots cold, +//! as before, whatever else it holds. A platform snapshot holds the platform's invariant +//! configuration alone, which every clone shares, so with that parameter, `boot` refuses any +//! parameter that [`is_platform_token`] does not accept: the platform tier's allowlist, the same +//! as OpenVMM's capture of a platform snapshot allows. The kernel drops double quotes from a +//! parameter, and hands what follows `--` to init rather than taking it as parameters, though init +//! may still read it from `/proc/cmdline`, so `boot` also refuses a template's command line that +//! holds either, as OpenVMM refuses them in the command line of a VM with a control console. +//! +//! The worker of a template's VM, and with it the VM, ends with [`COMMAND_LINE_STATUS`] once +//! `boot` refused its command line, with [`CAPTURE_STATUS_BASE`] plus the number of the stage of +//! the capture that failed, with [`CONTINUED_STATUS`] once the capture ended without a snapshot, +//! and with [`RESTORED_STATUS`] in a clone, which it does not repair yet. A committed capture ends +//! the template's VM before the worker returns. + +use std::fmt; + +use nvx::protocol::cmdline::{self, Parameter}; + +/// The parameter that names the platform snapshot tier, which OpenVMM adds to the kernel command +/// line of a template's VM. +pub const PLATFORM_TIER: &str = "nvx_snapshot_tier=platform"; + +/// The tokens of the platform tier's allowlist that it names whole: the consoles, the reboot +/// and panic settings, the sandbox's guest mode, NVX's configuration window, the tier itself, and +/// the control tty. +const PLATFORM_TOKENS: [&str; 9] = [ + "earlycon=xe9", + "console=hvc0", + "console=hvc1", + "reboot=t", + "panic=-1", + "nvx_sandbox=1", + "nvx_config=0xd0010000,65536", + PLATFORM_TIER, + "nvx_control_tty=hvc2", +]; + +/// The prefixes of the platform tier's tokens whose values the host chooses: the processor +/// capacity, the number of processors online at boot, the virtio-mmio devices, and the addresses +/// of the VM's network, if it has one. +const PLATFORM_PREFIXES: [&str; 9] = [ + "nr_cpus=", + "maxcpus=", + "virtio_mmio.device=", + "virtnet_ip=", + "virtnet_mask=", + "virtnet_gw=", + "virtnet_dns=", + "virtnet_ip6=", + "virtnet_gw6=", +]; + +/// The status with which the worker of a template's VM, and with it the VM, ends once [`boot`] +/// refused its kernel command line, or the worker could not read it. +pub const COMMAND_LINE_STATUS: u8 = 20; + +/// The base of the statuses with which the worker of a template's VM ends once a stage of its +/// capture failed: this base plus the stage's number, in the order in which +/// `nvx::guest::clone::CaptureStage` declares the stages, from 21 for the fence to 28 for the +/// release, or 29 for a stage that the worker does not know. +pub const CAPTURE_STATUS_BASE: u8 = 20; + +/// The status with which the worker of a template's VM ends once the capture ended without a +/// snapshot, and the template continued. +pub const CONTINUED_STATUS: u8 = 30; + +/// The status with which the worker of a template's VM ends in a clone, which a restore of the +/// template's snapshot started, and which the worker does not repair yet. +pub const RESTORED_STATUS: u8 = 31; + +/// How a VM boots, as its kernel command line tells the guest agent. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Boot { + /// A VM that serves the host program: its kernel command line does not name the platform + /// tier. + Cold, + /// A template's VM, which captures before it assembles any root: its kernel command line + /// names the platform tier, and holds that tier's tokens alone. + Template, +} + +/// Why [`boot`] refused the kernel command line of a template's VM. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CommandLineError { + /// A parameter that the platform tier's allowlist does not hold, as the command line spells + /// it. + Token(Vec), + /// A double quote, which the kernel drops from a parameter. + Quote, + /// A `--`, after which the kernel hands the rest of the command line to init. + Delimiter, +} + +impl fmt::Display for CommandLineError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .write_str("the kernel command line names the platform snapshot tier, but holds ")?; + match self { + Self::Token(token) => write!( + formatter, + "`{}`, which the tier's allowlist does not hold", + token.escape_ascii() + ), + Self::Quote => formatter.write_str("a double quote, which the kernel drops"), + Self::Delimiter => { + formatter.write_str("`--`, after which the kernel hands the rest to init") + } + } + } +} + +impl std::error::Error for CommandLineError {} + +/// Returns how a VM boots, as its kernel command line, `command_line`, such as `/proc/cmdline`, +/// tells, as the [module documentation](self) describes: a command line names the platform tier +/// if the kernel finds [`PLATFORM_TIER`] among its parameters. +/// +/// Fails with a [`CommandLineError`] if the command line names the platform tier, but holds a +/// double quote, or else a `--`, or else a parameter that [`is_platform_token`] does not accept, +/// the first in the command line's order. +pub fn boot(command_line: &[u8]) -> Result { + let template = cmdline::parameters(command_line) + .any(|parameter| token(¶meter) == PLATFORM_TIER.as_bytes()); + if !template { + return Ok(Boot::Cold); + } + if command_line.contains(&b'"') { + return Err(CommandLineError::Quote); + } + // Without quotes, the kernel splits the command line at its whitespace alone. + if command_line + .split(|&byte| cmdline::is_space(byte)) + .any(|word| word == b"--") + { + return Err(CommandLineError::Delimiter); + } + match cmdline::parameters(command_line) + .map(|parameter| token(¶meter)) + .find(|token| !is_platform_token(token)) + { + Some(token) => Err(CommandLineError::Token(token)), + None => Ok(Boot::Template), + } +} + +/// Returns whether `token`, a parameter of a kernel command line as the command line spells it, +/// is one of the platform tier's: `earlycon=xe9`, `console=hvc0`, `console=hvc1`, `reboot=t`, +/// `panic=-1`, `nvx_sandbox=1`, `nvx_config=0xd0010000,65536`, [`PLATFORM_TIER`], or +/// `nvx_control_tty=hvc2`, or a token that starts with `nr_cpus=`, `maxcpus=`, +/// `virtio_mmio.device=`, `virtnet_ip=`, `virtnet_mask=`, `virtnet_gw=`, `virtnet_dns=`, +/// `virtnet_ip6=`, or `virtnet_gw6=`. +/// +/// These are the tokens that OpenVMM's capture of a platform snapshot allows, byte for byte, so it +/// compares names exactly, though the kernel takes a dash in a parameter's name for an +/// underscore. +pub fn is_platform_token(token: &[u8]) -> bool { + PLATFORM_TOKENS + .iter() + .any(|whole| token == whole.as_bytes()) + || PLATFORM_PREFIXES + .iter() + .any(|prefix| token.starts_with(prefix.as_bytes())) +} + +/// Returns `parameter` as the command line spells it, without quotes: its name, then `=` and its +/// value, if it has one. +fn token(parameter: &Parameter<'_>) -> Vec { + let mut token = parameter.name.to_vec(); + if let Some(value) = parameter.value { + token.push(b'='); + token.extend_from_slice(value); + } + token +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::root::RootStage; + + /// The kernel command line of a template's VM, as OpenVMM writes it for a platform capture + /// without a network, with the newline that ends `/proc/cmdline`. + const TEMPLATE: &str = "earlycon=xe9 console=hvc0 reboot=t panic=-1 nvx_sandbox=1 \ + nvx_config=0xd0010000,65536 nvx_snapshot_tier=platform nr_cpus=4 \ + virtio_mmio.device=0x1000@0xd0001000:6 virtio_mmio.device=0x1000@0xd0003000:7 \ + virtio_mmio.device=0x1000@0xd0006000:10 virtio_mmio.device=0x1000@0xd0007000:11 \ + nvx_control_tty=hvc2\n"; + + /// The tokens of a template's VM with a network, which OpenVMM adds to [`TEMPLATE`]. + const NETWORK: &str = "virtio_mmio.device=0x1000@0xd0000000:5 virtnet_ip=10.0.2.15 \ + virtnet_mask=255.255.255.0 virtnet_gw=10.0.2.2 virtnet_dns=10.0.2.3,10.0.2.4 \ + virtnet_ip6=fd00::15/64 virtnet_gw6=fd00::2"; + + /// Returns [`TEMPLATE`] with `token` at the start, in the middle, and at the end. + fn with(token: &str) -> [String; 3] { + let template = TEMPLATE.trim_end(); + let middle = template.find(" nvx_snapshot_tier").unwrap(); + [ + format!("{token} {template}\n"), + format!("{} {token}{}\n", &template[..middle], &template[middle..]), + format!("{template} {token}\n"), + ] + } + + #[test] + fn the_platform_tier_s_allowlist_is_openvmm_s() { + for token in PLATFORM_TOKENS { + assert!(is_platform_token(token.as_bytes()), "{token}"); + } + for token in [ + "nr_cpus=1", + "nr_cpus=8", + "nr_cpus=", + "maxcpus=1", + "maxcpus=4", + "maxcpus=", + "virtio_mmio.device=0x1000@0xd0003000:7", + "virtio_mmio.device=4K@0xd0003000:7:2", + "virtnet_ip=10.0.2.15", + "virtnet_mask=255.255.255.0", + "virtnet_gw=10.0.2.2", + "virtnet_dns=10.0.2.3,10.0.2.4", + "virtnet_dns=", + "virtnet_ip6=fd00::15/64", + "virtnet_gw6=fd00::2", + ] { + assert!(is_platform_token(token.as_bytes()), "{token}"); + } + for token in [ + "", + "earlycon=xe9,keep", + "earlycon", + "console=hvc2", + "console=ttyS0", + "reboot=k", + "panic=0", + "nvx_sandbox=0", + "nvx_config=0xd0010000,65537", + "nvx_config=0xd0010000", + "nvx_snapshot_tier=workload-start", + "nvx_snapshot_tier=instance-checkpoint", + "nvx-snapshot-tier=platform", + "nvx_snapshot_tier=platform2", + "nvx_control_tty=hvc1", + "nvx-control-tty=hvc2", + "nr_cpus", + "NR_CPUS=4", + "nr-cpus=4", + "maxcpus", + "MAXCPUS=1", + "max_cpus=1", + "virtio_mmio.device", + "virtio-mmio.device=0x1000@0xd0003000:7", + "virtnet_ingress=allow", + "virtnet_probe=10.0.2.2", + "virtfs_dir=/run/nvx/hostfs/root", + "virtfs_tag=microvm", + "virtfs_mode=ro", + "virtfs_aggregate=1", + "nvx_workload_uid=1000", + "nvx_workload_gid=1000", + "nvx_lifecycle=managed", + "nvx_exec=/bin/sh", + "nvx_share=0,/data", + "driver_async_probe=virtio_mmio", + "init=/bin/sh", + "rdinit=/bin/sh", + "quiet", + "--", + "--=1", + ] { + assert!(!is_platform_token(token.as_bytes()), "{token}"); + } + } + + #[test] + fn a_command_line_without_the_platform_tier_boots_cold() { + for command_line in [ + "", + "\n", + "earlycon=xe9 console=hvc0 reboot=t panic=-1 nvx_sandbox=1 nvx_control_tty=hvc2\n", + // A layered root's tokens, which a template's command line never holds. + "nvx_sandbox=1 virtfs_tag=microvm quiet init=/bin/sh\n", + "nvx_snapshot_tier=workload-start nvx_sandbox=1 quiet\n", + "nvx-snapshot-tier=platform quiet\n", + "nvx_snapshot_tier=platform2 quiet\n", + "nvx_snapshot_tier quiet\n", + // A cold boot takes quotes and the delimiter as before. + "nvx_sandbox=1 note=\"left right\" -- extra\n", + // The kernel hands the tier after the delimiter to init, as an argument. + "nvx_sandbox=1 -- nvx_snapshot_tier=platform\n", + // The kernel stops at a NUL. + "nvx_sandbox=1\0nvx_snapshot_tier=platform\n", + ] { + assert_eq!( + boot(command_line.as_bytes()), + Ok(Boot::Cold), + "{command_line:?}" + ); + } + } + + #[test] + fn a_template_s_command_line_holds_the_platform_tier_s_tokens_alone() { + let network = TEMPLATE.replacen( + " nvx_control_tty", + &format!(" {NETWORK} nvx_control_tty"), + 1, + ); + for command_line in [ + TEMPLATE.to_owned(), + TEMPLATE.trim_end().to_owned(), + network, + // A template that brings fewer processors online at boot than its capacity. + TEMPLATE.replacen(" nvx_control_tty", " maxcpus=1 nvx_control_tty", 1), + // The tier alone, and each token whole once more, which OpenVMM leaves to its own + // checks. + format!("{PLATFORM_TIER}\n"), + format!("{} {}\n", TEMPLATE.trim_end(), PLATFORM_TOKENS.join(" ")), + // The kernel's whitespace: tabs, newlines, carriage returns, and no-break spaces. + TEMPLATE.replace(' ', " \t\r\n\u{0b}\u{0c}"), + ] { + assert_eq!( + boot(command_line.as_bytes()), + Ok(Boot::Template), + "{command_line:?}" + ); + } + let mut latin1 = TEMPLATE.as_bytes().to_vec(); + for byte in &mut latin1 { + if *byte == b' ' { + *byte = 0xa0; + } + } + assert_eq!(boot(&latin1), Ok(Boot::Template)); + } + + #[test] + fn template_mode_refuses_a_token_outside_the_allowlist() { + for token in [ + "quiet", + "init=/bin/sh", + "console=ttyS0", + "nvx_config=0xd0010000,65537", + "nvx_snapshot_tier=workload-start", + "nvx-sandbox=1", + "nvx_control_tty=hvc1", + "nvx_exec=/bin/sh", + "nvx_lifecycle=managed", + "nvx_workload_uid=1000", + "nvx_share=0,/data", + "virtfs_tag=microvm", + "virtfs_dir=/run/nvx/hostfs/root", + "virtnet_ingress=allow", + "driver_async_probe=virtio_mmio", + "nr_cpus", + "--=1", + "caf\u{e9}=1", + ] { + for command_line in with(token) { + let refused = CommandLineError::Token(token.as_bytes().to_vec()); + assert_eq!( + boot(command_line.as_bytes()), + Err(refused.clone()), + "{command_line:?}" + ); + assert_eq!( + refused.to_string(), + format!( + "the kernel command line names the platform snapshot tier, but holds \ + `{}`, which the tier's allowlist does not hold", + token.as_bytes().escape_ascii() + ) + ); + } + } + // The first of the parameters that the allowlist does not hold, in the command line's + // order. + assert_eq!( + boot(format!("a {} b\n", TEMPLATE.trim_end()).as_bytes()), + Err(CommandLineError::Token(b"a".to_vec())) + ); + assert_eq!( + boot(b"nvx_snapshot_tier=platform \xff=\x01\n"), + Err(CommandLineError::Token(b"\xff=\x01".to_vec())) + ); + // The kernel's whitespace includes the vertical tab and the byte 0xa0, with which the + // no-break space ends in UTF-8. A split at ASCII whitespace, as OpenVMM's checks of a + // platform snapshot's command line make, keeps either within a token, but to the kernel, + // what follows either in a platform token's value is a parameter of its own. + for separator in ["\u{0b}", "\u{a0}"] { + let command_line = format!( + "{} virtnet_dns=10.0.2.3{separator}quiet\n", + TEMPLATE.trim_end() + ); + assert_eq!( + boot(command_line.as_bytes()), + Err(CommandLineError::Token(b"quiet".to_vec())), + "{command_line:?}" + ); + } + assert_eq!( + CommandLineError::Token(b"\xff=\x01".to_vec()).to_string(), + "the kernel command line names the platform snapshot tier, but holds `\\xff=\\x01`, \ + which the tier's allowlist does not hold" + ); + } + + #[test] + fn template_mode_refuses_quotes_and_the_delimiter() { + for quoted in [ + "\"console=hvc0\"", + "nvx_config=\"0xd0010000,65536\"", + "virtnet_dns=\"10.0.2.3 10.0.2.4\"", + "note=\"left right\"", + ] { + for command_line in with(quoted) { + assert_eq!( + boot(command_line.as_bytes()), + Err(CommandLineError::Quote), + "{command_line:?}" + ); + } + } + // The tier, quoted, still names a template's VM, whose command line holds a quote. + assert_eq!( + boot( + TEMPLATE + .replace(PLATFORM_TIER, "\"nvx_snapshot_tier=platform\"") + .as_bytes() + ), + Err(CommandLineError::Quote) + ); + assert_eq!( + boot(format!("{} \"\n", TEMPLATE.trim_end()).as_bytes()), + Err(CommandLineError::Quote) + ); + // A quote that the kernel never closes holds the rest of the command line in one + // parameter, as the delimiter hands the rest to init: a tier there names no template. + assert_eq!(boot(format!("\" {TEMPLATE}").as_bytes()), Ok(Boot::Cold)); + let template = TEMPLATE.trim_end(); + for command_line in [ + format!("{template} --\n"), + format!("{template} -- quiet\n"), + format!("{template} -- nvx_sandbox=1\n"), + format!("{template}\t--\t\n"), + format!("{template}\u{0b}--\u{0b}quiet\n"), + format!("{PLATFORM_TIER} -- {template}\n"), + ] { + assert_eq!( + boot(command_line.as_bytes()), + Err(CommandLineError::Delimiter), + "{command_line:?}" + ); + } + assert_eq!( + CommandLineError::Quote.to_string(), + "the kernel command line names the platform snapshot tier, but holds a double quote, \ + which the kernel drops" + ); + assert_eq!( + CommandLineError::Delimiter.to_string(), + "the kernel command line names the platform snapshot tier, but holds `--`, after \ + which the kernel hands the rest to init" + ); + } + + #[test] + fn a_template_s_statuses_are_its_own() { + let mut statuses = vec![COMMAND_LINE_STATUS, CONTINUED_STATUS, RESTORED_STATUS]; + statuses.extend((1..=9).map(|stage| CAPTURE_STATUS_BASE + stage)); + assert_eq!(statuses, [20, 30, 31, 21, 22, 23, 24, 25, 26, 27, 28, 29]); + // None is a root stage's, the worker's after its session failed, or the usage error's. + for stage in RootStage::ALL { + assert!(!statuses.contains(&stage.exit_status()), "{stage}"); + } + assert!(!statuses.contains(&0) && !statuses.contains(&1) && !statuses.contains(&2)); + } +} diff --git a/tests/runtime-adapter/src/worker.rs b/tests/runtime-adapter/src/worker.rs index f38a1cc7f..dfeefb382 100644 --- a/tests/runtime-adapter/src/worker.rs +++ b/tests/runtime-adapter/src/worker.rs @@ -56,6 +56,11 @@ //! may close only the broker's end, the worker drops its executions, and writes what it queued, //! as once the host stopped it. //! +//! In a template's VM, whose kernel command line names the platform snapshot tier, the worker +//! assembles no root: on x86_64, the guest agent reads the kernel command line before anything +//! else with `crate::template::boot`, and the worker captures the VM instead, as the `capture` +//! module describes. +//! //! # The mapping table and the start //! //! The root configuration announces how many mappings the host's mapping table holds, and the host @@ -190,6 +195,8 @@ //! in all for those programs to end and those cgroups to empty, removing each cgroup that //! emptied once it collected the statuses of those programs. +#[cfg(target_arch = "x86_64")] +pub mod capture; mod connection; pub mod entries; mod executions; diff --git a/tests/runtime-adapter/src/worker/capture.rs b/tests/runtime-adapter/src/worker/capture.rs new file mode 100644 index 000000000..f85f106ef --- /dev/null +++ b/tests/runtime-adapter/src/worker/capture.rs @@ -0,0 +1,335 @@ +//! The worker of a template's VM, which captures the VM before it assembles any root, through +//! `nvx::guest::clone`, on x86_64, whose time ABI a capture needs. The +//! [`template`](crate::template) module tells such a VM by its kernel command line, and holds the +//! statuses with which its worker ends. +//! +//! [`capture_on_terminal`] opens the VM's control tty, in raw mode, as the worker of a VM that +//! boots cold does, and then, in order: +//! +//! 1. attaches the control session with `nvx::guest::nvxs::GuestConnection::attach`, and waits +//! until the tty has sent the attachment, so that the snapshot holds the guest attached. It +//! reads nothing from the session before the capture, and serves no application session: the +//! host opens none with a template's VM; +//! 2. prepares the guest for the capture with `nvx::guest::clone::CapturePreparation`, behind the +//! worker's [`Admission`], its fence of workload admission, and from whose [`devices`] nothing +//! may be mounted: the preparation closes the fence, checks the mount table, waits for the time +//! ABI's boot checks, runs its capture checks, and suppresses the stall detectors; +//! 3. requests the capture with `CaptureBarrier::capture`, with the platform tier's request, +//! which keeps no scratch. +//! +//! A committed capture ends the template's VM. The worker returns only once the capture ended +//! without a snapshot, or failed, or in a clone, which a restore of the snapshot started, as a +//! [`Templated`] tells, and with which the worker ends. The worker does not repair a clone yet: +//! the clone ends too, with its fence still closed. + +use std::fmt; +use std::io::{self, Read, Write}; +use std::path::Path; + +use nvx::guest::clone::{ + AdmissionFence, CaptureBarrier, CaptureError, CapturePreparation, CaptureStage, Captured, + Devices, Restored, +}; +use nvx::guest::nvxs::GuestConnection; +use nvx::guest::root::Device; +use nvx::guest::terminal::Terminal; +use nvx::protocol::root::{DISTRO_MMIO_BASE, SCRATCH_MMIO_BASE}; + +use crate::guest::{self, ServeError}; +use crate::template::{CAPTURE_STATUS_BASE, CONTINUED_STATUS, RESTORED_STATUS}; + +/// The virtio-fs tag of the VM's host filesystem device, OpenVMM's one filesystem slot: dormant in +/// a template's VM, and the device of the host export that the worker of a VM that boots cold +/// mounts. +pub const FILESYSTEM_TAG: &str = "microvm"; + +/// The worker's fence of workload admission in a template's VM, which it hands the capture's +/// preparation. +/// +/// The worker of a template's VM admits no execution before the capture, nor, in a clone, before +/// repair acknowledged the restore. Closing the fence acquires it, as `nvx::guest::clone` +/// requires: [`close`](AdmissionFence::close) fails while the fence is closed, whoever closed it, +/// and leaves it closed. +#[derive(Debug, Default)] +pub struct Admission { + /// Whether the fence is closed, so that no execution starts. + closed: bool, +} + +impl Admission { + /// Returns whether the fence is closed, so that no execution starts. + pub fn is_closed(&self) -> bool { + self.closed + } +} + +impl AdmissionFence for Admission { + fn close(&mut self) -> io::Result<()> { + if self.closed { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "the worker's admission of executions is fenced already", + )); + } + self.closed = true; + Ok(()) + } + + fn open(&mut self) { + self.closed = false; + } +} + +/// Returns the devices of a template's VM from which its capture must find nothing mounted: the +/// distro and the scratch block devices, at the MMIO windows from which the worker of a VM that +/// boots cold assembles its root, and the host filesystem device, by [`FILESYSTEM_TAG`]. +pub fn devices() -> Devices { + Devices { + distro: Device { + mmio: DISTRO_MMIO_BASE, + sectors: None, + }, + scratch: Device { + mmio: SCRATCH_MMIO_BASE, + sectors: None, + }, + filesystem_tag: FILESYSTEM_TAG.to_owned(), + } +} + +/// Returns the status with which the worker of a template's VM, and with it the VM, ends once its +/// capture failed at `stage`: [`CAPTURE_STATUS_BASE`] plus the stage's number, in the order in +/// which `CaptureStage` declares the stages, from 21 for the fence to 28 for the release, or 29 for +/// a stage that the worker does not know. +pub fn stage_status(stage: CaptureStage) -> u8 { + CAPTURE_STATUS_BASE + + match stage { + CaptureStage::Fence => 1, + CaptureStage::Mounts => 2, + CaptureStage::BootChecks => 3, + CaptureStage::CaptureChecks => 4, + CaptureStage::Suppression => 5, + CaptureStage::GenerationId => 6, + CaptureStage::Capture => 7, + CaptureStage::Release => 8, + _ => 9, + } +} + +/// How the capture of a template's VM ended, once the worker attached its session. +#[derive(Debug)] +pub enum Templated { + /// The capture ended without a snapshot, as without a destination or once OpenVMM rejected + /// it, and the template continued: the barrier opened the fence, and restored the stall + /// detectors' settings. + Continued, + /// A restore of the template's snapshot started this VM, a clone, whose wall clock the time + /// ABI repaired, with the fence still closed and the stall detectors still suppressed, as + /// `Restored` holds them for repair, which the worker does not run yet. + Restored(Restored), + /// A stage of the capture failed. The preparation released what it took, and a failed + /// request left the fence closed. + Failed(CaptureError), +} + +impl Templated { + /// Returns the status with which the worker, and with it the VM, ends: [`CONTINUED_STATUS`], + /// [`RESTORED_STATUS`], or the [status](stage_status) of the stage that failed. + pub fn status(&self) -> u8 { + match self { + Self::Continued => CONTINUED_STATUS, + Self::Restored(_) => RESTORED_STATUS, + Self::Failed(error) => stage_status(error.stage()), + } + } +} + +impl fmt::Display for Templated { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Continued => formatter.write_str( + "the template's capture ended without a snapshot, and the template continued", + ), + Self::Restored(_) => formatter.write_str( + "a restore of the template's snapshot started this clone, which the worker does \ + not repair", + ), + Self::Failed(error) => error.fmt(formatter), + } + } +} + +/// Captures the template's VM on its control tty at `tty`, as the [module documentation](self) +/// describes, and returns how the capture ended. +/// +/// It opens the tty through [`guest::on_terminal`], as the worker of a VM that boots cold does, +/// and fails as that does if the tty cannot be opened or put in raw mode. It fails with +/// [`ServeError::Session`] if the attachment cannot be written, and with [`ServeError::Terminal`] +/// if the tty does not send it, before the preparation. +pub fn capture_on_terminal(tty: &Path) -> Result { + guest::on_terminal(tty, |terminal| { + let send = |terminal: &&mut Terminal| { + terminal.drain().map_err(|error| { + let problem = format!("cannot send the attachment on {}: {error}", tty.display()); + io::Error::new(error.kind(), problem) + }) + }; + attached(terminal, send, || { + let preparation = CapturePreparation::new(Admission::default(), devices()); + match preparation.prepare().and_then(CaptureBarrier::capture) { + Ok(Captured::Continued) => Templated::Continued, + Ok(Captured::Restored(restored)) => Templated::Restored(restored), + Err(error) => Templated::Failed(error), + } + }) + }) +} + +/// Attaches the guest's end of the control session on `stream`, waits until `send` has sent the +/// attachment, then returns what `capture` returns, which runs without reading the stream. +/// +/// Fails with [`ServeError::Session`] if the attachment cannot be written, and with +/// [`ServeError::Terminal`] if `send` fails, before `capture` runs. +fn attached( + stream: S, + send: impl FnOnce(&S) -> io::Result<()>, + capture: impl FnOnce() -> T, +) -> Result { + let mut connection = GuestConnection::new(stream); + connection.attach()?; + send(connection.get_ref()).map_err(ServeError::Terminal)?; + Ok(capture()) +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::os::unix::net::UnixStream; + + use nvx::guest::nvxs::GuestConnectionError; + use nvx::protocol::nvxs::GuestSession; + + use super::*; + use crate::root::RootStage; + use crate::template::COMMAND_LINE_STATUS; + use crate::worker::SESSION_STATUS; + + #[test] + fn closing_the_fence_acquires_it_until_it_opens() { + let mut admission = Admission::default(); + assert!(!admission.is_closed()); + admission.close().unwrap(); + assert!(admission.is_closed()); + // A second close fails, and leaves the fence closed. + let error = admission.close().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::ResourceBusy); + assert_eq!( + error.to_string(), + "the worker's admission of executions is fenced already" + ); + assert!(admission.is_closed()); + admission.open(); + assert!(!admission.is_closed()); + admission.close().unwrap(); + assert!(admission.is_closed()); + } + + #[test] + fn each_stage_of_a_capture_ends_the_worker_with_a_status_of_its_own() { + let stages = [ + (CaptureStage::Fence, 21), + (CaptureStage::Mounts, 22), + (CaptureStage::BootChecks, 23), + (CaptureStage::CaptureChecks, 24), + (CaptureStage::Suppression, 25), + (CaptureStage::GenerationId, 26), + (CaptureStage::Capture, 27), + (CaptureStage::Release, 28), + ]; + for (stage, status) in stages { + assert_eq!(stage_status(stage), status, "{stage}"); + } + assert_eq!(Templated::Continued.status(), CONTINUED_STATUS); + assert_eq!( + Templated::Continued.to_string(), + "the template's capture ended without a snapshot, and the template continued" + ); + // No two outcomes of the worker share a status. + let mut statuses = stages.map(|(_, status)| status).to_vec(); + statuses.extend([ + COMMAND_LINE_STATUS, + CONTINUED_STATUS, + RESTORED_STATUS, + SESSION_STATUS, + 0, + ]); + statuses.extend(RootStage::ALL.map(RootStage::exit_status)); + let count = statuses.len(); + statuses.sort_unstable(); + statuses.dedup(); + assert_eq!(statuses.len(), count); + } + + #[test] + fn a_template_s_devices_are_those_of_a_root_and_the_host_filesystem() { + let devices = devices(); + assert_eq!(devices.distro.mmio, DISTRO_MMIO_BASE); + assert_eq!(devices.scratch.mmio, SCRATCH_MMIO_BASE); + assert_eq!( + (devices.distro.sectors, devices.scratch.sectors), + (None, None) + ); + assert_eq!(devices.filesystem_tag, "microvm"); + } + + #[test] + fn the_worker_attaches_its_session_before_the_capture_and_reads_nothing() { + let (guest, broker) = UnixStream::pair().unwrap(); + // The broker's record waits on the stream, as a reset would once a host attached. + (&broker).write_all(b"waiting").unwrap(); + let mut attachment = Vec::new(); + GuestSession::new().attach(&mut attachment).unwrap(); + let sent = Cell::new(false); + let captured = attached( + &guest, + |_| { + sent.set(true); + Ok(()) + }, + || { + // The capture runs once the attachment was sent, which the broker holds. + assert!(sent.get()); + let mut received = vec![0; attachment.len()]; + (&broker).read_exact(&mut received).unwrap(); + assert_eq!(received, attachment); + "captured" + }, + ) + .unwrap(); + assert_eq!(captured, "captured"); + // The worker read nothing of the session. + let mut waiting = [0; 7]; + (&guest).read_exact(&mut waiting).unwrap(); + assert_eq!(&waiting, b"waiting"); + } + + #[test] + fn no_capture_runs_without_the_attachment() { + let (guest, _broker) = UnixStream::pair().unwrap(); + let unsent = io::Error::new(io::ErrorKind::TimedOut, "the tty sent nothing"); + let error = attached::<_, ()>(&guest, |_| Err(unsent), || panic!("captured")).unwrap_err(); + assert!( + matches!(&error, ServeError::Terminal(error) if error.kind() == io::ErrorKind::TimedOut), + "{error}" + ); + + let (guest, broker) = UnixStream::pair().unwrap(); + drop(broker); + let error = + attached::<_, ()>(&guest, |_| panic!("sent"), || panic!("captured")).unwrap_err(); + assert!( + matches!(error, ServeError::Session(GuestConnectionError::Io(_))), + "{error}" + ); + } +}