From 7a068eaa5b281241abf965ccb2e16e764eb98d22 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Sat, 24 Oct 2026 22:36:41 -0700 Subject: [PATCH 1/7] nvx-rs: build the activation records of clones A clone's plan builds the activation record that the clone's guest applies in place of the configuration that its template captured, from the request's workload view, overlay mode and journal policy, and from the inputs that the plan checks: a restore section with a fresh transaction ID of 16 random bytes and the lease's template reference; the EROFS image with its binding's identity and the UUID that the adapter's plan expects; a tmpfs, or the scratch image's binding identity with the journal policy; no export; the network of the template's NIC, with the policy identity that its key records, the DNS server that the template's guest configured its resolver with, unpublished listeners and the guest's proxy URL; and the template's processors and the memory target. A binding's identity is the SHA-256 digest of a domain and of the binding's serde form, which reads no file content. TemplateKey::dns_server reads the DNS server from the virtnet_dns= token of the key's effective command line, which OpenVMM adds when the NIC's policy lets the guest reach the gateway's DNS server: once at most, for a template with a NIC, naming one IP address in its canonical spelling. Any other form is a malformed key, which planning a clone refuses as an I/O error. The plan refuses a GPT image, which a clone's guest does not assemble yet, and settings that no record holds, before it opens or creates anything. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a424884c-018a-45f5-a7fe-c57548a873f0 --- nvx-rs/README.md | 12 + nvx-rs/src/host/template/clone.rs | 95 +++++++- nvx-rs/src/host/template/clone/record.rs | 159 +++++++++++++ nvx-rs/src/host/template/clone/tests.rs | 282 ++++++++++++++++++++++- nvx-rs/src/host/template/key.rs | 52 ++++- nvx-rs/src/host/template/key/tests.rs | 68 ++++++ 6 files changed, 654 insertions(+), 14 deletions(-) create mode 100644 nvx-rs/src/host/template/clone/record.rs diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 66dfdd3eb..f4e895d1f 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -643,6 +643,18 @@ formats, copies, or reads, or for an upper in RAM a fresh sparse file, `clone-scratch.raw` in the sandbox's directory, which the release removes; and a fresh capability. +It builds the clone's activation record, `ClonePlan::activation`, which the +clone's guest applies in place of the configuration that the template captured, +from the request's `WorkloadView`, overlay mode, and journal policy, and from +the checked inputs: a restore section with a fresh transaction ID of 16 random +bytes and the lease's `TemplateReference`; the image, an EROFS layer, with its +binding's identity and the UUID that the adapter's plan expects of it, since a +GPT image is refused; a tmpfs, or the scratch image's binding identity with the +journal policy; no export; the template NIC's policy identity, as its key +records it, unpublished listeners, and the guest's proxy URL; and the template's +processors and the memory target. A binding's identity is the SHA-256 digest of +a domain and of the binding's serde form, never of a file's content. + The image and the scratch device must present the `BlockGeometry` of the template's, which a restore compares, and be, at the paths by which OpenVMM opens them, the regular files that the checks found, which are not links or diff --git a/nvx-rs/src/host/template/clone.rs b/nvx-rs/src/host/template/clone.rs index adc8ddda4..f705cbaf9 100644 --- a/nvx-rs/src/host/template/clone.rs +++ b/nvx-rs/src/host/template/clone.rs @@ -6,6 +6,8 @@ #[cfg(all(test, any(target_os = "linux", windows)))] mod tests; +mod record; + use std::fmt; use std::fs::{self, File}; use std::io; @@ -16,6 +18,7 @@ use super::key::TemplateKey; use super::manifest::BlockGeometry; use super::plan::MEMORY_BLOCK_SIZE; use super::store::{Acquired, Generation, Lease, Store}; +use crate::activation::{ActivationRecord, Journal, OverlayMode, WorkloadView}; use crate::contract; use crate::error::{Error, ErrorKind, Result}; use crate::host::filesystem; @@ -42,11 +45,13 @@ const SPARSE_WHAT: &str = "scratch device of the clone"; /// What the clone of a sandbox restores of the generation of a template that it leased: the /// adapter's plan of the clone's image and upper, the binding of the image, the binding of the /// scratch image of an upper on one, and the memory target, which [`ClonePlan::new`] checks -/// against the generation. +/// against the generation, and the settings of the clone's activation that the template does not +/// fix, which the clone's [activation record](ClonePlan::activation) names: the workload's view +/// of its VM, the overlay's mode, and the journal policy of a scratch image. #[derive(Debug, Clone)] pub struct CloneRequest { /// The adapter's plan, which [`SandboxPlan::adapter`] makes, of the clone's image, its - /// `distro`, over its upper: in RAM, a tmpfs, as + /// `distro`, an EROFS layer, over its upper: in RAM, a tmpfs, as /// [`SandboxUpper::RAM`](crate::host::sandbox::SandboxUpper::RAM) plans it, or on the /// caller's scratch image. Its network is the one of the template's NIC, which every restore /// of the template attaches, or none for a template without a NIC. @@ -62,6 +67,15 @@ pub struct CloneRequest { /// The memory target, in bytes: the RAM of the clone's VM once restored, to which the restore /// expands the template's RAM in whole blocks. pub memory: u64, + /// The view that the clone's workload has of its VM: its hostname, the machine ID that it + /// sees, and the identity in which every execution runs, unless the executions name their + /// own. + pub workload: WorkloadView, + /// The mode of the clone's overlay, which selects its options. + pub overlay: OverlayMode, + /// Whether the filesystem of the scratch image of an upper on one may have a journal. An + /// upper in RAM has none, so the record of a clone in RAM leaves the policy out. + pub journal: Journal, } /// The upper of a clone's plan, with the file of the clone's `scratch` block device, which every @@ -83,9 +97,9 @@ pub enum CloneUpper { } /// The plan of a clone: the generation of a template that the clone leased, with the clone's own -/// image, upper, and memory target, checked against the generation, and a fresh capability, which -/// [`new`](Self::new) makes under the lock of the sandbox that acquired the lease, before the -/// clone's launch. +/// image, upper, and memory target, checked against the generation, the clone's activation +/// record, and a fresh capability, which [`new`](Self::new) makes under the lock of the sandbox +/// that acquired the lease, before the clone's launch. /// /// A restore of a template attaches the template's two block devices, its `distro` and its /// `scratch`, with files of the clone's, which OpenVMM opens itself, by their paths, as @@ -122,6 +136,11 @@ pub enum CloneUpper { /// the claim. Nothing reads, formats, or copies the image. /// - The memory target must be whole blocks of [`MEMORY_BLOCK_SIZE`], and of the template's own /// memory blocks, from the template's RAM to its RAM capacity. +/// - The activation record, which the clone's guest applies in place of the configuration that +/// the template captured, names the restore, with a fresh transaction ID, the clone's image, +/// upper, and settings, the template's network, and the resources that the clone must have, +/// as [`activation`](Self::activation) describes. Its image must be an EROFS layer, and its +/// settings must keep the limits of a record, as [`Configuration::check`] checks them. /// - The capability is fresh: the plan draws it for the clone's launch. /// /// The plan's checks read the metadata of the clone's files, never their content. @@ -129,6 +148,7 @@ pub enum CloneUpper { /// ```no_run /// use std::time::{Duration, Instant}; /// +/// use nvx::activation::{Journal, MachineIdMode, OverlayMode, WorkloadView}; /// use nvx::host::sandbox::distro::{DISTRO_POLICY_REVISION, DistroBinding}; /// use nvx::host::sandbox::{SandboxDistro, SandboxPlan, SandboxUpper}; /// use nvx::host::store::CallerFiles; @@ -156,15 +176,25 @@ pub enum CloneUpper { /// policy_revision: DISTRO_POLICY_REVISION, /// scratch: None, /// memory: 8 * MEMORY_BLOCK_SIZE, +/// workload: WorkloadView { +/// hostname: b"clone".to_vec(), +/// machine_id: MachineIdMode::Image, +/// identity: None, +/// }, +/// overlay: OverlayMode::UntrustedLowers, +/// journal: Journal::Allowed, /// }; /// let deadline = Instant::now() + Duration::from_secs(5); /// let plan = ClonePlan::new(store, &mut acquired, &request, deadline)?; /// // ... the clone's launch, under `acquired.locked`, as the launch that the lease reserves, of -/// // `plan.snapshot()` with `plan.image().path()` and `plan.scratch_path()` ... +/// // `plan.snapshot()` with `plan.image().path()` and `plan.scratch_path()`, whose guest applies +/// // `plan.activation()` ... /// # let _ = plan; /// # Ok(()) /// # } /// ``` +/// +/// [`Configuration::check`]: crate::activation::Configuration::check pub struct ClonePlan { /// The lease of the generation. lease: Lease, @@ -180,6 +210,8 @@ pub struct ClonePlan { upper: CloneUpper, /// The memory target, in bytes. memory: u64, + /// The clone's activation record. + activation: ActivationRecord, /// The capability of the clone's launch. capability: [u8; CAPABILITY_LEN], } @@ -226,11 +258,13 @@ impl ClonePlan { /// - with [`ErrorKind::Unsupported`] for a request that no clone takes: a plan of the sandbox /// profile, with workload settings and its shares; a plan whose image is not the format and /// identity that the binding names, as for a binding whose identity is not a UUID in its - /// canonical text; an image or a scratch image whose path is relative, which OpenVMM would - /// resolve against the directory in which it runs, holds a comma or a semicolon, which - /// OpenVMM's block device options would split, or whose name ends in `.vhd`, `.vhdx`, - /// `.vmgs`, or `.iso`, which OpenVMM would not attach as a raw image; and an upper on a - /// scratch image without the image's binding, or one in RAM with a binding; + /// canonical text; an image that is a GPT disk, of which this build restores no clone, since + /// a clone's guest assembles its root only from an EROFS layer; an image or a scratch image + /// whose path is relative, which OpenVMM would resolve against the directory in which it + /// runs, holds a comma or a semicolon, which OpenVMM's block device options would split, or + /// whose name ends in `.vhd`, `.vhdx`, `.vmgs`, or `.iso`, which OpenVMM would not attach as + /// a raw image; and an upper on a scratch image without the image's binding, or one in RAM + /// with a binding; /// - with [`ErrorKind::Other`] for a lease of another store, and a lock that is not the one /// that acquired the lease, before the launch that it reserves began; /// - with [`ErrorKind::NotFound`] if the store no longer indexes the leased generation, with @@ -242,6 +276,14 @@ impl ClonePlan { /// NIC, or without one for a template with a NIC; and a memory target that is not whole /// blocks of [`MEMORY_BLOCK_SIZE`] and of the template's memory blocks, from the template's /// RAM to its RAM capacity; + /// - with [`ErrorKind::Io`] for a template whose command line names the DNS server of its + /// guest's resolver in another form than OpenVMM writes: more than once, without a NIC, or + /// other than as one IP address in its canonical spelling; + /// - with [`ErrorKind::Unsupported`] for settings that no activation record holds, as + /// [`Configuration::check`] finds them, such as a hostname that is empty, longer than + /// [`MAX_HOSTNAME_BYTES`](crate::activation::MAX_HOSTNAME_BYTES), or holds a NUL, and a + /// workload identity whose UID or GID is 0, and with [`ErrorKind::Other`] if the + /// transaction ID of the clone's restore cannot be drawn; /// - with [`ErrorKind::Unavailable`] for a link or a directory under a name that the sandbox's /// store owns, and with [`ErrorKind::Io`] if the store's files cannot be listed or examined; /// - as [`DistroBinding::check`] and [`CheckedDistro::geometry`] fail, with @@ -261,6 +303,8 @@ impl ClonePlan { /// the check found, of the current user or an administrator, and that no one else may /// change, and with [`ErrorKind::Unsupported`] for an image that is a file of the sandbox's /// store. + /// + /// [`Configuration::check`]: crate::activation::Configuration::check pub fn new( store: &Store, acquired: &mut Acquired, @@ -310,6 +354,7 @@ impl ClonePlan { let key = store.read_key(&generation)?; let snapshot = store.snapshot_dir(&generation)?; check_template(request, &key, &generation)?; + let activation = record::build(request, lease, &key)?; let owned = StoreFiles::of(locked)?; let image = check_image(request, &key, &owned, access)?; let capability = generate_capability()?; @@ -329,6 +374,7 @@ impl ClonePlan { image, upper, memory: request.memory, + activation, capability, }) } @@ -381,6 +427,29 @@ impl ClonePlan { self.memory } + /// Returns the clone's activation record, which its guest applies in place of the + /// configuration that the template captured, and whose restore transaction ID, as + /// [`ReadyActivation::of`](crate::activation::ReadyActivation::of) returns it, the guest's + /// Ready reports. + /// + /// The record holds the restore section, with the restore's transaction ID, 16 random bytes + /// that the plan drew, and the template that the [lease](Lease::template) names; the image, + /// an EROFS layer, with the identity of its binding and the UUID that the guest expects of its + /// filesystem, if the adapter's plan names one; the upper, a tmpfs, or the scratch image, with + /// the identity of its binding and the request's journal policy; the request's overlay mode + /// and workload view; no export; the network of the template's NIC, if it has one, with the + /// identity of the policy that OpenVMM enforces there, as the template's key records it, the + /// DNS server with which the template's guest configured its resolver at boot, if the + /// template's [command line](TemplateKey::command_line) names one, unpublished listeners, and + /// the URL of the policy's proxy as the guest sees it, if any; and the template's processors, + /// all online, and the memory target, as the resources that the clone must have. + /// + /// The identity of a binding is the SHA-256 digest of a domain of its kind and of its serde + /// form, a few hundred bytes of metadata: the plan reads no content to compute it. + pub fn activation(&self) -> &ActivationRecord { + &self.activation + } + /// Returns the capability of the clone's launch, which the plan drew, and which the VMM /// receives, as [`LockedStore::begin_launch`] stores it. pub fn capability(&self) -> &[u8; CAPABILITY_LEN] { @@ -399,6 +468,7 @@ impl fmt::Debug for ClonePlan { .field("image", &self.image) .field("upper", &self.upper) .field("memory", &self.memory) + .field("activation", &self.activation) .finish_non_exhaustive() } } @@ -423,6 +493,9 @@ fn check_request(request: &CloneRequest) -> Result<()> { binding.identity() ))); } + if matches!(binding.format(), DistroFormat::Gpt) { + return Err(record::gpt_refused()); + } check_disk_path(distro.path(), "image")?; match (adapter.upper().scratch_path(), &request.scratch) { (Some(path), Some(_)) => check_disk_path(path, "scratch image"), diff --git a/nvx-rs/src/host/template/clone/record.rs b/nvx-rs/src/host/template/clone/record.rs new file mode 100644 index 000000000..f644a6fc2 --- /dev/null +++ b/nvx-rs/src/host/template/clone/record.rs @@ -0,0 +1,159 @@ +//! The activation record of a clone, which its [plan](super::ClonePlan) builds before the clone's +//! launch from the request and the checked generation: the record of the settings that the +//! clone's guest applies in place of the configuration that its template captured, as +//! [`nvx::activation`](crate::activation) describes it. +//! +//! A record names the bindings of the clone's image and scratch image by their identities: the +//! SHA-256 digest of a domain of each kind of binding and of the binding's serde form, which its +//! callers record. The digest covers a few hundred bytes of the binding's metadata, never the +//! content of a file. + +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use super::CloneRequest; +use crate::activation::{ + ActivationRecord, Configuration, ExpectedResources, Identity, Image, Network, Publication, + RestoreSection, TransactionId, Upper, +}; +use crate::error::{Error, Result}; +use crate::host::sandbox::distro::{DistroBinding, DistroFormat}; +use crate::host::sandbox::scratch::ScratchBinding; +use crate::host::template::key::TemplateKey; +use crate::host::template::store::Lease; + +/// What the identity of an image's binding digests before the binding's serde form: a domain +/// that no other digest of this crate shares, and the NUL that ends it. +const DISTRO_DOMAIN: &[u8] = b"nvx distro binding\0"; + +/// What the identity of a scratch image's binding digests before the binding's serde form, as +/// [`DISTRO_DOMAIN`] does for an image's. +const SCRATCH_DOMAIN: &[u8] = b"nvx scratch binding\0"; + +/// Returns the activation record of the clone that `request` asks for, of the template whose key +/// is `key`, which `lease` leases, with a fresh transaction ID. +/// +/// The record holds: +/// +/// - the restore section: a transaction ID of 16 random bytes, and the template that `lease` +/// names, its generation and the identity of its key; +/// - the image: an EROFS layer, the identity of its binding, and the UUID of its filesystem, +/// which the adapter's plan names, as the binding does; +/// - the upper: a tmpfs, or the scratch, with the identity of its binding and the request's +/// journal policy; +/// - the request's overlay mode and workload view; +/// - no export, and so no mappings; +/// - the network of the template's NIC, if it has one: the identity of the policy that OpenVMM +/// enforces there, the digest that OpenVMM computed and the template's key records; the DNS +/// server with which the template's guest configured its resolver at boot, which the template's +/// command line names, [`TemplateKey::dns_server`], if it names one; unpublished listeners, +/// since a template's policy allows neither ingress nor host-loopback access both ways; and the +/// URL of the policy's proxy as the guest sees it, if it has one; +/// - the expected resources: the template's processors, all of which a template's VM keeps +/// online, and the request's memory target. +/// +/// Fails with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a request whose image +/// is not an EROFS layer, and for a configuration that no record holds, as +/// [`Configuration::check`] finds it, such as a hostname that is empty or longer than +/// [`MAX_HOSTNAME_BYTES`](crate::activation::MAX_HOSTNAME_BYTES), or a workload identity whose +/// UID or GID is 0; with [`ErrorKind::Io`](crate::ErrorKind::Io) for a key whose command line +/// names its DNS server in a form that OpenVMM does not write, as [`TemplateKey::dns_server`] +/// finds it; and with [`ErrorKind::Other`](crate::ErrorKind::Other) if the transaction ID cannot +/// be drawn. +pub(super) fn build( + request: &CloneRequest, + lease: &Lease, + key: &TemplateKey, +) -> Result { + let image = match request.binding.format() { + DistroFormat::Erofs => Image::Erofs { + binding: distro_identity(&request.binding)?, + uuid: request.adapter.distro().image().identity, + }, + DistroFormat::Gpt => return Err(gpt_refused()), + }; + let upper = match &request.scratch { + None => Upper::Tmpfs, + Some(binding) => Upper::Scratch { + binding: scratch_identity(binding)?, + journal: request.journal, + }, + }; + let dns = key.dns_server()?; + let network = key.network_policy_digest().map(|policy| Network { + policy: Identity::from_bytes(policy), + dns, + publication: Publication::Unpublished, + proxy: request + .adapter + .network() + .and_then(|network| network.proxy_url()), + }); + let configuration = Configuration { + image, + upper, + overlay: request.overlay, + export: None, + mappings: 0, + network, + workload: request.workload.clone(), + resources: ExpectedResources { + cpus: key.processors(), + memory: request.memory, + }, + }; + let restore = RestoreSection { + transaction: transaction()?, + template: lease.template(), + }; + ActivationRecord::new(Some(restore), configuration).map_err(|error| { + Error::unsupported(format!( + "no clone activates with the settings of the request: {error}" + )) + }) +} + +/// Returns the error of a request whose image is a GPT disk, which no clone restores yet. +pub(super) fn gpt_refused() -> Error { + Error::unsupported( + "the image of a clone must be an EROFS layer: this build restores no clone of a GPT \ + image", + ) +} + +/// Returns the identity of `binding`, the binding of a clone's image, as the [module](self) +/// describes it. +pub(super) fn distro_identity(binding: &DistroBinding) -> Result { + identity(DISTRO_DOMAIN, binding, "image") +} + +/// Returns the identity of `binding`, the binding of a clone's scratch image, as the +/// [module](self) describes it. +pub(super) fn scratch_identity(binding: &ScratchBinding) -> Result { + identity(SCRATCH_DOMAIN, binding, "scratch image") +} + +/// Returns the SHA-256 digest of `domain` and of the serde form of `binding`, the binding of a +/// clone's file, which messages call `what`. +fn identity(domain: &[u8], binding: &impl Serialize, what: &str) -> Result { + let form = serde_json::to_vec(binding).map_err(|error| { + Error::other(format!( + "cannot write the binding of the clone's {what} in its serde form" + )) + .with_source(error) + })?; + let digest = Sha256::new() + .chain_update(domain) + .chain_update(form) + .finalize(); + Ok(Identity::from_bytes(digest.into())) +} + +/// Returns a fresh transaction ID: 16 random bytes. +fn transaction() -> Result { + let mut bytes = [0; TransactionId::LEN]; + getrandom::fill(&mut bytes).map_err(|error| { + Error::other("cannot draw the transaction ID of a clone's restore").with_source(error) + })?; + Ok(TransactionId::from_bytes(bytes)) +} diff --git a/nvx-rs/src/host/template/clone/tests.rs b/nvx-rs/src/host/template/clone/tests.rs index 7808cb36a..a34880567 100644 --- a/nvx-rs/src/host/template/clone/tests.rs +++ b/nvx-rs/src/host/template/clone/tests.rs @@ -3,12 +3,19 @@ //! the sparse scratch device of an upper in RAM. use std::io::Read; +use std::net::{IpAddr, Ipv4Addr}; use std::time::Duration; +use sha2::{Digest, Sha256}; + use super::*; use crate::ErrorCode; +use crate::activation::{ + Configuration, ExpectedResources, Identity, Image, MachineIdMode, Network, Publication, Upper, + WorkloadIdentity, +}; use crate::host::filesystem; -use crate::host::network::{Access as NetworkAccess, NetworkPolicy}; +use crate::host::network::{Access as NetworkAccess, NetworkPolicy, NetworkProxy, ProxyScheme}; use crate::host::paths; use crate::host::sandbox::distro::{DISTRO_POLICY_REVISION, PolicyVerdict}; use crate::host::sandbox::scratch::CLAIM_RELEASE_WAIT; @@ -152,7 +159,9 @@ fn layer(path: &Path) -> SandboxLayer { } /// Returns the request of a clone of the image at `image`, which `binding` binds, over an upper in -/// RAM, with the template's RAM as its memory target. +/// RAM, with the template's RAM as its memory target, and the settings of [`workload`], for +/// untrusted lowers, and with a journal policy, [`Journal::Forbidden`], that its record leaves +/// out. fn tmpfs_request(image: &Path, binding: &DistroBinding) -> CloneRequest { CloneRequest { adapter: SandboxPlan::adapter( @@ -163,6 +172,21 @@ fn tmpfs_request(image: &Path, binding: &DistroBinding) -> CloneRequest { policy_revision: DISTRO_POLICY_REVISION, scratch: None, memory: MEMORY, + workload: workload(), + overlay: OverlayMode::UntrustedLowers, + journal: Journal::Forbidden, + } +} + +/// Returns the view that these tests' clones give their workloads of their VMs. +fn workload() -> WorkloadView { + WorkloadView { + hostname: b"clone".to_vec(), + machine_id: MachineIdMode::Generated, + identity: Some(WorkloadIdentity { + uid: 1000, + gid: 1000, + }), } } @@ -1467,3 +1491,257 @@ fn files_that_their_paths_do_not_name_as_checked_are_refused() { ); assert_untouched(&acquired); } + +/// Returns the SHA-256 digest of `domain` and of the serde form of `binding`, as the identity of a +/// binding is documented. +fn digest_of(domain: &[u8], binding: &impl serde::Serialize) -> Identity { + let digest = Sha256::new() + .chain_update(domain) + .chain_update(serde_json::to_vec(binding).unwrap()) + .finalize(); + Identity::from_bytes(digest.into()) +} + +#[test] +fn the_activation_record_of_a_clone_in_ram_names_its_restore_image_and_settings() { + let fixture = Fixture::new(); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let request = tmpfs_request(&image, &binding); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let record = plan.activation(); + let restore = record.restore().unwrap(); + assert_eq!(restore.template, acquired.lease.template()); + assert_eq!( + record.configuration(), + &Configuration { + image: Image::Erofs { + binding: digest_of(b"nvx distro binding\0", &binding), + uuid: Some(Uuid::parse(UUID).unwrap()), + }, + // A clone in RAM has no journal policy, whatever the request's. + upper: Upper::Tmpfs, + overlay: OverlayMode::UntrustedLowers, + export: None, + mappings: 0, + network: None, + workload: workload(), + resources: ExpectedResources { + cpus: plan.key().processors(), + memory: MEMORY, + }, + } + ); + // The plan shows the record, which holds nothing secret. + let shown_plan = format!("{plan:?}"); + assert!( + shown_plan.contains(&format!("activation: {record:?}")), + "{shown_plan}" + ); + // Each plan draws its own transaction. + let transaction = restore.transaction; + drop(plan); + let again = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let restored = again.activation().restore().unwrap(); + assert_ne!(restored.transaction, transaction); + assert_eq!(restored.template, acquired.lease.template()); + // A larger memory target is the resources that the clone must have. + drop(again); + let larger = CloneRequest { + memory: 2 * MEMORY, + ..request + }; + let plan = fixture.plan(&mut acquired, &larger, NO_CONTENT).unwrap(); + assert_eq!( + plan.activation().configuration().resources.memory, + 2 * MEMORY + ); +} + +#[test] +fn the_activation_record_of_a_clone_on_a_scratch_image_names_its_binding_and_journal() { + let fixture = Fixture::new(); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); + let mut acquired = fixture.acquire("sandbox"); + for journal in [Journal::Forbidden, Journal::Allowed] { + let request = CloneRequest { + overlay: OverlayMode::TrustedLowers, + journal, + ..scratch_request(&image, &binding, &scratch, &scratch_binding) + }; + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let configuration = plan.activation().configuration(); + assert_eq!( + configuration.upper, + Upper::Scratch { + binding: digest_of(b"nvx scratch binding\0", &scratch_binding), + journal, + } + ); + assert_eq!(configuration.overlay, OverlayMode::TrustedLowers); + assert_eq!( + *configuration.image.binding(), + digest_of(b"nvx distro binding\0", &binding) + ); + } + // The identities of two bindings differ, as their files do. + let (other, other_binding) = fixture.scratch("other.ext4", SCRATCH_LEN); + let request = scratch_request(&image, &binding, &other, &other_binding); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + assert_ne!( + plan.activation().configuration().upper, + Upper::Scratch { + binding: digest_of(b"nvx scratch binding\0", &scratch_binding), + journal: Journal::Forbidden, + } + ); +} + +#[test] +fn the_activation_record_of_a_clone_of_a_template_with_a_nic_names_its_policy_and_dns_server() { + let fixture = Fixture::with(|class, spec, physical| { + *class = network_class(); + *spec = ManifestSpec::with_network(); + let (distro, scratch) = ( + geometry(IMAGE_LEN, physical), + geometry(SCRATCH_LEN, physical), + ); + set_geometries(class, spec, distro, scratch); + }); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + // The policy's identity is the digest that OpenVMM computed, as the template's key records + // it, [7; 32] in this manifest, and the network names no proxy without one. + let attached = with_network( + tmpfs_request(&image, &binding), + Some(network(NetworkAccess::Allow)), + ); + let plan = fixture.plan(&mut acquired, &attached, NO_CONTENT).unwrap(); + let expected = Network { + policy: Identity::from_bytes([7; 32]), + dns: None, + publication: Publication::Unpublished, + proxy: None, + }; + assert_eq!( + plan.activation().configuration().network.as_ref(), + Some(&expected) + ); + drop(plan); + // A proxy's URL is the one that the guest sees, at its gateway. + let proxy = NetworkProxy::new(3128).with_scheme(ProxyScheme::Http); + let proxied = with_network( + tmpfs_request(&image, &binding), + Some(SandboxNetwork::new("10.0.0.2/24", NetworkPolicy::proxy_only(proxy)).unwrap()), + ); + let plan = fixture.plan(&mut acquired, &proxied, NO_CONTENT).unwrap(); + assert_eq!( + plan.activation().configuration().network, + Some(Network { + proxy: Some("http://10.0.0.1:3128".to_owned()), + ..expected.clone() + }) + ); + drop((plan, acquired, fixture)); + // The DNS server with which the template's guest configured its resolver at boot, as its + // command line asks, is the clone's. + let fixture = Fixture::with(|class, spec, physical| { + *class = network_class(); + *spec = ManifestSpec::with_network(); + spec.contract() + .command_line + .push_str(" virtnet_dns=10.0.0.1"); + let (distro, scratch) = ( + geometry(IMAGE_LEN, physical), + geometry(SCRATCH_LEN, physical), + ); + set_geometries(class, spec, distro, scratch); + }); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let attached = with_network( + tmpfs_request(&image, &binding), + Some(network(NetworkAccess::Allow)), + ); + let plan = fixture.plan(&mut acquired, &attached, NO_CONTENT).unwrap(); + assert_eq!( + plan.activation().configuration().network, + Some(Network { + dns: Some(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + ..expected + }) + ); +} + +#[test] +fn settings_that_no_activation_record_holds_are_refused_before_anything_is_created() { + let fixture = Fixture::new(); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let request = tmpfs_request(&image, &binding); + let with_workload = |workload: WorkloadView| CloneRequest { + workload, + ..request.clone() + }; + for (workload, problem) in [ + ( + WorkloadView { + hostname: Vec::new(), + ..workload() + }, + "the hostname is not 1 to 64 bytes without a NUL", + ), + ( + WorkloadView { + hostname: vec![b'h'; 65], + ..workload() + }, + "the hostname is not 1 to 64 bytes without a NUL", + ), + ( + WorkloadView { + hostname: b"clone\0".to_vec(), + ..workload() + }, + "the hostname is not 1 to 64 bytes without a NUL", + ), + ( + WorkloadView { + identity: Some(WorkloadIdentity { uid: 0, gid: 1000 }), + ..workload() + }, + "the workload identity's UID or GID is 0 or 4294967295", + ), + ] { + assert_eq!( + failure(fixture.plan(&mut acquired, &with_workload(workload), NO_CONTENT)), + ( + ErrorKind::Unsupported, + format!("no clone activates with the settings of the request: {problem}") + ) + ); + assert_untouched(&acquired); + } + // A GPT image, whose binding names it, is refused before anything is opened. + let path = fixture.path("images", "distro.gpt"); + stage(&path, IMAGE_LEN); + let gpt = SandboxLayer::gpt(LayerRole::Distro, path.to_str().unwrap(), UUID); + let gpt_binding = DistroBinding::import(&gpt, PolicyVerdict::ACCEPTED).unwrap(); + let gpt_request = CloneRequest { + adapter: SandboxPlan::adapter( + SandboxDistro::new(path.to_str().unwrap(), bound_image(&gpt_binding).unwrap()), + SandboxUpper::RAM, + ), + binding: gpt_binding, + ..request.clone() + }; + let (kind, message) = failure(fixture.plan(&mut acquired, &gpt_request, NO_CONTENT)); + assert_eq!(kind, ErrorKind::Unsupported, "{message}"); + assert!( + message.starts_with("the image of a clone must be an EROFS layer"), + "{message}" + ); + assert_untouched(&acquired); + drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); +} diff --git a/nvx-rs/src/host/template/key.rs b/nvx-rs/src/host/template/key.rs index a28c0d68a..0c64ed6e1 100644 --- a/nvx-rs/src/host/template/key.rs +++ b/nvx-rs/src/host/template/key.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use std::fmt; -use std::net::{Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use sha2::{Digest, Sha256}; @@ -34,6 +34,10 @@ const CLASS_DOMAIN: &[u8] = b"nvx capture class\0"; /// The generation of a CPU profile that a host derives, rather than a built-in one. const HOST_GENERATION: &str = "host"; +/// The prefix of the command line's token that names the DNS server of the guest's resolver, +/// which OpenVMM writes once at most, with an IP address in its canonical spelling. +const DNS_TOKEN: &str = "virtnet_dns="; + /// The versions of the contracts of a template's clones that its runtime identity does not /// record: the snapshot format, the saved device state, the time ABI, the restore packet, and the /// activation record. @@ -759,6 +763,52 @@ impl TemplateKey { self.machine.nic.is_some() } + /// Returns the SHA-256 digest of the canonical encoding of the network policy that OpenVMM + /// enforces on the template's NIC, which OpenVMM computed and the manifest records, or `None` + /// for a template without a NIC. + pub(crate) fn network_policy_digest(&self) -> Option<[u8; 32]> { + self.machine.nic.as_ref().map(|nic| nic.policy.digest) + } + + /// Returns the address of the DNS server with which the template's guest configured its + /// resolver at boot: the one that the `virtnet_dns=` token of the command line names, which + /// OpenVMM adds when the policy of the template's NIC lets the guest reach the gateway's DNS + /// server, or `None` for a command line without the token. + /// + /// Fails with [`ErrorKind::Io`](crate::ErrorKind::Io) for a command line that OpenVMM does + /// not write: one with more than one such token, with a token for a template without a NIC, + /// or with a token that does not name one IP address in its canonical spelling. + pub(crate) fn dns_server(&self) -> Result> { + let mut tokens = self + .machine + .command_line + .split_ascii_whitespace() + .filter_map(|token| token.strip_prefix(DNS_TOKEN)); + let Some(token) = tokens.next() else { + return Ok(None); + }; + let malformed = |problem: &str| { + Err(Error::io(format!( + "the command line of the template's key is malformed: {problem}" + ))) + }; + if tokens.next().is_some() { + return malformed(&format!("it holds more than one {DNS_TOKEN} token")); + } + if self.machine.nic.is_none() { + return malformed(&format!( + "it names a DNS server, {token:?}, for a template without a NIC" + )); + } + match token.parse::() { + Ok(server) if server.to_string() == token => Ok(Some(server)), + _ => malformed(&format!( + "its {DNS_TOKEN} token names {token:?}, which is not an IP address in its \ + canonical spelling" + )), + } + } + /// Returns the tag of the filesystem slot, on which a clone's export is bound. pub fn filesystem_tag(&self) -> &str { &self.filesystem_tag diff --git a/nvx-rs/src/host/template/key/tests.rs b/nvx-rs/src/host/template/key/tests.rs index 5203c4292..7a665b828 100644 --- a/nvx-rs/src/host/template/key/tests.rs +++ b/nvx-rs/src/host/template/key/tests.rs @@ -365,6 +365,74 @@ fn a_key_holds_what_its_capture_made() { assert_eq!(made.invariants(), &Invariants::new()); } +#[test] +fn a_key_names_the_dns_server_that_its_command_line_names() { + // Neither manifest of the tests names a DNS server. + assert_eq!(network_key().dns_server().unwrap(), None); + let template = key(&support::class(), &ManifestSpec::template()); + assert_eq!(template.dns_server().unwrap(), None); + let with_tokens = |class: CaptureClass, mut spec: ManifestSpec, tokens: &str| { + spec.contract().command_line.push_str(tokens); + key(&class, &spec) + }; + let with_dns = |tokens| with_tokens(network_class(), ManifestSpec::with_network(), tokens); + for (tokens, server) in [ + ( + " virtnet_dns=10.0.0.1", + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), + ), + ( + " virtnet_dns=fd00::a00:1", + IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0xa00, 1)), + ), + ] { + assert_eq!(with_dns(tokens).dns_server().unwrap(), Some(server)); + } + // OpenVMM writes one token at most, with one address in its canonical spelling, and only + // for a NIC. + let noncanonical = |value: &str| { + format!( + "its virtnet_dns= token names {value:?}, which is not an IP address in its canonical \ + spelling" + ) + }; + let without_nic = with_tokens( + support::class(), + ManifestSpec::template(), + " virtnet_dns=10.0.0.1", + ); + for (made, problem) in [ + ( + with_dns(" virtnet_dns=10.0.0.1 virtnet_dns=10.0.0.1"), + "it holds more than one virtnet_dns= token".to_owned(), + ), + (with_dns(" virtnet_dns="), noncanonical("")), + ( + with_dns(" virtnet_dns=010.0.0.1"), + noncanonical("010.0.0.1"), + ), + ( + with_dns(" virtnet_dns=fd00:0::a00:1"), + noncanonical("fd00:0::a00:1"), + ), + ( + with_dns(" virtnet_dns=10.0.0.1,fd00::a00:1"), + noncanonical("10.0.0.1,fd00::a00:1"), + ), + ( + without_nic, + "it names a DNS server, \"10.0.0.1\", for a template without a NIC".to_owned(), + ), + ] { + let error = made.dns_server().unwrap_err(); + assert_eq!(error.code(), ErrorCode::Io, "{error}"); + assert_eq!( + error.message(), + format!("the command line of the template's key is malformed: {problem}") + ); + } +} + /// Asserts that the capture of `class` whose manifest is `spec` does not match its class, with /// a message that holds `expected`. fn assert_mismatched(class: &CaptureClass, spec: &ManifestSpec, expected: &str) { From 1fe03218b0af05c23392831971ac15707452323b Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Sun, 25 Oct 2026 09:50:21 -0700 Subject: [PATCH 2/7] nvx-rs: keep a clone's activation record in its sandbox The sandbox store owns two more names for a clone's launch: its activation record, clone-activation.bin, and the socket at which the launch awaits the VMM's restore-ready event on Linux, restore.sock, as long a name as control.sock. A release removes them with the scratch device's file of a clone in RAM, before the clone binding, whose durable removal makes theirs durable too. A removal of the sandbox now does the same: it removes the clone's files with the binding, before the sync that makes the binding's absence durable, rather than with the store's other files after it. Both take a clone's file whose deletion is pending, as a Windows file system without POSIX deletion leaves one that a reader holds, for removed, so a release that failed part of the way can be retried. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a424884c-018a-45f5-a7fe-c57548a873f0 --- nvx-rs/src/host/store/lifecycle.rs | 107 +++++++++------- nvx-rs/src/host/store/lifecycle/tests.rs | 116 +++++++++++++++--- nvx-rs/src/host/store/mod.rs | 77 ++++++++++-- nvx-rs/src/host/store/tests.rs | 27 ++++ nvx-rs/src/host/template/store/leases.rs | 22 ++-- .../src/host/template/store/leases/tests.rs | 22 +++- 6 files changed, 284 insertions(+), 87 deletions(-) diff --git a/nvx-rs/src/host/store/lifecycle.rs b/nvx-rs/src/host/store/lifecycle.rs index 0310337b3..f28adf52f 100644 --- a/nvx-rs/src/host/store/lifecycle.rs +++ b/nvx-rs/src/host/store/lifecycle.rs @@ -11,8 +11,8 @@ use std::time::Instant; use super::caller::{is_report_staging, is_store_temporary}; use super::{ - CLONE_BINDING_NAME, CallerFiles, FILE_NAMES, JOURNAL_NAME, LOCK_NAME, LockedStore, - RUNTIME_NAME, Races, SandboxStore, Taken, absolute, pause, transition_in_progress, + CLONE_BINDING_NAME, CLONE_FILE_NAMES, CallerFiles, FILE_NAMES, JOURNAL_NAME, LOCK_NAME, + LockedStore, RUNTIME_NAME, Races, SandboxStore, Taken, absolute, pause, transition_in_progress, }; use crate::error::{Error, ErrorCode, ErrorKind, Result}; use crate::host::paths::not_unicode; @@ -269,32 +269,32 @@ impl LockedStore { /// [`supervisor::reattach`](crate::host::supervisor::reattach) does. It also refuses, with /// [`RemovalError::Io`], and removes nothing, once the directory that its path names holds /// another lock file than this lock's, as after a link in the path changed. It makes both - /// checks, and removes the sandbox's [clone binding](CLONE_BINDING_NAME), if it holds one, - /// through one handle of the directory that it opens, so that no change of a link in the path - /// meanwhile redirects the binding's removal to another sandbox, whose VM may run. It makes - /// the binding's absence durable before anything else goes, so that no crash after the - /// removal of the directory, or of one of its ancestors, brings back a binding whose lease a - /// retirement may have ended. It then removes the store's files but its lock, the caller's - /// files, its configuration record last, so that an interrupted removal leaves a provisioned - /// sandbox, which can be removed again, and the temporary files that the store owns. It - /// refuses the entries that are neither the store's nor the caller's, which it names in the - /// order in which the directory lists them, and keeps them and the directory; the lock file - /// goes as the lock is released, since the configuration record is gone. Otherwise it removes - /// its lock file while it still holds the lock, so that a transition that waited for the lock - /// finds the sandbox gone rather than halfway removed, and then the directory, as long as the - /// path still names it. + /// checks, and removes what a clone kept of its launch, as a release does, and the sandbox's + /// [clone binding](CLONE_BINDING_NAME), if it holds one, through one handle of the directory + /// that it opens, so that no change of a link in the path meanwhile redirects their removal + /// to another sandbox, whose VM may run. It makes their absence durable before anything else + /// goes, so that no crash after the removal of the directory, or of one of its ancestors, + /// brings back a binding whose lease a retirement may have ended. It then removes the store's + /// other files but its lock, the caller's files, its configuration record last, so that an + /// interrupted removal leaves a provisioned sandbox, which can be removed again, and the + /// temporary files that the store owns. It refuses the entries that are neither the store's + /// nor the caller's, which it names in the order in which the directory lists them, and keeps + /// them and the directory; the lock file goes as the lock is released, since the + /// configuration record is gone. Otherwise it removes its lock file while it still holds the + /// lock, so that a transition that waited for the lock finds the sandbox gone rather than + /// halfway removed, and then the directory, as long as the path still names it. /// - /// Lock files of other transitions, a clone binding whose deletion is pending, and on NFS other - /// removed files that handles still hold, can keep the emptied directory, and are not refused: - /// NFS renames a removed file that is still open to `.nfs*`; a Windows file system without - /// POSIX deletion keeps a removed lock file, or a removed binding that a reader holds, under - /// its own name until its last handle closes; and a transition that opens the lock file - /// between the removals of the lock file and of the directory creates a new one. Each of those - /// transitions finds no sandbox, and removes its lock file as it releases its lock, and the - /// handles of other removed files close, so the directory empties. The removal then holds the - /// directory's identity, releases the lock, and returns the [`Lingering`] removal, whose - /// [`wait`](Lingering::wait) removes the directory once it empties. It returns `None` once the - /// directory is gone, whoever removed it. + /// Lock files of other transitions, a clone binding or a file that the clone kept of its launch + /// whose deletion is pending, and on NFS other removed files that handles still hold, can keep + /// the emptied directory, and are not refused: NFS renames a removed file that is still open to + /// `.nfs*`; a Windows file system without POSIX deletion keeps a removed lock file, or a + /// removed binding or clone's file that a reader holds, under its own name until its last + /// handle closes; and a transition that opens the lock file between the removals of the lock + /// file and of the directory creates a new one. Each of those transitions finds no sandbox, and + /// removes its lock file as it releases its lock, and the handles of other removed files close, + /// so the directory empties. The removal then holds the directory's identity, releases the + /// lock, and returns the [`Lingering`] removal, whose [`wait`](Lingering::wait) removes the + /// directory once it empties. It returns `None` once the directory is gone, whoever removed it. pub fn remove(self) -> std::result::Result, RemovalError> { self.remove_contents()?; self.remove_directory() @@ -302,8 +302,9 @@ impl LockedStore { /// Empties the directory but for lingering files, its own lock file included, as /// [`remove`](Self::remove) does first: it refuses without the caller's configuration record, - /// removes the clone binding as [`unbind`](Self::unbind) does, then the store's files and the - /// caller's, and the temporary files that the store owns, and refuses any other entry. + /// removes what a clone kept of its launch and the clone binding as [`unbind`](Self::unbind) + /// does, then the store's other files and the caller's, and the temporary files that the store + /// owns, and refuses any other entry. fn remove_contents(&self) -> std::result::Result<(), RemovalError> { self.remove_contents_with(&|| {}, &platform::Directory::sync) } @@ -319,10 +320,14 @@ impl LockedStore { return Err(RemovalError::Undeclared); } self.unbind(checked, sync)?; - // The binding is removed once: one whose deletion is pending refuses a second removal. + // What a clone kept of its launch, and the binding, go once, through the directory's + // handle: one whose deletion is pending refuses a second removal, and its path may name + // another sandbox meanwhile. let owned: Vec = FILE_NAMES .into_iter() - .filter(|&name| name != LOCK_NAME && name != CLONE_BINDING_NAME) + .filter(|&name| { + name != LOCK_NAME && name != CLONE_BINDING_NAME && !CLONE_FILE_NAMES.contains(&name) + }) .chain(self.files.names()) .map(|name| self.path(name)) .collect(); @@ -335,9 +340,9 @@ impl LockedStore { })?; let mut foreign = Vec::new(); for name in listed { - // Lock files, a binding whose deletion is pending, and removed files that NFS keeps - // while handles hold them, go once their holders release them, which the removal - // waits for, if they keep the directory. + // Lock files, a binding or a clone's file whose deletion is pending, and removed files + // that NFS keeps while handles hold them, go once their holders release them, which + // the removal waits for, if they keep the directory. if lingers(self.dir(), &name) { continue; } @@ -354,17 +359,18 @@ impl LockedStore { } } - /// Removes the sandbox's clone binding, if it holds one, and makes its absence durable with - /// `sync`, before anything else of the sandbox goes, through one handle of the directory that - /// the path names. Through that handle, it checks first that the directory holds this lock's - /// file, and refuses otherwise, then that the store records no VM and no launch, and runs - /// `checked` once both checks pass, before it removes the binding. No change of a link in the - /// path meanwhile redirects these steps to another sandbox, whose VM may run, and whose - /// binding then stays. + /// Removes what the clone of the sandbox kept of its launch, as a release does, and then the + /// sandbox's clone binding, if it holds one, and makes their absence durable with `sync`, + /// before anything else of the sandbox goes, through one handle of the directory that the path + /// names. Through that handle, it checks first that the directory holds this lock's file, and + /// refuses otherwise, then that the store records no VM and no launch, and runs `checked` once + /// both checks pass, before it removes anything. No change of a link in the path meanwhile + /// redirects these steps to another sandbox, whose VM may run, and whose files then stay. /// /// The binding's absence, after this removal or after a release that a crash interrupted /// before its sync, is then durable while the directory, which keeps the sandbox provisioned, - /// remains. A binding whose deletion is pending already counts as removed. + /// remains. A binding, or a file of the clone, whose deletion is pending already counts as + /// removed. fn unbind( &self, checked: &dyn Fn(), @@ -400,6 +406,11 @@ impl LockedStore { } } checked(); + // What a clone kept of its launch goes before its binding, as a release removes it, so + // that the binding's durable absence makes theirs durable too. + for name in CLONE_FILE_NAMES { + directory.remove_file(name).map_err(entry_failed(name))?; + } directory .remove_file(CLONE_BINDING_NAME) .map_err(entry_failed(CLONE_BINDING_NAME))?; @@ -489,8 +500,9 @@ impl Lingering { /// appears in it, as a provision that took the lock after the removal released it writes it, /// which it looks up by its path, so that a file system that ignores case finds it whatever /// the case of the name that the provision gave it, with [`RemovalError::Foreign`] for other - /// entries than lock files, a clone binding whose deletion is pending, and the removed files - /// that NFS keeps, sorted, and with [`RemovalError::TimedOut`] once the deadline passes. + /// entries than lock files, a clone binding or a clone's file whose deletion is pending, and + /// the removed files that NFS keeps, sorted, and with [`RemovalError::TimedOut`] once the + /// deadline passes. pub fn wait(&mut self, deadline: Option) -> std::result::Result<(), RemovalError> { loop { let error = match remove_held_directory(&self.dir, self.identity) { @@ -651,12 +663,13 @@ fn remove_entry(path: PathBuf) -> std::result::Result<(), RemovalError> { /// a handle still keeps, which goes once its holder releases or closes it: NFS renames a removed /// file that is still open to `.nfs*`, and Windows file systems without POSIX deletion keep a /// removed lock file under its own name, as they keep the lock files that waiting transitions -/// create until they release them, and a removed clone binding until its readers close it, whose -/// deletion is then pending. +/// create until they release them, and a removed clone binding, or a removed file that the clone +/// kept of its launch, until its readers close it, whose deletion is then pending. fn lingers(dir: &Path, name: &str) -> bool { name == LOCK_NAME || name.starts_with(".nfs") - || (name == CLONE_BINDING_NAME && platform::is_deleted(&dir.join(name))) + || ((name == CLONE_BINDING_NAME || CLONE_FILE_NAMES.contains(&name)) + && platform::is_deleted(&dir.join(name))) } /// Returns whether `error` is how the removal of a directory reports one that is not empty. diff --git a/nvx-rs/src/host/store/lifecycle/tests.rs b/nvx-rs/src/host/store/lifecycle/tests.rs index cdafeaaf7..8ca75742b 100644 --- a/nvx-rs/src/host/store/lifecycle/tests.rs +++ b/nvx-rs/src/host/store/lifecycle/tests.rs @@ -11,7 +11,8 @@ use std::time::Duration; use super::*; use crate::host::store::{ - CAPABILITY_NAME, CLONE_SCRATCH_NAME, CONSOLE_NAME, LOG_NAME, OUTCOME_NAME, SOCKET_NAME, + CAPABILITY_NAME, CLONE_ACTIVATION_NAME, CLONE_SCRATCH_NAME, CONSOLE_NAME, LOG_NAME, + OUTCOME_NAME, RESTORE_READY_NAME, SOCKET_NAME, }; use crate::host::test_support::released; @@ -324,6 +325,8 @@ fn a_removal_removes_the_store_the_callers_files_and_the_directory() { SOCKET_NAME, CLONE_BINDING_NAME, CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, BOOT, ".runtime.json.12345.tmp", ".sandbox.lock.7.tmp", @@ -408,10 +411,23 @@ fn a_removal_is_refused_while_a_vm_or_a_launch_is_recorded() { } } +/// Writes the files that a clone keeps of its launch in the sandbox `dir`, its binding included. +fn write_clone_files(dir: &Path) { + for name in [ + CLONE_BINDING_NAME, + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, + ] { + fs::write(dir.join(name), b"{}").unwrap(); + } +} + #[test] -fn a_removal_makes_the_absence_of_the_clone_binding_durable_before_it_removes_anything_else() { +fn a_removal_makes_the_absence_of_the_clones_files_durable_before_it_removes_anything_else() { let (_directory, dir) = sandbox(); - for name in [CLONE_BINDING_NAME, LOG_NAME, BOOT] { + write_clone_files(&dir); + for name in [LOG_NAME, BOOT] { fs::write(dir.join(name), b"{}").unwrap(); } let locked = SandboxStore::lock_existing(&dir, &files(), None).unwrap(); @@ -425,8 +441,8 @@ fn a_removal_makes_the_absence_of_the_clone_binding_durable_before_it_removes_an }) .unwrap(); // A crash after the sync, even once the directory or one of its ancestors is gone, cannot - // bring the binding back, and one before the configuration record goes leaves a provisioned - // sandbox, whose removal can be retried. + // bring the binding or the clone's files back, and one before the configuration record goes + // leaves a provisioned sandbox, whose removal can be retried. assert_eq!( synced.into_inner(), [( @@ -440,11 +456,10 @@ fn a_removal_makes_the_absence_of_the_clone_binding_durable_before_it_removes_an } #[test] -fn a_removal_that_cannot_make_the_absence_of_the_clone_binding_durable_keeps_the_sandbox() { +fn a_removal_that_cannot_make_the_absence_of_the_clones_files_durable_keeps_the_sandbox() { let (_directory, dir) = sandbox(); - for name in [CLONE_BINDING_NAME, LOG_NAME] { - fs::write(dir.join(name), b"{}").unwrap(); - } + write_clone_files(&dir); + fs::write(dir.join(LOG_NAME), b"{}").unwrap(); let locked = SandboxStore::lock_existing(&dir, &files(), None).unwrap(); let error = locked .remove_contents_with(&|| {}, &|_| Err(io::Error::other("the sync failed"))) @@ -455,7 +470,8 @@ fn a_removal_that_cannot_make_the_absence_of_the_clone_binding_durable_keeps_the ); assert_eq!(Error::from(error).kind(), ErrorKind::Io); drop(locked); - // Only the binding went, so the sandbox is still provisioned, and its removal can be retried. + // Only the clone's files went, so the sandbox is still provisioned, and its removal can be + // retried. assert_eq!(entries_of(&dir), sorted(&[CONFIG, LOCK_NAME, LOG_NAME])); assert!(remove(&dir).unwrap().is_none()); assert!(!dir.exists()); @@ -501,24 +517,40 @@ fn relink(target: &Path, link: &Path) { } /// Provisions the sandboxes `one/sandbox` and `two/sandbox` in `root`, each holding a clone -/// binding, links `root/link` to `one`, and returns `one`, `two`, and the link. +/// binding and the files that a clone keeps of its launch, links `root/link` to `one`, and returns +/// `one`, `two`, and the link. fn linked_sandboxes(root: &Path) -> (PathBuf, PathBuf, PathBuf) { let (one, two, link) = (root.join("one"), root.join("two"), root.join("link")); for parent in [&one, &two] { fs::create_dir(parent).unwrap(); let sandbox = parent.join("sandbox"); provision(&sandbox); - fs::write(sandbox.join(CLONE_BINDING_NAME), b"{}").unwrap(); + write_clone_files(&sandbox); } link_dir(&one, &link); (one, two, link) } +/// The entries of a provisioned sandbox that [`linked_sandboxes`] made, with `more`, sorted. +fn linked_entries(more: &[&str]) -> Vec { + let mut names = vec![ + CLONE_ACTIVATION_NAME, + CLONE_BINDING_NAME, + CLONE_SCRATCH_NAME, + CONFIG, + LOCK_NAME, + RESTORE_READY_NAME, + ]; + names.extend_from_slice(more); + sorted(&names) +} + #[test] fn a_removal_unbinds_the_sandbox_whose_lock_it_holds_whatever_its_path_names_meanwhile() { let directory = tempfile::tempdir().unwrap(); let (one, two, link) = linked_sandboxes(directory.path()); - // The other sandbox records a launch, so its VM may run, and its binding must stay. + // The other sandbox records a launch, so its VM may run, and its binding and the files of its + // clone's launch must stay. fs::write(two.join("sandbox").join(JOURNAL_NAME), b"{}").unwrap(); let locked = SandboxStore::lock_existing(link.join("sandbox"), &files(), None).unwrap(); // The link names the other sandbox's parent from the checks to the binding's removal, and @@ -532,7 +564,7 @@ fn a_removal_unbinds_the_sandbox_whose_lock_it_holds_whatever_its_path_names_mea assert_eq!(entries_of(&one.join("sandbox")), [LOCK_NAME]); assert_eq!( entries_of(&two.join("sandbox")), - sorted(&[CLONE_BINDING_NAME, CONFIG, JOURNAL_NAME, LOCK_NAME]) + linked_entries(&[JOURNAL_NAME]) ); assert!(locked.remove_directory().unwrap().is_none()); assert!(!one.join("sandbox").exists()); @@ -554,10 +586,7 @@ fn a_removal_through_a_path_that_names_another_sandbox_meanwhile_removes_nothing ); assert_eq!(Error::from(error).kind(), ErrorKind::Io); for parent in [&one, &two] { - assert_eq!( - entries_of(&parent.join("sandbox")), - sorted(&[CLONE_BINDING_NAME, CONFIG, LOCK_NAME]) - ); + assert_eq!(entries_of(&parent.join("sandbox")), linked_entries(&[])); } } @@ -1139,6 +1168,57 @@ mod windows { assert!(!dir.exists()); } + #[test] + fn a_removal_takes_the_clone_files_whose_deletion_is_pending_for_removed() { + let (_directory, dir) = sandbox(); + fs::write(dir.join(CLONE_BINDING_NAME), b"{}").unwrap(); + for name in CLONE_FILE_NAMES { + fs::write(dir.join(name), b"").unwrap(); + } + // The removal of what a clone kept of its launch leaves the deletion of each file that a + // reader holds pending, which the removal's own removal of them then finds. + let pending: Vec = CLONE_FILE_NAMES + .into_iter() + .map(|name| delete_pending(&dir.join(name))) + .collect(); + let mut lingering = remove(&dir) + .unwrap() + .expect("the files whose deletion is pending keep the directory"); + // Everything else went, the binding included, and the clone's files linger, as a removed + // lock file does, rather than being refused as entries that are not the store's. + assert_eq!(entries_of(&dir), sorted(&CLONE_FILE_NAMES)); + let error = lingering.wait(Some(Instant::now())).unwrap_err(); + assert!(matches!(error, RemovalError::TimedOut(_)), "{error}"); + drop(pending); + lingering.wait(within(10)).unwrap(); + assert!(!dir.exists()); + } + + #[test] + fn a_retried_removal_of_the_clone_files_takes_those_whose_deletion_is_pending_for_removed() { + let (_directory, dir) = sandbox(); + for name in CLONE_FILE_NAMES { + fs::write(dir.join(name), b"").unwrap(); + } + // A release that failed once it removed the clone's files, which readers hold, retries + // their removal, which finds each one's deletion pending. + let pending: Vec = CLONE_FILE_NAMES + .into_iter() + .map(|name| delete_pending(&dir.join(name))) + .collect(); + let locked = SandboxStore::lock_existing(&dir, &files(), None).unwrap(); + locked.remove_clone_files().unwrap(); + drop(locked); + drop(pending); + let names = entries_of(&dir); + assert!( + names + .iter() + .all(|name| !CLONE_FILE_NAMES.contains(&name.as_str())), + "{names:?}" + ); + } + #[test] fn locks_refuse_a_lock_file_that_refuses_access_for_good() { // A read-only lock file, and a directory at its path, refuse an open for writing as a diff --git a/nvx-rs/src/host/store/mod.rs b/nvx-rs/src/host/store/mod.rs index 0bef346a1..179c7a7f8 100644 --- a/nvx-rs/src/host/store/mod.rs +++ b/nvx-rs/src/host/store/mod.rs @@ -17,6 +17,9 @@ //! clone.json the binding of a clone to the generation of a template that it leases //! clone-scratch.raw the file of the scratch device of a clone whose upper is in RAM, a //! sparse file that the clone's plan creates +//! clone-activation.bin the activation record of a clone, which its launch writes +//! restore.sock the endpoint at which the launch of a clone awaits the restore-ready +//! event of its VMM, on Linux //! ... the caller's configuration record and other files, which it declares //! ``` //! @@ -197,12 +200,13 @@ //! # Removal //! //! [`LockedStore::remove`] removes a sandbox that no VMM can run in: it refuses while the store -//! records a VM or a launch, which the caller reattaches to, or recovers, first. It removes the -//! sandbox's [clone binding](CLONE_BINDING_NAME) first, and makes its absence durable before -//! anything else goes. It checks for those records, and removes the binding, through one handle of -//! the directory, once it has checked through that handle that the directory holds its lock, so -//! that no change of a link in the path meanwhile makes it remove another sandbox's binding. It -//! then removes the store's files and the caller's, the configuration record last, and the +//! records a VM or a launch, which the caller reattaches to, or recovers, first. It removes what a +//! clone kept of its launch, as a release does, and the sandbox's +//! [clone binding](CLONE_BINDING_NAME) first, and makes their absence durable before anything +//! else goes. It checks for those records, and removes those files, through one handle of the +//! directory, once it has checked through that handle that the directory holds its lock, so that +//! no change of a link in the path meanwhile makes it remove another sandbox's files. It then +//! removes the store's other files and the caller's, the configuration record last, and the //! temporary files that the store owns: those that an interrupted replacement of one of its files //! leaves, `.NAME.NUMBER.tmp`, and the VMM's [report staging file](#the-report-staging-file). It //! then refuses entries that are neither the store's nor the caller's, naming them, and keeps the @@ -293,11 +297,32 @@ pub const CLONE_BINDING_NAME: &str = "clone.json"; /// [`template::Store::release`](crate::host::template::Store::release) removes once the clone's VM /// has exited, as a [removal](self#removal) of the sandbox does. pub const CLONE_SCRATCH_NAME: &str = "clone-scratch.raw"; +/// Name of the activation record of a clone, in its canonical encoding, which the clone's launch +/// writes before it spawns the clone's VMM, so that the restore activates the clone with that +/// record and its transaction whatever happens to the launch's caller, and which +/// [`template::Store::release`](crate::host::template::Store::release) removes once the clone's VM +/// has exited, as a [removal](self#removal) of the sandbox does. +pub const CLONE_ACTIVATION_NAME: &str = "clone-activation.bin"; +/// Name of the Linux socket at which the launch of a clone listens for the restore-ready event of +/// its VMM, which OpenVMM sends once it has prepared the restore, before the restored VM runs; on +/// Windows, the endpoint is a named pipe, whose name each launch draws at random. +/// [`template::Store::release`](crate::host::template::Store::release) removes it once the clone's +/// VM has exited, as a [removal](self#removal) of the sandbox does. Its name is as long as +/// [`SOCKET_NAME`], so that both sockets' paths fit a Unix socket's address alike. +pub const RESTORE_READY_NAME: &str = "restore.sock"; +/// The files that the clone of a sandbox keeps of its launch, which a release removes once the +/// clone's VM has exited, as a [removal](self#removal) of the sandbox does, before the clone +/// binding. +const CLONE_FILE_NAMES: [&str; 3] = [ + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, +]; /// Every file that the store keeps in a sandbox's directory. Replacing a record leaves a /// temporary file named `.NAME.NUMBER.tmp` beside it only if the replacement was interrupted, and /// a VMM that is killed while it writes its outcome report leaves the file in which it stages it; /// see [the report staging file](self#the-report-staging-file). -pub const FILE_NAMES: [&str; 10] = [ +pub const FILE_NAMES: [&str; 12] = [ LOCK_NAME, JOURNAL_NAME, RUNTIME_NAME, @@ -308,6 +333,8 @@ pub const FILE_NAMES: [&str; 10] = [ OUTCOME_NAME, CLONE_BINDING_NAME, CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, ]; /// Interval at which a lock that another holder has is tried again: the sandbox's lock, by a @@ -506,6 +533,17 @@ impl SandboxStore { self.path(CLONE_SCRATCH_NAME) } + /// Returns the path of the activation record of a clone, [`CLONE_ACTIVATION_NAME`]. + pub fn clone_activation_path(&self) -> PathBuf { + self.path(CLONE_ACTIVATION_NAME) + } + + /// Returns the path of the Linux socket at which the launch of a clone awaits the + /// restore-ready event of its VMM, [`RESTORE_READY_NAME`]. + pub fn restore_ready_path(&self) -> PathBuf { + self.path(RESTORE_READY_NAME) + } + /// Reads the runtime record, which names the sandbox's VM from the commit of its launch until /// its runtime is cleared. pub fn runtime(&self) -> Result> { @@ -705,6 +743,31 @@ impl LockedStore { Ok(owned) } + /// Removes what the clone of the sandbox kept of its launch in the sandbox's directory, whose + /// VM has exited: the file of its scratch device, [`CLONE_SCRATCH_NAME`], its activation + /// record, [`CLONE_ACTIVATION_NAME`], and the socket of its restore-ready event, + /// [`RESTORE_READY_NAME`]. A release, as a [removal](self#removal) does, removes them before + /// the clone binding, whose removal, once durable, makes theirs durable too, since all are + /// entries of the sandbox's directory. A file whose deletion is pending, as a Windows file + /// system without POSIX deletion leaves a removed file that another handle holds, is removed + /// already, as [`was_removed`](regular::was_removed) tells, so that a release that failed once + /// it removed the file can be retried. + pub(crate) fn remove_clone_files(&self) -> Result<()> { + for name in CLONE_FILE_NAMES { + let path = self.path(name); + match std::fs::remove_file(&path) { + Err(error) + if error.kind() != io::ErrorKind::NotFound + && !regular::was_removed(&path, &error) => + { + return Err(io_error(format!("cannot remove {}", path.display()))(error)); + } + _ => {} + } + } + Ok(()) + } + /// Returns the caller's files that the lock knows: those that a [lifecycle /// lock](self#lifecycle-locks) was taken with, and none for [`SandboxStore::lock`]. pub fn files(&self) -> &CallerFiles { diff --git a/nvx-rs/src/host/store/tests.rs b/nvx-rs/src/host/store/tests.rs index 44300b42d..abba2a806 100644 --- a/nvx-rs/src/host/store/tests.rs +++ b/nvx-rs/src/host/store/tests.rs @@ -401,6 +401,33 @@ fn a_lock_file_that_is_a_link_fails_at_once() { assert_eq!(fs::read(&target).unwrap(), CAPABILITY); } +#[test] +fn the_files_that_a_clone_kept_of_its_launch_go_and_nothing_else() { + let (_directory, store) = store(); + let locked = store.lock(None).unwrap(); + for path in [ + store.clone_scratch_path(), + store.clone_activation_path(), + store.restore_ready_path(), + store.log_path(), + store.binding_path(), + ] { + fs::write(path, b"left").unwrap(); + } + locked.remove_clone_files().unwrap(); + let mut left: Vec = fs::read_dir(store.dir()) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + // The binding goes later, once the release or the removal has removed the clone's files. + let mut expected = [CLONE_BINDING_NAME, LOCK_NAME, LOG_NAME]; + expected.sort_unstable(); + assert_eq!(left, expected); + // Nothing to remove is no failure. + locked.remove_clone_files().unwrap(); +} + #[cfg(unix)] #[test] fn records_and_capabilities_that_are_not_regular_files_fail_at_once() { diff --git a/nvx-rs/src/host/template/store/leases.rs b/nvx-rs/src/host/template/store/leases.rs index 87672428b..e0114c979 100644 --- a/nvx-rs/src/host/template/store/leases.rs +++ b/nvx-rs/src/host/template/store/leases.rs @@ -318,13 +318,16 @@ impl Store { /// [`ErrorCode::SandboxRunning`] while the sandbox records a VM or a launch, which may still /// run. The sandbox's store clears those records only once it has verified that the VM's /// process exited, through its recorded [`ProcessRef`](crate::host::process::ProcessRef), as - /// a stop or a [`reattach`](crate::host::supervisor::reattach) does. The release removes the - /// file of the scratch device of a clone whose upper is in RAM, - /// [`CLONE_SCRATCH_NAME`](crate::host::store::CLONE_SCRATCH_NAME), which the clone's - /// [plan](crate::host::template::ClonePlan) created, then the clone binding, and once their - /// removal is durable, the lease's record in the store, so that an interrupted release leaves - /// a record that no binding names, which a [retirement](Self::retire) removes, and never a - /// binding without its record. It takes no lock of the store. + /// a stop or a [`reattach`](crate::host::supervisor::reattach) does. The release removes what + /// the clone kept of its launch: the file of the scratch device of a clone whose upper is in + /// RAM, [`CLONE_SCRATCH_NAME`](crate::host::store::CLONE_SCRATCH_NAME), which the clone's + /// [plan](crate::host::template::ClonePlan) created, the clone's activation record, + /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME), and the socket of + /// its restore-ready event, [`RESTORE_READY_NAME`](crate::host::store::RESTORE_READY_NAME); + /// then the clone binding, and once their removal is durable, the lease's record in the store, + /// so that an interrupted release leaves a record that no binding names, which a + /// [retirement](Self::retire) removes, and never a binding without its record. It takes no + /// lock of the store. /// /// The caller takes the sandbox's lock through the lease's path, as [`Lease::sandbox`] /// returns it, and as [`acquire`](Self::acquire) takes it, which has no links: a link in the @@ -365,8 +368,9 @@ impl Store { let recorded = self.check_records_dir(LEASES_NAME, LEASES_WHAT)?; // The clone's VM has exited, and the sandbox launches nothing else under the lease, so no // VM holds the scratch device's file of a RAM upper. The binding's removal makes the - // file's durable too, since both are entries of the sandbox's directory. - remove_if_present(&locked.clone_scratch_path())?; + // removal of the clone's files durable too, since all are entries of the sandbox's + // directory. + locked.remove_clone_files()?; unbind(locked.dir(), sync)?; locked.reserve(None); if recorded { diff --git a/nvx-rs/src/host/template/store/leases/tests.rs b/nvx-rs/src/host/template/store/leases/tests.rs index 1c8c26d9e..5018935ca 100644 --- a/nvx-rs/src/host/template/store/leases/tests.rs +++ b/nvx-rs/src/host/template/store/leases/tests.rs @@ -18,7 +18,8 @@ use crate::activation::TemplateReference; use crate::host::process::{Process, ProcessRef}; use crate::host::store::files::write_private; use crate::host::store::{ - CLONE_SCRATCH_NAME, FORMAT, LaunchState, RUNTIME_NAME, Recovery, RuntimeIdentity, + CLONE_ACTIVATION_NAME, CLONE_SCRATCH_NAME, FORMAT, LaunchState, RESTORE_READY_NAME, + RUNTIME_NAME, Recovery, RuntimeIdentity, }; use crate::host::supervisor::{self, Reattached, StopOutcome, VmmCommand}; use crate::host::template::eligibility::MAX_DOWNTIME_NS; @@ -786,18 +787,27 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco .dir() .join(LEASES_NAME) .join(format!("{}.json", lease.id())); - // Each sync notes its directory, and whether the scratch device's file of a clone in RAM, - // the binding, and the record existed then. + // Each sync notes its directory, and whether any file that the clone kept of its launch, the + // binding, and the record existed then. + let kept = [ + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, + ]; let synced = RefCell::new(Vec::new()); let sync = |dir: &Path| -> Result<()> { - let scratch = sandbox.join(CLONE_SCRATCH_NAME).exists(); + let clone = kept + .iter() + .any(|name| fs::symlink_metadata(sandbox.join(name)).is_ok()); let bound = sandbox.join(CLONE_BINDING_NAME).exists(); synced .borrow_mut() - .push((dir.to_owned(), scratch, bound, record.exists())); + .push((dir.to_owned(), clone, bound, record.exists())); Ok(()) }; - fs::write(sandbox.join(CLONE_SCRATCH_NAME), b"").unwrap(); + for name in kept { + fs::write(sandbox.join(name), b"").unwrap(); + } let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); assert_eq!(store.release_with(&mut locked, &sync).unwrap(), Some(lease)); drop(locked); From 82c5164d2987485b416ad25f76d155d6e4e75344 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Sun, 25 Oct 2026 10:15:14 -0700 Subject: [PATCH 3/7] nvx-rs: launch clones as OpenVMM restores of their leased snapshots ClonePlan::launch runs the launch that a clone's lease reserves, under the lock that acquired the lease, once the sandbox's clone binding records the lease. The VMM restores the generation's snapshot in place, which nothing copies, links, or changes, so clones of one generation restore it at once. OpenVMM requires a restore's control socket in the snapshot's parent on Linux, so the lease reserves the clone's control endpoint there: .sock in the generation's payload, named after the launch, which the journal and the runtime record name as any launch's endpoint. On Windows the VMM listens at a fresh named pipe, as every launch does. The launch refuses, before it writes anything, a socket that is not beside the plan's snapshot, a payload whose mode is not exactly 0700, a socket path that, with or without the payload's links resolved, would not fit a Unix socket's address or is not valid UTF-8, as OpenVMM and the host take it, and a control socket path that holds ',' or '=', which end the path in OpenVMM's option listen=. The VMM restores a generation's snapshot in place while the clone's guest writes its scratch image, so a plan refuses a scratch image that could be a file of a snapshot. Once it claims the image, it refuses one in the template store's directory, where the image is now, wherever the links on its path lead: the store's captures write their snapshots there, and their publications make them generations' by renaming directories within it, while nothing that the host does moves a file into the store. It refuses one with another name too, such as a hard link to a file of a generation's snapshot, of a capture's, or of another store's, which its location does not show. These checks open nothing in the store, however many generations it holds. They resolve the file system's own aliases, links, junctions, directories held open, and other names, but not a mount or a network share that exposes the store's directories elsewhere, so a store's directories must not be exposed that way. On Windows, the location names a volume without a drive letter or a mount point by its GUID. The launch also refuses, before it writes anything, a plan in RAM whose scratch device's file is gone, as when a later plan of the clone replaced it and failed. It writes the clone's activation record to clone-activation.bin, listens for the VMM's restore-ready event, and runs supervisor::launch with OpenVMM's restore of the clone's devices: --restore-snapshot with the generation's snapshot, --restore-memory with the memory target, --restore-ready-path, the template's backend, CPU profile, and processors, a boot console that discards its output, as cold boots have, and the control console, without a kernel, command line, or guest address. The VMM inherits a scratch image's claim: its descriptor on Linux, and on Windows the image's guard, once the launch has closed the claim's handle, which OpenVMM's open of the image, sharing no writing, would refuse. The VMM removes its socket as it exits. The end of a launch and the clearing of a runtime record remove one that an abrupt exit leaves, and a release removes it and syncs the payload before it ends the lease, so a payload holds clone sockets only while its generation has leases; a retirement removes the payload with whatever a crash left in it. A release ends the lock's reservation of the launch before it removes anything, so one that fails part of the way leaves no launch of a plan whose files it removed. Tests run the launch with a fake VMM in a process of its own, which listens beside the snapshot on Linux and holds the snapshot open sharing only reads on Windows, as OpenVMM does: the rendered arguments, the record, the claim, two clones of one generation launched at once, the socket's removal at teardown, at release, and after a crash, the refusals before any spawn, and crashes of the launch's caller before it begins, once it began, and once its VMM started. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a424884c-018a-45f5-a7fe-c57548a873f0 --- nvx-rs/README.md | 28 + nvx-rs/src/host/platform/linux.rs | 32 +- nvx-rs/src/host/platform/other.rs | 12 + nvx-rs/src/host/platform/windows.rs | 109 +- nvx-rs/src/host/seal.rs | 8 + nvx-rs/src/host/seal/windows.rs | 41 +- nvx-rs/src/host/store/lease.rs | 24 + nvx-rs/src/host/store/mod.rs | 133 +- nvx-rs/src/host/supervisor/mod.rs | 20 +- nvx-rs/src/host/template/clone.rs | 192 ++- nvx-rs/src/host/template/clone/launch.rs | 562 +++++++ nvx-rs/src/host/template/clone/tests.rs | 232 ++- .../src/host/template/clone/tests/launch.rs | 1407 +++++++++++++++++ nvx-rs/src/host/template/mod.rs | 5 +- nvx-rs/src/host/template/store.rs | 21 +- nvx-rs/src/host/template/store/capture.rs | 5 +- nvx-rs/src/host/template/store/leases.rs | 87 +- .../src/host/template/store/leases/tests.rs | 65 +- 18 files changed, 2872 insertions(+), 111 deletions(-) create mode 100644 nvx-rs/src/host/template/clone/launch.rs create mode 100644 nvx-rs/src/host/template/clone/tests/launch.rs diff --git a/nvx-rs/README.md b/nvx-rs/README.md index f4e895d1f..938ba8b97 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -664,6 +664,34 @@ scratch image may be a file that the sandbox's store owns, such as transitions replace or remove them, and a link or a directory under one of the store's names is refused. +`ClonePlan::launch` then launches the clone as the launch that its lease +reserves, once it has checked that the sandbox's clone binding records the +lease, and returns a `Restoring`: it writes the clone's activation record in its +canonical encoding, `clone-activation.bin`, from which a caller resends it +unchanged; it listens for the VMM's restore-ready event, at the socket +`restore.sock` on Linux, and at a named pipe of a random name on Windows; and it +runs `supervisor::launch` with OpenVMM's restore of the clone's devices, +`--restore-snapshot` with the generation's snapshot in place, which no clone +copies or changes, so that clones of one generation restore it at once, +`--restore-memory` with the memory target, and `--restore-ready-path`, under the +template's backend, CPU profile, and processors, with a boot console that +discards its output, as cold boots have, and the control console, but no kernel, +initramfs, command line, or guest address, which the snapshot fixes. + +On Linux, where OpenVMM requires a restore's control socket in the snapshot's +parent, the clone's VMM listens in the generation's payload, at `.sock`, +named after the launch that the lease reserves, whose path must be shorter than +108 bytes with or without the payload's links resolved; on Windows, it listens +at a named pipe, as every launch does. The VMM inherits the claim of a scratch +image, on Windows through the image's guard, once the launch has closed the +claim's handle, which OpenVMM's open of the image would refuse. + +A payload holds the sockets of its clones only while they lease it: the VMM +removes its socket as it exits, a stop or a recovery removes the one that an +abrupt exit leaves, and the release makes the socket's absence durable before +the lease ends, and removes the record and `restore.sock`, as it removes +`clone-scratch.raw`. + `Store::tombstone` quarantines a generation, which takes no new lease while its leases keep working, and `Store::retire` deletes it once its leases have drained, crash-safely, removing the records of leases that no clone binding diff --git a/nvx-rs/src/host/platform/linux.rs b/nvx-rs/src/host/platform/linux.rs index c26a28516..39f0c80a2 100644 --- a/nvx-rs/src/host/platform/linux.rs +++ b/nvx-rs/src/host/platform/linux.rs @@ -2,7 +2,7 @@ use std::fs::{self, DirBuilder, File, OpenOptions, Permissions, TryLockError}; use std::io::{self, Read}; use std::os::fd::{AsFd, AsRawFd, RawFd}; use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}; -use std::os::unix::net::UnixStream; +use std::os::unix::net::{UnixListener, UnixStream}; use std::path::Path; use std::process::Command; use std::thread; @@ -639,6 +639,11 @@ pub(crate) fn opened_file_identity(file: &File) -> io::Result<(u64, u64)> { Ok((metadata.dev(), metadata.ino())) } +/// Returns the number of names, its hard links, of the file that `file` holds open. +pub(crate) fn link_count(file: &File) -> io::Result { + Ok(file.metadata()?.nlink()) +} + /// An open directory, through which operations name its entries, so that they reach the entries /// of the directory that it opened, whatever its path names meanwhile. pub(crate) struct Directory(File); @@ -753,6 +758,31 @@ pub(crate) fn control_endpoint(socket_path: &Path) -> io::Result { .ok_or_else(|| io::Error::other("control socket path is not valid UTF-8")) } +/// The listener at which the launch of a clone awaits the restore-ready event of its VMM: a Unix +/// socket, which the VMM connects to once, and which queues its connection until the host +/// accepts it. +#[derive(Debug)] +pub(crate) struct ReadyListener { + /// The listening socket, which the VMM finds as long as it is open. + _socket: UnixListener, +} + +/// Listens at the Unix socket `socket_path`, in place of a socket that an earlier launch left +/// there, as the restore-ready endpoint of a clone's launch, and returns the endpoint, as +/// OpenVMM's `--restore-ready-path` takes it, and its listener. +pub(crate) fn listen_ready(socket_path: &Path) -> io::Result<(String, ReadyListener)> { + let endpoint = socket_path + .to_str() + .map(str::to_owned) + .ok_or_else(|| io::Error::other("restore-ready socket path is not valid UTF-8"))?; + match fs::remove_file(socket_path) { + Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error), + _ => {} + } + let socket = UnixListener::bind(socket_path)?; + Ok((endpoint, ReadyListener { _socket: socket })) +} + /// Connects to a control endpoint served by the process `expected_pid`. /// /// The connection attempt never blocks: a listener whose backlog is full yields diff --git a/nvx-rs/src/host/platform/other.rs b/nvx-rs/src/host/platform/other.rs index 57af35f57..537e71959 100644 --- a/nvx-rs/src/host/platform/other.rs +++ b/nvx-rs/src/host/platform/other.rs @@ -98,6 +98,10 @@ pub(crate) fn opened_file_identity(_file: &File) -> io::Result<(u64, u64)> { Err(unsupported()) } +pub(crate) fn link_count(_file: &File) -> io::Result { + Err(unsupported()) +} + /// An open directory, which this host never holds: it keeps no sandbox's store. pub(crate) enum Directory {} @@ -210,6 +214,14 @@ pub(crate) fn control_endpoint(_socket_path: &Path) -> io::Result { Err(unsupported()) } +/// The listener of the restore-ready endpoint of a clone's launch, which this host never makes. +#[derive(Debug)] +pub(crate) struct ReadyListener; + +pub(crate) fn listen_ready(_socket_path: &Path) -> io::Result<(String, ReadyListener)> { + Err(unsupported()) +} + pub(crate) fn connect_endpoint( _endpoint: &str, _expected_pid: u32, diff --git a/nvx-rs/src/host/platform/windows.rs b/nvx-rs/src/host/platform/windows.rs index 287f4a566..5099d7533 100644 --- a/nvx-rs/src/host/platform/windows.rs +++ b/nvx-rs/src/host/platform/windows.rs @@ -33,10 +33,10 @@ use windows_sys::Win32::Security::{ use windows_sys::Win32::Storage::FileSystem::{ BY_HANDLE_FILE_INFORMATION, CreateDirectoryW, CreateFileW, DELETE, FILE_ALL_ACCESS, FILE_APPEND_DATA, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_DELETE_CHILD, - FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_FLAG_OVERLAPPED, - FILE_WRITE_ATTRIBUTES, FILE_WRITE_DATA, FILE_WRITE_EA, GetFileInformationByHandle, MoveFileExW, - OPEN_EXISTING, READ_CONTROL, ReadFile, SECURITY_IDENTIFICATION, SECURITY_SQOS_PRESENT, - WRITE_DAC, WRITE_OWNER, WriteFile, + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OPEN_REPARSE_POINT, + FILE_FLAG_OVERLAPPED, FILE_WRITE_ATTRIBUTES, FILE_WRITE_DATA, FILE_WRITE_EA, + GetFileInformationByHandle, MoveFileExW, OPEN_EXISTING, PIPE_ACCESS_INBOUND, READ_CONTROL, + ReadFile, SECURITY_IDENTIFICATION, SECURITY_SQOS_PRESENT, WRITE_DAC, WRITE_OWNER, WriteFile, }; #[cfg(feature = "cli")] use windows_sys::Win32::System::Console::{ @@ -49,7 +49,10 @@ use windows_sys::Win32::System::IO::{ use windows_sys::Win32::System::LibraryLoader::{ GetProcAddress, LOAD_LIBRARY_SEARCH_SYSTEM32, LoadLibraryExW, }; -use windows_sys::Win32::System::Pipes::{GetNamedPipeServerProcessId, WaitNamedPipeW}; +use windows_sys::Win32::System::Pipes::{ + CreateNamedPipeW, GetNamedPipeServerProcessId, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, + PIPE_TYPE_BYTE, PIPE_WAIT, WaitNamedPipeW, +}; use windows_sys::Win32::System::Registry::{ HKEY_LOCAL_MACHINE, RRF_RT_REG_DWORD, RRF_RT_REG_SZ, RegGetValueW, }; @@ -1007,6 +1010,35 @@ impl CurrentUser { /// that allows others is not restricted, since a handle that they opened keeps its access. pub(crate) fn create_store_dir(path: &Path) -> io::Result> { let user = CurrentUser::get()?; + let name = wide(path); + with_owner_only_attributes( + &user, + FILE_ALL_ACCESS, + OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE, + |attributes| { + // SAFETY: the name is NUL-terminated, and the attributes hold a valid descriptor. + if unsafe { CreateDirectoryW(name.as_ptr(), attributes) } == 0 { + let error = last_error(); + if error != ERROR_ALREADY_EXISTS { + return Err(io::Error::from_raw_os_error(error as i32)); + } + } + Ok(()) + }, + )??; + store_dir_problem(path, &user) +} + +/// Runs `create` with the attributes of a new object that `user` owns, whose protected access +/// control list allows `user` alone, `access`, with an entry that the objects created in it +/// inherit as `inheritance` says, and returns what `create` returns. The attributes let no child +/// process inherit the object's handle. +fn with_owner_only_attributes( + user: &CurrentUser, + access: u32, + inheritance: u32, + create: impl FnOnce(&SECURITY_ATTRIBUTES) -> T, +) -> io::Result { let sid = user.sid(); // SAFETY: the identifier is valid. let sid_len = unsafe { GetLengthSid(sid) } as usize; @@ -1025,13 +1057,7 @@ pub(crate) fn create_store_dir(path: &Path) -> io::Result> { // its use below. let built = unsafe { InitializeAcl(acl_ptr, acl_len, ACL_REVISION) != 0 - && AddAccessAllowedAceEx( - acl_ptr, - ACL_REVISION, - OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE, - FILE_ALL_ACCESS, - sid, - ) != 0 + && AddAccessAllowedAceEx(acl_ptr, ACL_REVISION, inheritance, access, sid) != 0 && InitializeSecurityDescriptor(descriptor_ptr, SECURITY_DESCRIPTOR_REVISION) != 0 && SetSecurityDescriptorOwner(descriptor_ptr, sid, 0) != 0 && SetSecurityDescriptorDacl(descriptor_ptr, 1, acl_ptr, 0) != 0 @@ -1046,15 +1072,7 @@ pub(crate) fn create_store_dir(path: &Path) -> io::Result> { lpSecurityDescriptor: descriptor_ptr, bInheritHandle: 0, }; - let name = wide(path); - // SAFETY: the name is NUL-terminated, and the attributes hold a valid descriptor. - if unsafe { CreateDirectoryW(name.as_ptr(), &attributes) } == 0 { - let error = last_error(); - if error != ERROR_ALREADY_EXISTS { - return Err(io::Error::from_raw_os_error(error as i32)); - } - } - store_dir_problem(path, &user) + Ok(create(&attributes)) } /// Returns why the directory `path` cannot hold a sandbox store, if it cannot: unless it is @@ -1576,6 +1594,13 @@ pub(crate) fn opened_file_identity(file: &fs::File) -> io::Result<(u64, u64)> { handle_identity(file.as_raw_handle() as HANDLE) } +/// Returns the number of names, its hard links, of the file that `file` holds open. +pub(crate) fn link_count(file: &fs::File) -> io::Result { + Ok(u64::from( + handle_information(file.as_raw_handle() as HANDLE)?.nNumberOfLinks, + )) +} + /// Returns the volume serial number and file index of the file that `handle` holds open. fn handle_identity(handle: HANDLE) -> io::Result<(u64, u64)> { let information = handle_information(handle)?; @@ -1647,6 +1672,48 @@ pub(crate) fn control_endpoint(_socket_path: &Path) -> io::Result { Ok(format!("//./pipe/openvmm-microvm-{suffix}")) } +/// The bytes that the inbound buffer of a restore-ready pipe holds, more than OpenVMM's event. +const READY_BUFFER_LEN: u32 = 4096; + +/// The listener at which the launch of a clone awaits the restore-ready event of its VMM: the +/// one instance of a named pipe, inbound, which only the current user may open, from this host +/// alone, and which takes the connection of the VMM before the host accepts it. +#[derive(Debug)] +pub(crate) struct ReadyListener { + /// The pipe's instance, which the VMM finds as long as it is open. + _pipe: OwnedHandle, +} + +/// Listens at a fresh named pipe, whose name it draws at random, as the restore-ready endpoint of +/// a clone's launch, and returns the endpoint, as OpenVMM's `--restore-ready-path` takes it, and +/// its listener. The first instance of the pipe is the listener's, so no other process holds the +/// name, and the pipe's protected access control list allows the current user alone. +pub(crate) fn listen_ready(_socket_path: &Path) -> io::Result<(String, ReadyListener)> { + let mut bytes = [0u8; 16]; + getrandom::fill(&mut bytes).map_err(io::Error::other)?; + let suffix: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + let endpoint = format!("//./pipe/nvx-restore-ready-{suffix}"); + let name = wide(endpoint.replace('/', "\\")); + let pipe = + with_owner_only_attributes(&CurrentUser::get()?, FILE_ALL_ACCESS, 0, |attributes| { + // SAFETY: the name is NUL-terminated, and the attributes hold a valid descriptor. + let handle = unsafe { + CreateNamedPipeW( + name.as_ptr(), + PIPE_ACCESS_INBOUND | FILE_FLAG_FIRST_PIPE_INSTANCE, + PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, + 1, + 0, + READY_BUFFER_LEN, + 0, + attributes, + ) + }; + owned(handle).ok_or_else(io::Error::last_os_error) + })??; + Ok((endpoint, ReadyListener { _pipe: pipe })) +} + /// Connects to a named pipe served by the process `expected_pid`. pub(crate) fn connect_endpoint( endpoint: &str, diff --git a/nvx-rs/src/host/seal.rs b/nvx-rs/src/host/seal.rs index 91f5e75dc..02a0844ff 100644 --- a/nvx-rs/src/host/seal.rs +++ b/nvx-rs/src/host/seal.rs @@ -237,6 +237,14 @@ pub(crate) fn final_path(file: &File) -> io::Result { imp::final_path(file) } +/// Returns the final path of the open `file` as [`final_path`] does, but with its volume named by +/// its GUID, `\\?\Volume{GUID}\`, which names a volume that has no drive letter or mount point, +/// and so no path by a drive, too. +#[cfg(windows)] +pub(crate) fn final_volume_path(file: &File) -> io::Result { + imp::final_volume_path(file) +} + /// Opens `path` for its metadata alone, so that no read or mapping of its content can succeed /// through the file: with `O_PATH` on Linux, and on Windows with `FILE_READ_ATTRIBUTES` alone, /// sharing every access. diff --git a/nvx-rs/src/host/seal/windows.rs b/nvx-rs/src/host/seal/windows.rs index ea6236c20..27c5e00d2 100644 --- a/nvx-rs/src/host/seal/windows.rs +++ b/nvx-rs/src/host/seal/windows.rs @@ -12,8 +12,9 @@ use std::path::{Path, PathBuf}; use windows_sys::Win32::Foundation::{FILETIME, GetLastError, HANDLE}; use windows_sys::Win32::Storage::FileSystem::{ BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, FILE_FLAG_BACKUP_SEMANTICS, FILE_ID_INFO, - FILE_SHARE_READ, FILE_TYPE_DISK, FILE_TYPE_UNKNOWN, FileIdInfo, GetFileInformationByHandle, - GetFileInformationByHandleEx, GetFileType, GetFinalPathNameByHandleW, VOLUME_NAME_DOS, + FILE_SHARE_READ, FILE_TYPE_DISK, FILE_TYPE_UNKNOWN, FileIdInfo, GETFINALPATHNAMEBYHANDLE_FLAGS, + GetFileInformationByHandle, GetFileInformationByHandleEx, GetFileType, + GetFinalPathNameByHandleW, VOLUME_NAME_DOS, VOLUME_NAME_GUID, }; use super::{FileId, FileKind, FileSeal, SEAL_VERSION, SealScheme, SealTime}; @@ -110,18 +111,23 @@ pub(super) fn open_read(path: &Path) -> io::Result { /// See [`super::final_path`]. pub(super) fn final_path(file: &File) -> io::Result { + final_path_by(file, VOLUME_NAME_DOS) +} + +/// See [`super::final_volume_path`]. +pub(super) fn final_volume_path(file: &File) -> io::Result { + final_path_by(file, VOLUME_NAME_GUID) +} + +/// Returns the final path of the open `file`, which names its volume as `volume` asks. +fn final_path_by(file: &File, volume: GETFINALPATHNAMEBYHANDLE_FLAGS) -> io::Result { let handle = file.as_raw_handle() as HANDLE; let mut buffer = vec![0u16; 260]; loop { // SAFETY: the handle is open for as long as `file` is borrowed, and the buffer holds // `buffer.len()` units. let length = unsafe { - GetFinalPathNameByHandleW( - handle, - buffer.as_mut_ptr(), - buffer.len() as u32, - VOLUME_NAME_DOS, - ) + GetFinalPathNameByHandleW(handle, buffer.as_mut_ptr(), buffer.len() as u32, volume) } as usize; if length == 0 { return Err(io::Error::last_os_error()); @@ -216,4 +222,23 @@ mod tests { let file = File::open(deep.join("long")).unwrap(); assert!(final_path(&file).unwrap().as_os_str().len() > 260); } + + #[test] + fn final_volume_paths_name_the_file_by_its_volume() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("file"); + std::fs::write(&path, b"").unwrap(); + let found = final_volume_path(&File::open(&path).unwrap()).unwrap(); + assert!( + found.to_string_lossy().starts_with(r"\\?\Volume{"), + "{}", + found.display() + ); + // The path names the file, which its final path by its drive names too. + let reopened = File::open(&found).unwrap(); + assert_eq!( + final_path(&reopened).unwrap(), + std::fs::canonicalize(&path).unwrap() + ); + } } diff --git a/nvx-rs/src/host/store/lease.rs b/nvx-rs/src/host/store/lease.rs index 1ce7ac867..73096bb21 100644 --- a/nvx-rs/src/host/store/lease.rs +++ b/nvx-rs/src/host/store/lease.rs @@ -6,6 +6,7 @@ //! [`template::Store`](crate::host::template::Store), grants and ends leases, and reads its own //! records, with these, and [`template`](crate::host::template) re-exports what is public here. +use std::ffi::OsStr; use std::fmt; use std::io; use std::num::NonZeroU64; @@ -24,6 +25,9 @@ use crate::host::platform::{self, parse_hex_id, parse_hex32}; /// Version of the store's records: its identity record and its index. pub const STORE_FORMAT: u32 = 1; +/// Name of the directory of the generations' payloads. +pub const GENERATIONS_NAME: &str = "generations"; + /// Bytes of a [`StoreId`] and of a [`StorageGeneration`](crate::host::template::StorageGeneration). pub(crate) const ID_LEN: usize = 16; @@ -247,6 +251,26 @@ impl fmt::Debug for Lease { } } +/// Returns the name of the control socket of the VMM of the clone whose launch is `launch`, in the +/// payload of the leased generation: `.sock`. +pub(crate) fn clone_socket_name(launch: &LaunchId) -> String { + format!("{launch}.sock") +} + +/// Returns whether `path` names the control socket of the VMM of the clone that holds `lease`, as +/// [`template::Store::clone_socket_path`](crate::host::template::Store::clone_socket_path) returns +/// it: `generations//.sock`, in a template store, which a lease names by its +/// identity alone. +pub(crate) fn is_clone_socket(lease: &Lease, path: &Path) -> bool { + let name = clone_socket_name(&lease.launch); + let generation = lease.generation.to_string(); + let payload = path.parent(); + path.file_name() == Some(OsStr::new(&name)) + && payload.and_then(Path::file_name) == Some(OsStr::new(&generation)) + && payload.and_then(Path::parent).and_then(Path::file_name) + == Some(OsStr::new(GENERATIONS_NAME)) +} + /// Reads the clone binding of the sandbox in `dir`, or returns `None` if it holds none, or if the /// directory does not exist. pub(crate) fn read_binding(dir: &Path) -> Result> { diff --git a/nvx-rs/src/host/store/mod.rs b/nvx-rs/src/host/store/mod.rs index 179c7a7f8..2ae8d97cb 100644 --- a/nvx-rs/src/host/store/mod.rs +++ b/nvx-rs/src/host/store/mod.rs @@ -8,7 +8,8 @@ //! launch.json the journal of a launch in progress, or of one that was interrupted //! runtime.json the runtime record of its running VM //! control.capability the launch capability of its VM -//! control.sock the control endpoint of its VM, on Linux +//! control.sock the control endpoint of its VM, on Linux, but for a clone, whose VMM +//! listens beside the snapshot that it restores //! openvmm.log the VMM's output, claimed by the launch that created it //! console.log the guest console of the latest launch, or of every launch, if the //! caller captures it @@ -144,7 +145,8 @@ //! the VMM through the control endpoint that the journal names, holds it, confirms that the //! endpoint still names it once held, and terminates it. //! On Windows the endpoint is a pipe whose name each launch draws at random, which names the -//! VMM by itself. On Linux it is the same socket path for every launch of the sandbox, so the +//! VMM by itself. On Linux it is the same socket path for every launch of the sandbox, but the +//! one launch of a clone, whose VMM listens at a socket that its lease reserves, so the //! VMM must also hold the claimed log as its standard output. Until the VMM serves its //! endpoint, recovery is indeterminate and keeps the journal; once the VMM exits, the released //! claim proves it. @@ -249,7 +251,7 @@ use self::files::{io_error, remove_if_present}; pub use self::identity::{ ArtifactIdentity, IdentityCheck, IdentityPart, MAX_IDENTITY_LEN, RuntimeIdentity, }; -use self::lease::Lease; +use self::lease::{Lease, is_clone_socket}; pub use self::lifecycle::{Lingering, RemovalError}; pub use self::record::{ LaunchId, LaunchJournal, LaunchState, LogClaim, MAX_ENDPOINT_LEN, RuntimeRecord, @@ -275,7 +277,10 @@ pub const JOURNAL_NAME: &str = "launch.json"; pub const RUNTIME_NAME: &str = "runtime.json"; /// Name of the file that holds the launch capability. pub const CAPABILITY_NAME: &str = "control.capability"; -/// Name of the Linux control socket. +/// Name of the Linux control socket, at which the VMM of every launch of the sandbox listens, but +/// that of a clone, which listens beside the snapshot that it restores, at the socket that its +/// lease reserves, as [`template::Store::acquire`](crate::host::template::Store::acquire) +/// describes. pub const SOCKET_NAME: &str = "control.sock"; /// Name of the VMM's log. pub const LOG_NAME: &str = "openvmm.log"; @@ -297,18 +302,19 @@ pub const CLONE_BINDING_NAME: &str = "clone.json"; /// [`template::Store::release`](crate::host::template::Store::release) removes once the clone's VM /// has exited, as a [removal](self#removal) of the sandbox does. pub const CLONE_SCRATCH_NAME: &str = "clone-scratch.raw"; -/// Name of the activation record of a clone, in its canonical encoding, which the clone's launch -/// writes before it spawns the clone's VMM, so that the restore activates the clone with that -/// record and its transaction whatever happens to the launch's caller, and which +/// Name of the activation record of a clone, in its canonical encoding, which the clone's launch, +/// [`template::ClonePlan::launch`](crate::host::template::ClonePlan::launch), writes before it +/// spawns the clone's VMM, so that the restore activates the clone with that record and its +/// transaction whatever happens to the launch's caller, and which /// [`template::Store::release`](crate::host::template::Store::release) removes once the clone's VM /// has exited, as a [removal](self#removal) of the sandbox does. pub const CLONE_ACTIVATION_NAME: &str = "clone-activation.bin"; -/// Name of the Linux socket at which the launch of a clone listens for the restore-ready event of -/// its VMM, which OpenVMM sends once it has prepared the restore, before the restored VM runs; on -/// Windows, the endpoint is a named pipe, whose name each launch draws at random. -/// [`template::Store::release`](crate::host::template::Store::release) removes it once the clone's -/// VM has exited, as a [removal](self#removal) of the sandbox does. Its name is as long as -/// [`SOCKET_NAME`], so that both sockets' paths fit a Unix socket's address alike. +/// Name of the Linux socket at which the launch of a clone, +/// [`template::ClonePlan::launch`](crate::host::template::ClonePlan::launch), listens for the +/// restore-ready event of its VMM, which OpenVMM sends once it has prepared the restore, before +/// the restored VM runs; on Windows, the endpoint is a named pipe, whose name each launch draws at +/// random. [`template::Store::release`](crate::host::template::Store::release) removes it once the +/// clone's VM has exited, as a [removal](self#removal) of the sandbox does. pub const RESTORE_READY_NAME: &str = "restore.sock"; /// The files that the clone of a sandbox keeps of its launch, which a release removes once the /// clone's VM has exited, as a [removal](self#removal) of the sandbox does, before the clone @@ -686,7 +692,24 @@ pub struct LockedStore { lock_removed: bool, /// The launch of the sandbox's clone that the lease acquired under this lock reserves, until /// [`begin_launch`](Self::begin_launch) begins it, or a release ends the lease. - reserved: Option, + reserved: Option, +} + +/// The launch of a sandbox's clone that a lease reserves, as +/// [`template::Store::acquire`](crate::host::template::Store::acquire) acquires the lease, and +/// where the clone's VMM listens. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Reservation { + /// The identity of the launch, [`Lease::launch`]. + pub(crate) launch: LaunchId, + /// The path of the Linux socket at which the clone's VMM listens, its control endpoint: beside + /// the snapshot that it restores, in the payload of the leased generation, since OpenVMM + /// requires a restore's control socket in the snapshot's parent, rather than the sandbox's + /// own control socket. Windows endpoints are named pipes, which [`begin_launch`] draws as it + /// does for every launch. + /// + /// [`begin_launch`]: LockedStore::begin_launch + pub(crate) socket: PathBuf, } impl Deref for LockedStore { @@ -710,17 +733,28 @@ impl LockedStore { } } - /// Lets [`begin_launch`](Self::begin_launch) begin `launch`, once, as the launch of the - /// sandbox's clone that the lease acquired under this lock reserves, or, given `None`, no - /// launch of a clone, once a release ends the lease. - pub(crate) fn reserve(&mut self, launch: Option) { - self.reserved = launch; + /// Lets [`begin_launch`](Self::begin_launch) begin the launch that `reservation` names, once, + /// as the launch of the sandbox's clone that the lease acquired under this lock reserves, + /// whose VMM listens at the reservation's socket, or, given `None`, no launch of a clone, once + /// a release ends the lease. + pub(crate) fn reserve(&mut self, reservation: Option) { + self.reserved = reservation; } /// Returns the launch of the sandbox's clone that the lease acquired under this lock reserves, /// until [`begin_launch`](Self::begin_launch) begins it, or a release ends the lease. pub(crate) fn reserved(&self) -> Option<&LaunchId> { - self.reserved.as_ref() + self.reserved + .as_ref() + .map(|reservation| &reservation.launch) + } + + /// Returns the path of the Linux socket at which the VMM of the launch that + /// [`reserved`](Self::reserved) returns listens. + pub(crate) fn reserved_socket(&self) -> Option<&Path> { + self.reserved + .as_ref() + .map(|reservation| reservation.socket.as_path()) } /// Returns the paths at which the store reaches the entries of the sandbox's directory that @@ -784,6 +818,12 @@ impl LockedStore { /// of a caller that keeps the console of every launch, as [`CallerFiles::keeping_console`] /// declares. Nothing is spawned yet: the caller spawns the VMM with [`Launch::take_log`]. /// + /// The control endpoint is a named pipe of a random name on Windows. On Linux, it is the + /// sandbox's control socket, [`socket_path`](SandboxStore::socket_path), but for the launch + /// that the lease of the sandbox's clone reserves: its VMM restores the leased generation's + /// snapshot, and listens beside it, at the socket that the lease reserves in the generation's + /// payload, as [`template::Store::acquire`](crate::host::template::Store::acquire) describes. + /// /// Fails if the sandbox's VM runs, or a launch was interrupted and is not recovered yet, and /// with [`ErrorKind::Unsupported`] for an all-zero capability or an invalid runtime identity. /// A sandbox whose clone holds a lease of a template's generation, which its @@ -817,7 +857,7 @@ impl LockedStore { ))); } if let Some(lease) = Lease::of(self)? - && (lease.launch() != launch || self.reserved.as_ref() != Some(launch)) + && (lease.launch() != launch || self.reserved() != Some(launch)) { let reservation = if lease.launch() == launch { format!("whose launch {launch} begins once, under the lock that acquired the lease") @@ -844,7 +884,13 @@ impl LockedStore { remove_if_present(&self.console_path())?; } self.remove_report_staging()?; - let endpoint = platform::control_endpoint(&socket) + // The VMM of a clone listens at the socket that its lease reserves, beside the snapshot + // that it restores. + let listening = match &self.reserved { + Some(reservation) if reservation.launch == *launch => reservation.socket.as_path(), + _ => &socket, + }; + let endpoint = platform::control_endpoint(listening) .map_err(|error| Error::io("cannot choose a control endpoint").with_source(error))?; let (log, claim) = self.create_log()?; files::write_private(&self.capability_path(), capability)?; @@ -917,6 +963,7 @@ impl LockedStore { for name in [CAPABILITY_NAME, SOCKET_NAME] { remove_if_present(&self.path(name))?; } + self.remove_clone_socket(journal.endpoint())?; self.remove_report_staging()?; // The journal goes last, so that an interrupted removal leaves it for a later // recovery to finish. @@ -1025,14 +1072,54 @@ impl LockedStore { /// Removes the runtime record, the launch capability, and the control socket of a VM that has /// exited, and any outcome report that its VMM staged without publishing it, the runtime - /// record last. The log, console log, and outcome report outlive the run. + /// record last. The control socket of a clone's VMM, beside the snapshot that it restores, goes + /// too. The log, console log, and outcome report outlive the run. pub fn clear_runtime(&mut self) -> Result<()> { + // Only a Linux endpoint is a file, which may be a clone's beside its snapshot. A record that + // cannot be read names none, and a release removes a clone's socket all the same. + let record = if cfg!(unix) { + self.runtime().ok().flatten() + } else { + None + }; + self.clear_runtime_of(record.as_ref()) + } + + /// Clears the runtime record of a VM that has exited, as [`clear_runtime`](Self::clear_runtime) + /// does, given `record`, the runtime record that the caller read under this lock, if any, whose + /// control endpoint is the VM's. + pub(crate) fn clear_runtime_of(&mut self, record: Option<&RuntimeRecord>) -> Result<()> { for name in [CAPABILITY_NAME, SOCKET_NAME] { remove_if_present(&self.path(name))?; } + if let Some(record) = record { + self.remove_clone_socket(record.endpoint())?; + } self.remove_report_staging()?; remove_if_present(&self.path(RUNTIME_NAME)) } + + /// Removes the control socket at `endpoint`, the control endpoint that a launch of the + /// sandbox recorded, if it is the socket of the sandbox's clone beside the snapshot that its + /// VMM restores, which the lease that the sandbox's clone binding records reserved, as + /// [`template::Store::clone_socket_path`](crate::host::template::Store::clone_socket_path) + /// names it. Any other endpoint stays: the sandbox's own control socket, which the caller + /// removes, and a named pipe. Neither reads the binding. + /// + /// The removal is not made durable here: + /// [`template::Store::release`](crate::host::template::Store::release) makes it durable before + /// the lease ends, and a retirement removes whatever a crash brings back with the generation's + /// payload. + fn remove_clone_socket(&self, endpoint: &str) -> Result<()> { + let socket = Path::new(endpoint); + if !cfg!(unix) || socket == self.socket_path() { + return Ok(()); + } + match Lease::of(self)? { + Some(lease) if is_clone_socket(&lease, socket) => remove_if_present(socket), + _ => Ok(()), + } + } } /// Ends a launch by terminating its verified VMM `process`. diff --git a/nvx-rs/src/host/supervisor/mod.rs b/nvx-rs/src/host/supervisor/mod.rs index 1b2bc240e..49299a3e2 100644 --- a/nvx-rs/src/host/supervisor/mod.rs +++ b/nvx-rs/src/host/supervisor/mod.rs @@ -115,12 +115,22 @@ //! upper, and memory target against the generation, and claims the clone's scratch image, if //! it has one, as //! [`ScratchBinding::check`](crate::host::sandbox::scratch::ScratchBinding::check) does. It -//! then runs [`launch`] as the reserved launch, which [`LockedStore::begin_launch`] requires of -//! a sandbox that holds a lease, and begins once, under that lock alone, whatever its outcome. +//! then launches the clone with +//! [`ClonePlan::launch`](crate::host::template::ClonePlan::launch), which checks that the +//! binding records the lease, writes the clone's files in the sandbox, and runs [`launch`] as +//! the reserved launch, which [`LockedStore::begin_launch`] requires of a sandbox that holds a +//! lease, and begins once, under that lock alone, whatever its outcome. The clone's VMM +//! restores the generation's snapshot in place, and on Linux listens beside it, at the control +//! socket that the lease reserves in the generation's payload, named after the launch, which +//! [`Starting::endpoint`] and the runtime record name, as every launch's endpoint, rather than +//! at the sandbox's own socket. //! 3. Once the clone's VM has exited, as a stop, a termination, or a reattach verifies, //! [`Store::release`](crate::host::template::Store::release) ends the lease, under the //! sandbox's lock, taken through the lease's path, -//! [`Lease::sandbox`](crate::host::template::Lease::sandbox), as the acquisition took it. +//! [`Lease::sandbox`](crate::host::template::Lease::sandbox), as the acquisition took it. The +//! VMM removes its control socket as it exits, the stop or the reattach removes the one that an +//! abrupt exit leaves, and the release makes the socket's absence durable before the lease +//! ends. //! //! ## The lock order //! @@ -543,7 +553,7 @@ impl Vmm { .runtime()? .is_some_and(|record| record == self.record) { - locked.clear_runtime()?; + locked.clear_runtime_of(Some(&self.record))?; } Ok(()) } @@ -761,7 +771,7 @@ fn reattach_with( let process = match open(record.process()) { Ok(Some(process)) => process, Ok(None) => { - locked.clear_runtime()?; + locked.clear_runtime_of(Some(&record))?; return Ok(Reattached::Exited { record: Some(record), recovery, diff --git a/nvx-rs/src/host/template/clone.rs b/nvx-rs/src/host/template/clone.rs index f705cbaf9..07a20a326 100644 --- a/nvx-rs/src/host/template/clone.rs +++ b/nvx-rs/src/host/template/clone.rs @@ -1,13 +1,17 @@ //! The plan of a clone, [`ClonePlan`]: the generation of a template that a clone leased, with the //! clone's own image, upper, and memory target, which the plan checks against the generation //! before the clone's launch, so that a request that the template cannot serve fails before any -//! VMM starts. +//! VMM starts; and the clone's launch, [`ClonePlan::launch`], OpenVMM's restore of the +//! generation's snapshot with what the plan checked. #[cfg(all(test, any(target_os = "linux", windows)))] mod tests; +mod launch; mod record; +pub use self::launch::Restoring; + use std::fmt; use std::fs::{self, File}; use std::io; @@ -29,7 +33,8 @@ use crate::host::sandbox::distro::{self, CheckedDistro, DistroBinding, DistroFor use crate::host::sandbox::plan::bound_image; use crate::host::sandbox::scratch::{self, CheckedScratch, ScratchBinding}; use crate::host::sandbox::{ - LayerRole, SandboxLayer, SandboxPlan, block_geometry, reject_container_path, reject_disk_path, + LayerRole, SandboxLayer, SandboxPlan, SandboxUpper, block_geometry, reject_container_path, + reject_disk_path, }; use crate::host::store::LockedStore; use crate::host::store::files::{create_new, remove_if_present}; @@ -132,8 +137,20 @@ pub enum CloneUpper { /// for the clone, and present the geometry of the template's scratch, its length included, /// which [`CheckedScratch::geometry`] reads through the checked file. At its path, it must be a /// regular file, not a link, nor on Windows any other reparse point, that is the file that the -/// check found, and that no one else may change. The plan holds the checked image, and with it -/// the claim. Nothing reads, formats, or copies the image. +/// check found, and that no one else may change. Nor may it be a file of the snapshot of any of +/// the store's generations, by any path, through a hard link, say: every clone of a generation +/// restores its snapshot in place, while its guest writes its scratch image. So, once claimed, +/// it must be outside the store's directory, wherever the links on its path lead: the store's +/// generations hold their snapshots there, and its captures write theirs there, which their +/// publications make generations' by renaming directories within it, so a file in the store +/// could be, or become as the plan examines it, a file of a snapshot, while nothing that the +/// host does moves a file into the store's directory. And it must have a single name: another, +/// a hard link, could make it a file of a snapshot, of this store or of another, that its +/// location does not show. These checks open nothing in the store, however many generations it +/// holds. They resolve the aliases that the file system makes, links, junctions, directories +/// held open, and other names, but not a mount or a network share that exposes the store's +/// directories elsewhere: a store's directories must not be exposed that way. The plan holds +/// the checked image, and with it the claim. Nothing reads, formats, or copies the image. /// - The memory target must be whole blocks of [`MEMORY_BLOCK_SIZE`], and of the template's own /// memory blocks, from the template's RAM to its RAM capacity. /// - The activation record, which the clone's guest applies in place of the configuration that @@ -141,17 +158,18 @@ pub enum CloneUpper { /// upper, and settings, the template's network, and the resources that the clone must have, /// as [`activation`](Self::activation) describes. Its image must be an EROFS layer, and its /// settings must keep the limits of a record, as [`Configuration::check`] checks them. -/// - The capability is fresh: the plan draws it for the clone's launch. +/// - The capability is fresh: the plan draws it for the clone's launch, [`launch`](Self::launch). /// /// The plan's checks read the metadata of the clone's files, never their content. /// /// ```no_run +/// use std::path::Path; /// use std::time::{Duration, Instant}; /// /// use nvx::activation::{Journal, MachineIdMode, OverlayMode, WorkloadView}; /// use nvx::host::sandbox::distro::{DISTRO_POLICY_REVISION, DistroBinding}; /// use nvx::host::sandbox::{SandboxDistro, SandboxPlan, SandboxUpper}; -/// use nvx::host::store::CallerFiles; +/// use nvx::host::store::{CallerFiles, RuntimeIdentity}; /// use nvx::host::template::{ /// CloneRequest, ClonePlan, Generation, HostTime, MEMORY_BLOCK_SIZE, Store, /// }; @@ -161,6 +179,7 @@ pub enum CloneUpper { /// # generation: &Generation, /// # image: SandboxDistro, /// # binding: DistroBinding, +/// # runtime: &RuntimeIdentity, /// # ) -> nvx::Result<()> { /// let files = CallerFiles::new("config.json")?; /// let mut acquired = store.acquire( @@ -186,10 +205,12 @@ pub enum CloneUpper { /// }; /// let deadline = Instant::now() + Duration::from_secs(5); /// let plan = ClonePlan::new(store, &mut acquired, &request, deadline)?; -/// // ... the clone's launch, under `acquired.locked`, as the launch that the lease reserves, of -/// // `plan.snapshot()` with `plan.image().path()` and `plan.scratch_path()`, whose guest applies -/// // `plan.activation()` ... -/// # let _ = plan; +/// // OpenVMM restores the generation's snapshot under the runtime that captured the template. +/// let restoring = plan.launch(&mut acquired, Path::new("openvmm"), runtime)?; +/// // ... the restore, whose VMM sends its restore-ready event to `restoring.ready_endpoint()`, +/// // serves its control console at `restoring.starting().endpoint()`, and whose guest applies +/// // `restoring.activation()` ... +/// # let _ = restoring; /// # Ok(()) /// # } /// ``` @@ -214,6 +235,9 @@ pub struct ClonePlan { activation: ActivationRecord, /// The capability of the clone's launch. capability: [u8; CAPABILITY_LEN], + /// The clone's devices, which its restore attaches: the adapter's plan of its image and + /// network, over the file of its `scratch` block device. + devices: SandboxPlan, } /// How a plan reaches the clone's files: as the checks of their bindings reach them, and as it @@ -245,7 +269,7 @@ impl ClonePlan { /// /// The plan is made under the lock that acquired the lease, which `acquired` holds, before the /// launch that the lease reserves, [`Lease::launch`], which the caller then runs under the - /// same lock: the sandbox records no VM and no launch, so no VM of an earlier clone holds a + /// same lock, with [`launch`](Self::launch): the sandbox records no VM and no launch, so no VM of an earlier clone holds a /// file that the plan replaces. It takes `acquired` exclusively while it plans, since it may /// create a file in the sandbox's directory, as the transitions of a sandbox's store take its /// lock, and borrows nothing of it once it returns. It reads the generation's key, @@ -296,13 +320,15 @@ impl ClonePlan { /// comma or a semicolon, or whose filesystem presents another geometry than the template's /// scratch, and with [`ErrorKind::Io`] if the sparse file cannot be created, sized, or /// checked; - /// - for an upper on a scratch image, as [`ScratchBinding::check`] and - /// [`CheckedScratch::geometry`] fail, with [`ErrorKind::Unsupported`] for an image of - /// another geometry than the template's scratch, its length included, with - /// [`ErrorKind::Unavailable`] for an image that is not, at its path, the regular file that - /// the check found, of the current user or an administrator, and that no one else may - /// change, and with [`ErrorKind::Unsupported`] for an image that is a file of the sandbox's - /// store. + /// - for an upper on a scratch image, with [`ErrorKind::Io`] if the image's location cannot be + /// compared with the template store's directory, or its names cannot be counted, as + /// [`ScratchBinding::check`] and [`CheckedScratch::geometry`] fail, with + /// [`ErrorKind::Unsupported`] for an image of another geometry than the template's scratch, + /// its length included, with [`ErrorKind::Unavailable`] for an image that is not, at its + /// path, the regular file that the check found, of the current user or an administrator, + /// and that no one else may change, and with [`ErrorKind::Unsupported`] for an image that + /// is a file of the sandbox's store, or that is in the template store's directory, as the + /// files of its snapshots are, or has another name. /// /// [`Configuration::check`]: crate::activation::Configuration::check pub fn new( @@ -360,12 +386,18 @@ impl ClonePlan { let capability = generate_capability()?; let upper = match (request.adapter.upper().scratch_path(), &request.scratch) { (Some(path), Some(binding)) => CloneUpper::Scratch(check_scratch( - path, binding, &key, &owned, deadline, access, + path, + binding, + &key, + (&owned, store), + deadline, + access, )?), _ => CloneUpper::Tmpfs { path: create_scratch(locked, &key, access)?, }, }; + let devices = devices_of(request, &upper); Ok(Self { lease: lease.clone(), generation, @@ -376,6 +408,7 @@ impl ClonePlan { memory: request.memory, activation, capability, + devices, }) } @@ -473,6 +506,18 @@ impl fmt::Debug for ClonePlan { } } +/// Returns the devices of the clone of `request`, whose upper is `upper`: the adapter's plan of its +/// image and network, over the file of its `scratch` block device, which every restore attaches, +/// the sparse file of an upper in RAM included. +fn devices_of(request: &CloneRequest, upper: &CloneUpper) -> SandboxPlan { + let adapter = &request.adapter; + let scratch = match upper { + CloneUpper::Tmpfs { path } => SandboxUpper::scratch_at(paths::normalized(path)), + CloneUpper::Scratch(_) => adapter.upper().clone(), + }; + SandboxPlan::adapter(adapter.distro().clone(), scratch).with_network(adapter.network().cloned()) +} + /// Refuses a request that no clone takes, as [`ClonePlan::new`] describes, without reading or /// opening anything. fn check_request(request: &CloneRequest) -> Result<()> { @@ -647,14 +692,15 @@ fn check_image( /// Checks and claims the scratch image at `path` against `binding`, through `access`, waiting /// until `deadline` for a claim that another launch holds to be released, and then checks its -/// geometry, against the template's scratch that `key` records, the image at its path, and that -/// it is none of `owned`, the files of the sandbox's store, as [`ClonePlan`] describes, and +/// geometry, against the template's scratch that `key` records, the image at its path, that it +/// is none of `owned`, the files of the sandbox's store, that it is outside the directory of +/// `store`, the template store, and that it has no other name, as [`ClonePlan`] describes, and /// returns the image that the binding's check opened and claimed. fn check_scratch( path: &Path, binding: &ScratchBinding, key: &TemplateKey, - owned: &StoreFiles, + (owned, store): (&StoreFiles, &Store), deadline: Instant, access: Access<'_>, ) -> Result { @@ -672,9 +718,111 @@ fn check_scratch( .map_err(|error| refused(format!("cannot be identified: {error}")))?; check_at_path(path, identity, refused)?; owned.check(identity, "scratch image", path)?; + check_outside_store(&checked, store, path)?; + check_single_name(&checked, path)?; Ok(checked) } +/// Refuses the clone's scratch image at `path`, which `checked` holds open and claims, if it is in +/// the directory of `store`, the template store, where it is now, every link on the way to it +/// resolved: the store's generations hold their snapshots there, which every clone of a +/// generation restores in place, and its captures write theirs there, which their publications +/// make generations' by renaming directories within it, so a file in the store could be, or +/// become as the plan examines it, a file of such a snapshot. Nothing that the host does moves a +/// file into the store's directory, so one outside it stays outside it. The location resolves the +/// aliases that the file system makes, links and junctions on the way and directories held open, +/// but not a mount or a network share that exposes the store's directories elsewhere, which +/// [`ClonePlan`] leaves to the system's administration. +fn check_outside_store(checked: &CheckedScratch, store: &Store, path: &Path) -> Result<()> { + let failed = |error: io::Error| { + Error::io(format!( + "cannot compare the location of the scratch image {} with the template store in {}", + path.display(), + store.dir().display() + )) + .with_source(error) + }; + let location = location_of(checked.file()).map_err(failed)?; + if is_within(&location, store.dir()).map_err(failed)? { + return Err(Error::unsupported(format!( + "the scratch image {} is {}, in the directory of the template store {}, where its \ + captures write the snapshots that every clone of a generation restores in place: a \ + clone's scratch image must be outside it", + path.display(), + location.display(), + store.dir().display() + ))); + } + Ok(()) +} + +/// Returns whether the directory `dir` is one of those above `location`, by their identities, past +/// those that went from their paths meanwhile, as a publication renames a capture's transaction, +/// which `dir`, the store's own directory, outlives. +fn is_within(location: &Path, dir: &Path) -> io::Result { + let dir = platform::file_identity(dir)?; + for ancestor in location.ancestors().skip(1) { + match platform::file_identity(ancestor) { + Ok(identity) if identity == dir => return Ok(true), + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::NotADirectory + ) => {} + Err(error) => return Err(error), + } + } + Ok(false) +} + +/// Returns where the open `file` is now, every link on the way to it resolved: on Linux, the +/// target of its link in `/proc/self/fd`, which follows renames of its directories, and on +/// Windows its final path, which names its volume by a drive letter or a mount point, or else by +/// the volume's GUID. +fn location_of(file: &File) -> io::Result { + #[cfg(target_os = "linux")] + { + use std::os::fd::AsRawFd; + + fs::read_link(Path::new("/proc/self/fd").join(file.as_raw_fd().to_string())) + } + #[cfg(windows)] + { + // A volume without a drive letter or a mount point has no path by a drive. + crate::host::seal::final_path(file) + .or_else(|error| crate::host::seal::final_volume_path(file).map_err(|_| error)) + } + #[cfg(not(any(target_os = "linux", windows)))] + { + let _ = file; + Err(io::Error::from(io::ErrorKind::Unsupported)) + } +} + +/// Refuses the clone's scratch image at `path`, which `checked` holds open, unless it has a single +/// name: another name, a hard link, could make it a file of a snapshot that the clone's guest then +/// writes, of one of the store's generations or captures, or of another store, which the image's +/// location does not show. +fn check_single_name(checked: &CheckedScratch, path: &Path) -> Result<()> { + let links = platform::link_count(checked.file()).map_err(|error| { + Error::io(format!( + "cannot count the names of the scratch image {}", + path.display() + )) + .with_source(error) + })?; + if links > 1 { + return Err(Error::unsupported(format!( + "the scratch image {} has {links} names, but a clone's scratch image must have a \ + single name: another, a hard link, could make it a file of a snapshot, which every \ + clone of its generation restores in place", + path.display() + ))); + } + Ok(()) +} + /// The regular files that the store of a sandbox owns, with their identities, which no image or /// scratch image of a clone may be. struct StoreFiles(Vec<(PathBuf, (u64, u128))>); diff --git a/nvx-rs/src/host/template/clone/launch.rs b/nvx-rs/src/host/template/clone/launch.rs new file mode 100644 index 000000000..f1ba7bd3c --- /dev/null +++ b/nvx-rs/src/host/template/clone/launch.rs @@ -0,0 +1,562 @@ +//! The launch of a clone, [`ClonePlan::launch`]: OpenVMM's restore of the snapshot of the leased +//! generation, as the launch that the clone's lease reserves, with the clone's devices, its +//! activation record, and the endpoint at which the launch awaits the VMM's restore-ready event. + +use std::ffi::{OsStr, OsString}; +use std::fmt; +use std::path::{Path, PathBuf}; + +use super::{ClonePlan, CloneUpper, MIB, SPARSE_WHAT, check_geometry}; +use crate::activation::ActivationRecord; +use crate::error::{Error, Result}; +use crate::host::artifacts; +use crate::host::filesystem; +use crate::host::platform::{self, ReadyListener}; +use crate::host::sandbox::distro::CheckedDistro; +use crate::host::sandbox::{SandboxPlan, block_geometry}; +use crate::host::store::files::write_private; +use crate::host::store::regular::open_regular; +use crate::host::store::{LockedStore, Recovery, RuntimeIdentity}; +use crate::host::supervisor::{self, Inherited, Starting, VmmCommand}; +use crate::host::template::key::TemplateKey; +use crate::host::template::manifest::BlockGeometry; +use crate::host::template::store::{Acquired, Lease, SOCKET_PATH_LIMIT}; +use crate::protocol::CAPABILITY_LEN; + +/// A point of a clone's launch at which a test interrupts it, before the step that follows the +/// point. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Point { + /// The sandbox holds the clone's activation record, the launch listens for the restore-ready + /// event, and the launch has not begun. + Prepared, + /// The launch's journal names it, and its VMM is not spawned yet. + Begun, +} + +/// The seams of a clone's launch, through which tests observe and interrupt it. +#[derive(Clone, Copy)] +pub(super) struct Hooks<'a> { + /// Returns the command that starts the VMM, given the one that the launch made. + pub(super) command: &'a dyn for<'h> Fn(VmmCommand<'h>) -> VmmCommand<'h>, + /// Fails the launch at a point, as a test asks. + pub(super) interrupt: &'a dyn Fn(Point) -> Result<()>, +} + +impl Hooks<'static> { + /// The seams of every launch outside tests. + pub(super) const LAUNCH: Self = Self { + command: &unchanged, + interrupt: &uninterrupted, + }; +} + +/// Returns `command`, as launches outside tests start their VMMs. +fn unchanged(command: VmmCommand<'_>) -> VmmCommand<'_> { + command +} + +/// Lets a launch pass every point, as launches outside tests do. +fn uninterrupted(_: Point) -> Result<()> { + Ok(()) +} + +impl ClonePlan { + /// Launches the clone that the plan describes, under the lock that acquired its lease, which + /// `acquired` holds, as the launch that the lease reserves: the OpenVMM executable `openvmm`, + /// which a relative path names from the current directory, restores the leased generation's + /// snapshot with the clone's devices, under the runtime identity `runtime`, which must be the + /// template's, [`TemplateKey::runtime`], and receives the plan's capability. Returns the + /// launch, whose VMM runs, as a [`Restoring`]. + /// + /// The launch first checks, without writing anything, that `acquired` holds the plan's lease, + /// under the lock that acquired it, before the launch that it reserves began, and that the + /// sandbox's clone binding records the lease: the acquisition made the binding durable, and no + /// VMM of the clone starts without it. For an upper in RAM, it checks that the file of the + /// scratch device that the plan created is still there: a later plan under the same + /// acquisition replaces it, and removes it if that plan fails. It then: + /// + /// 1. Writes the clone's activation record, in its canonical encoding, as the file + /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME) in the sandbox's + /// directory, in place of one that an earlier launch left, before the VMM starts: the + /// record, with the transaction ID of its restore section, is the one that the clone + /// activates with, which the caller, or a caller that takes the sandbox over after a + /// crash, reads there again and resends unchanged. + /// 2. Listens at the endpoint of the VMM's restore-ready event, which OpenVMM connects to once + /// it has prepared the restore, before the restored VM runs, and which must exist before + /// the VMM starts: on Linux, the socket + /// [`RESTORE_READY_NAME`](crate::host::store::RESTORE_READY_NAME) in the sandbox's + /// directory, in place of one that an earlier launch left; on Windows, a named pipe whose + /// name the launch draws at random. + /// 3. Runs [`supervisor::launch`] as the launch that the lease reserves, which begins it with + /// the store's launch journal, starts the VMM, and records its identity. + /// + /// The VMM restores the generation's snapshot in place, in the generation's payload, which it + /// only reads, and which the launch neither copies nor changes, so any number of clones + /// restore one generation at once. OpenVMM requires the control socket of a restore's VMM on + /// Linux in the snapshot's parent, so the VMM listens beside the snapshot, at the socket that + /// the lease reserved, `.sock` in the payload, after the launch's identity, rather + /// than at the sandbox's own control socket, as + /// [`Store::acquire`](crate::host::template::Store::acquire) describes; on Windows, at a named + /// pipe, as every launch does. [`Starting::endpoint`] returns it. + /// + /// The VMM's arguments are those of the clone's devices: the microVM machine, the clone's + /// image as the read-only `distro` block device and the file of its `scratch` block device, + /// which OpenVMM opens at their paths, as [Block devices by + /// path](crate::host::template::Store#block-devices-by-path) describes, and the policy of the + /// template's NIC, if it has one, without the guest's address, which OpenVMM takes from the + /// snapshot. Then come `--single-process`, the template's backend, CPU profile, and + /// processors, `--restore-snapshot` with the directory of the generation's snapshot, + /// [`ClonePlan::snapshot`], `--restore-memory` with the memory target, `--restore-ready-path` + /// with the endpoint of the restore-ready event, a boot console that discards its output, as + /// cold boots have, the control console at the launch's endpoint, which authenticates with + /// the capability that the VMM reads from its standard input, and the outcome report, + /// [`SandboxStore::outcome_path`](crate::host::store::SandboxStore::outcome_path). They name + /// no kernel, initramfs, kernel command line, or RAM, which the snapshot fixes. The VMM runs + /// in the sandbox's directory. + /// + /// The claim of a scratch image passes to the VMM, which holds it until it exits. On Linux, + /// the VMM inherits the claim's descriptor, which no argument names. On Windows, the claim's + /// handle, open for writing, would make OpenVMM's open of the image fail, since that open + /// shares no writing: the launch closes it before the VMM starts, and the VMM inherits the + /// image's guard instead, which keeps every other claim out, and the image's path naming the + /// image, while the VMM runs. The [`Restoring`] holds the plan's image, open for reading. + /// + /// A crash of the caller at any step leaves the clone's files in the sandbox, which a release + /// of the lease, [`Store::release`](crate::host::template::Store::release), removes, and a + /// launch that the journal names, which a reattach ends, as + /// [Crashes](crate::host::supervisor#crashes) describes, removing the control socket that the + /// VMM, if it started, left beside the snapshot. + /// + /// Like every launch of the sandbox's store, the launch names the sandbox's files by their + /// paths, under its lock: the activation record that it writes, the restore-ready socket at + /// which it listens on Linux, and the journal, capability, and log with which it begins; and + /// the VMM, which runs in the sandbox's directory, opens the files that its arguments name + /// there. So the directories above the sandbox's must be ones that only the current user, or + /// an administrator, may change, as the [store](crate::host::store) requires of them, and as + /// [Block devices by path](crate::host::template::Store#block-devices-by-path) requires of + /// those above the clone's block devices. + /// + /// Fails, and starts no VMM, checking in this order: + /// + /// - with [`ErrorKind::Other`](crate::ErrorKind::Other) if `acquired` holds another lease + /// than the plan's, under another lock than the one that acquired it, or after the launch + /// that it reserves began, and if the sandbox's clone binding does not record the plan's + /// lease, and as [`Lease::of`] fails, before anything is written; + /// - with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a runtime identity + /// other than the template's; + /// - for an upper in RAM, with [`ErrorKind::Unavailable`](crate::ErrorKind::Unavailable) if + /// the file of its scratch device, which the plan created, cannot be opened, as when a later + /// plan of the clone replaced it and failed, and with + /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) if it no longer presents the + /// template's scratch geometry; + /// - on Linux, with [`ErrorKind::Other`](crate::ErrorKind::Other) if the control socket that + /// the lease reserves is not beside the plan's snapshot, as for a plan and a lease that + /// name the template store by different directories; with + /// [`ErrorKind::Io`](crate::ErrorKind::Io) if the generation's payload cannot be resolved + /// or opened, or is not a directory of the current user with mode 0700, as OpenVMM requires + /// of the directory of a control socket; and with + /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) if the path of the control + /// socket, with or without the payload's links resolved, or of the restore-ready socket, is + /// not shorter than 108 bytes, as a Unix socket's address requires, or is not valid UTF-8, + /// as OpenVMM and the host take it, or if the path of the control socket holds a comma or an + /// equals sign, which OpenVMM's option `listen=` cannot take; + /// - with [`ErrorKind::Unavailable`](crate::ErrorKind::Unavailable) for an `openvmm` whose + /// path cannot be resolved; + /// - with [`ErrorKind::Io`](crate::ErrorKind::Io) if the activation record cannot be written, + /// or the endpoint of the restore-ready event cannot be opened; + /// - and as [`supervisor::launch`] fails, which ends the launch. + /// + /// A launch that fails leaves the clone's files that it wrote for the release. + pub fn launch<'s>( + self, + acquired: &'s mut Acquired, + openvmm: &Path, + runtime: &RuntimeIdentity, + ) -> Result> { + self.launch_with(acquired, openvmm, runtime, Hooks::LAUNCH) + } + + /// Launches the clone as [`launch`](Self::launch) does, through `hooks`. + pub(super) fn launch_with<'s>( + self, + acquired: &'s mut Acquired, + openvmm: &Path, + runtime: &RuntimeIdentity, + hooks: Hooks<'_>, + ) -> Result> { + let Acquired { lease, locked } = acquired; + self.check_launch(lease, locked, runtime)?; + let openvmm = artifacts::absolute(openvmm)?; + let Self { + lease: _, + generation: _, + key, + snapshot, + image, + upper, + memory, + activation, + capability, + devices, + } = self; + let devices = device_arguments(&devices)?; + write_private(&locked.clone_activation_path(), &activation.encode())?; + let (ready, listener) = + platform::listen_ready(&locked.restore_ready_path()).map_err(|error| { + Error::io(format!( + "cannot listen for the restore-ready event of the clone in {}", + locked.dir().display() + )) + .with_source(error) + })?; + (hooks.interrupt)(Point::Prepared)?; + // On Windows, this closes the claim's handle, and keeps the image's guard. + let claim = match upper { + CloneUpper::Scratch(checked) => Some(checked.into_file()), + CloneUpper::Tmpfs { .. } => None, + }; + // No spawn passes a descriptor of a standard stream, on which the claim is if the caller + // had closed that stream: it moves above them, with its open file description. + #[cfg(target_os = "linux")] + let claim = match claim { + Some(mut file) => { + crate::host::sys::move_above_standard_streams(&mut file).map_err(|error| { + Error::io("cannot pass the claim of the clone's scratch image to its VMM") + .with_source(error) + })?; + Some(file) + } + None => None, + }; + let restore = Restore { + devices, + key: &key, + snapshot, + memory, + ready: &ready, + report: locked.outcome_path(), + }; + let dir = locked.dir().to_owned(); + let inherit: Vec> = claim.iter().map(Inherited::new).collect(); + let starting = + supervisor::launch(locked, lease.launch(), runtime, &capability, |endpoint| { + (hooks.interrupt)(Point::Begun)?; + let mut command = VmmCommand::new(&openvmm, restore.arguments(endpoint), &dir); + command.inherit = inherit; + Ok((hooks.command)(command)) + }); + // The VMM holds the claim, if it started. + drop(claim); + Ok(Restoring { + starting: starting?, + ready, + listener, + image, + activation, + capability, + }) + } + + /// Refuses a launch of the plan through `lease` and `locked` that is not the one that the + /// plan's lease reserves, or whose binding the sandbox does not record, one under `runtime` + /// that the template does not take, and one of a plan in RAM whose scratch device's file is + /// gone, as [`launch`](Self::launch) describes, without writing anything. + fn check_launch( + &self, + lease: &Lease, + locked: &LockedStore, + runtime: &RuntimeIdentity, + ) -> Result<()> { + let planned = &self.lease; + if lease != planned + || locked.dir() != planned.sandbox() + || locked.reserved() != Some(planned.launch()) + { + return Err(Error::other(format!( + "a clone is launched under the lock that acquired the lease of its plan, {}, \ + before the launch that the lease reserves begins, but the lock of the sandbox in \ + {} holds the lease {}, and reserves no launch {} of it", + planned.id(), + locked.dir().display(), + lease.id(), + planned.launch() + ))); + } + if Lease::of(locked)?.as_ref() != Some(planned) { + return Err(Error::other(format!( + "the sandbox in {} records no clone binding of the lease {}, which its \ + acquisition makes durable before any VMM of the clone starts", + locked.dir().display(), + planned.id() + ))); + } + if runtime != self.key.runtime() { + return Err(Error::unsupported(format!( + "the template's generation {} restores under the runtime that captured it, {:?}, \ + not under {runtime:?}", + self.generation.generation(), + self.key.runtime() + ))); + } + if let CloneUpper::Tmpfs { path } = &self.upper { + check_sparse(path, self.key.scratch())?; + } + if cfg!(target_os = "linux") + && let Some(socket) = locked.reserved_socket() + { + self.check_sockets(socket, &locked.restore_ready_path())?; + } + Ok(()) + } + + /// Refuses, on Linux, a control socket of the clone's VMM at `socket`, beside the plan's + /// snapshot, and a socket of the restore-ready event at `ready`, that OpenVMM or the host + /// could not use, as [`launch`](Self::launch) describes, without writing anything. + /// + /// OpenVMM requires the control socket of a restore in the snapshot's parent, a directory of + /// the current user with mode 0700, which the payload of a generation is, and listens at the + /// socket's path with the parent's links resolved, while the host connects at the path that it + /// recorded: each must be shorter than 108 bytes, as a Unix socket's address requires, and + /// valid UTF-8, as OpenVMM and the host take them, as must the path of the restore-ready + /// socket. OpenVMM takes the control socket's path in its option `listen=`, among + /// options that commas separate, and whose values follow an equals sign, so the path must hold + /// neither. + fn check_sockets(&self, socket: &Path, ready: &Path) -> Result<()> { + let (Some(payload), Some(name)) = (socket.parent(), socket.file_name()) else { + return Err(Error::other(format!( + "the control socket {} of the clone's VMM names no directory", + socket.display() + ))); + }; + let resolve = |path: &Path| { + filesystem::canonicalize(path).map_err(|error| { + Error::io(format!( + "cannot resolve the payload of the generation {}", + path.display() + )) + .with_source(error) + }) + }; + let canonical = resolve(payload)?; + let beside = match self.snapshot.parent() { + Some(parent) if parent == payload => true, + Some(parent) => resolve(parent)? == canonical, + None => false, + }; + if !beside { + return Err(Error::other(format!( + "the control socket {} of the clone's VMM, which its lease reserves, is not beside \ + the snapshot {} of its plan, where OpenVMM requires it", + socket.display(), + self.snapshot.display() + ))); + } + let problem = platform::check_store_dir(payload).map_err(|error| { + Error::io(format!( + "cannot open the payload of the generation {}", + payload.display() + )) + .with_source(error) + })?; + if let Some(problem) = problem { + return Err(Error::io(format!( + "the payload of the generation {} {problem}", + payload.display() + ))); + } + let resolved = canonical.join(name); + for path in [socket, &resolved, ready] { + if path.as_os_str().len() >= SOCKET_PATH_LIMIT { + return Err(Error::unsupported(format!( + "the path of the socket {} of a clone's launch must be shorter than \ + {SOCKET_PATH_LIMIT} bytes, as a Unix socket's address requires", + path.display() + ))); + } + } + for path in [socket, &resolved, ready] { + if path.to_str().is_none() { + return Err(Error::unsupported(format!( + "the path of the socket {} of a clone's launch must be valid UTF-8, as \ + OpenVMM takes it", + path.display() + ))); + } + } + if socket.to_string_lossy().contains([',', '=']) { + return Err(Error::unsupported(format!( + "the path of the control socket {} of a clone's VMM must hold no ',' or '=', \ + which end the path in OpenVMM's option `listen=`", + socket.display() + ))); + } + Ok(()) + } +} + +/// Refuses the file of the `scratch` block device of a clone whose upper is in RAM, at `path`, +/// which the clone's plan created, unless it is still a regular file that presents the template's +/// scratch geometry, `scratch`: OpenVMM opens it by its path, and a later plan under the same +/// acquisition replaces it, or removes it if that plan fails. +fn check_sparse(path: &Path, scratch: BlockGeometry) -> Result<()> { + let shown = path.display(); + let file = open_regular(path).map_err(|error| { + Error::unavailable(format!( + "the {SPARSE_WHAT} {shown}, which the clone's plan created, cannot be opened, as when \ + a later plan of the clone replaced it and failed" + )) + .with_source(error) + })?; + check_geometry( + block_geometry(&file, path, SPARSE_WHAT)?, + scratch, + &format!("the {SPARSE_WHAT} {shown}"), + "scratch", + ) +} + +/// Returns OpenVMM's arguments of the devices of a clone's restore, `devices`: those that +/// [`SandboxPlan::openvmm_arguments`] renders, without `--net` and the guest's address, which +/// OpenVMM takes from the snapshot, and refuses beside it. +fn device_arguments(devices: &SandboxPlan) -> Result> { + let mut arguments = devices.openvmm_arguments()?; + if let Some(at) = arguments.iter().position(|argument| argument == "--net") { + arguments.drain(at..(at + 2).min(arguments.len())); + } + Ok(arguments) +} + +/// What OpenVMM's arguments of a clone's restore name besides its control endpoint. +struct Restore<'a> { + /// The arguments of the clone's devices, as [`device_arguments`] renders them. + devices: Vec, + /// The generation's key. + key: &'a TemplateKey, + /// The directory of the snapshot that the VMM restores. + snapshot: PathBuf, + /// The memory target, in bytes. + memory: u64, + /// The endpoint of the restore-ready event. + ready: &'a str, + /// The path of the VMM's outcome report. + report: PathBuf, +} + +impl Restore<'_> { + /// Returns OpenVMM's arguments of the restore, whose control console listens on `endpoint`, + /// as [`ClonePlan::launch`] describes them. + fn arguments(&self, endpoint: &str) -> Vec { + let key = self.key; + let profile = key.cpu_profile(); + let processors = key.processors().to_string(); + let memory = format!("{}M", self.memory / MIB); + let listen = format!("listen={endpoint}"); + let mut arguments: Vec = self.devices.iter().map(OsString::from).collect(); + for argument in [ + OsStr::new("--single-process"), + OsStr::new("--hypervisor"), + OsStr::new(key.hypervisor().as_str()), + OsStr::new("--cpu-profile"), + OsStr::new(&profile), + OsStr::new("--processors"), + OsStr::new(&processors), + OsStr::new("--restore-snapshot"), + self.snapshot.as_os_str(), + OsStr::new("--restore-memory"), + OsStr::new(&memory), + OsStr::new("--restore-ready-path"), + OsStr::new(self.ready), + OsStr::new("--virtio-console"), + OsStr::new("none"), + OsStr::new("--microvm-control-console"), + OsStr::new(&listen), + OsStr::new("--microvm-control-auth-stdin"), + OsStr::new("--microvm-report"), + self.report.as_os_str(), + ] { + arguments.push(argument.to_owned()); + } + arguments + } +} + +/// The launch of a clone whose VMM restores the leased generation's snapshot, as +/// [`ClonePlan::launch`] returns it: the [`Starting`] launch of the VMM, whose journal records +/// the VMM's identity, the endpoint at which the launch awaits the VMM's restore-ready event, the +/// clone's activation record and capability, and its image, open for reading. +/// +/// It borrows the sandbox's locked store exclusively, as [`Starting`] does. Dropping it abandons +/// the launch, as [`abandon`](Self::abandon) does, ignoring failures, and stops listening for the +/// restore-ready event; a crash of the caller leaves the launch's journal instead, for +/// [`LockedStore::recover`]. +pub struct Restoring<'s> { + /// The launch, whose journal records the VMM. + starting: Starting<'s>, + /// The endpoint of the restore-ready event, as OpenVMM's `--restore-ready-path` takes it. + ready: String, + /// The listener of the restore-ready event. + listener: ReadyListener, + /// The clone's image, open for reading, which the plan checked. + image: CheckedDistro, + /// The clone's activation record. + activation: ActivationRecord, + /// The capability of the launch. + capability: [u8; CAPABILITY_LEN], +} + +impl<'s> Restoring<'s> { + /// Returns the launch, whose VMM runs, and whose journal records its identity. + pub fn starting(&self) -> &Starting<'s> { + &self.starting + } + + /// Returns the endpoint at which the launch awaits the restore-ready event of the VMM, as + /// OpenVMM's `--restore-ready-path` takes it: the path of a Unix socket on Linux, and the name + /// of a named pipe, `//./pipe/NAME`, on Windows. + pub fn ready_endpoint(&self) -> &str { + &self.ready + } + + /// Returns the clone's activation record, as the sandbox holds it. + pub fn activation(&self) -> &ActivationRecord { + &self.activation + } + + /// Returns the capability of the launch, with which the control console of the VMM + /// authenticates the caller. + pub fn capability(&self) -> &[u8; CAPABILITY_LEN] { + &self.capability + } + + /// Returns the clone's image, open for reading, which the plan checked. + pub fn image(&self) -> &CheckedDistro { + &self.image + } + + /// Ends the launch, as [`Starting::abandon`] does, which terminates its VMM and ends the + /// launch once nothing of it runs, and stops listening for the restore-ready event. + pub fn abandon(self) -> Result { + self.starting.abandon() + } + + /// Returns the launch, which a test commits as a clone's restore does once its VMM runs. + #[cfg(test)] + pub(super) fn into_starting(self) -> Starting<'s> { + self.starting + } +} + +impl fmt::Debug for Restoring<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + // The capability authenticates control sessions, so it is never shown. + formatter + .debug_struct("Restoring") + .field("starting", &self.starting) + .field("ready", &self.ready) + .field("listener", &self.listener) + .field("image", &self.image) + .field("activation", &self.activation) + .finish_non_exhaustive() + } +} diff --git a/nvx-rs/src/host/template/clone/tests.rs b/nvx-rs/src/host/template/clone/tests.rs index a34880567..fc686ce6c 100644 --- a/nvx-rs/src/host/template/clone/tests.rs +++ b/nvx-rs/src/host/template/clone/tests.rs @@ -1,6 +1,8 @@ //! Tests of the plans of clones: the checks of their images, uppers, and memory targets against //! the generation that they lease, which read no content, the claims of their scratch images, and -//! the sparse scratch device of an upper in RAM. +//! the sparse scratch device of an upper in RAM; and, in [`launch`], of their launches. + +mod launch; use std::io::Read; use std::net::{IpAddr, Ipv4Addr}; @@ -22,6 +24,7 @@ use crate::host::sandbox::scratch::CLAIM_RELEASE_WAIT; use crate::host::sandbox::{SandboxDistro, SandboxNetwork, SandboxSettings, SandboxUpper}; use crate::host::seal::{FileSeal, open_metadata_only}; use crate::host::store::{CLONE_SCRATCH_NAME, CallerFiles, LOG_NAME, SandboxStore}; +use crate::host::template::store::{CAPTURES_NAME, SNAPSHOT_NAME, TRANSACTION_NAME}; use crate::host::template::test_support::{MIB, ManifestSpec, class, network_class}; use crate::host::template::{CaptureClass, HostTime}; use crate::host::test_support::released; @@ -87,6 +90,25 @@ fn stage(path: &Path, length: u64) { options.open(path).unwrap().set_len(length).unwrap(); } +/// Makes `link` a junction to the directory `target`, quoting both paths, since `cmd` splits its +/// arguments at commas and semicolons. +#[cfg(windows)] +fn junction(link: &Path, target: &Path) { + use std::os::windows::process::CommandExt; + use std::process::{Command, Stdio}; + + let status = Command::new("cmd") + .raw_arg(format!( + "/C mklink /J \"{}\" \"{}\"", + link.display(), + target.display() + )) + .stdout(Stdio::null()) + .status() + .unwrap(); + assert!(status.success(), "cannot link {}", link.display()); +} + /// Returns the physical block size that OpenVMM presents for the files of the directory `dir`: /// the I/O block size of its filesystem on Linux, and 4096 bytes on Windows. fn physical_block_size(dir: &Path) -> u32 { @@ -267,9 +289,18 @@ impl Fixture { /// Publishes a template as [`new`](Self::new) does, of the class and manifest that `change` /// changes, given the physical block size of the files of the directory. fn with(change: impl FnOnce(&mut CaptureClass, &mut ManifestSpec, u32)) -> Self { + Self::in_store(|directory| directory.join("templates"), change) + } + + /// Publishes a template as [`with`](Self::with) does, in a store at the path that `store` + /// returns, given the temporary directory, in which it creates the store's parent directory. + fn in_store( + store: impl FnOnce(&Path) -> PathBuf, + change: impl FnOnce(&mut CaptureClass, &mut ManifestSpec, u32), + ) -> Self { let directory = tempfile::tempdir().unwrap(); let physical = physical_block_size(directory.path()); - let store = Store::create(directory.path().join("templates")).unwrap(); + let store = Store::create(store(directory.path())).unwrap(); let (mut class, mut spec) = (class(), ManifestSpec::template()); set_geometries( &mut class, @@ -313,17 +344,24 @@ impl Fixture { (path, binding) } - /// Provisions the sandbox `name`, and leases the generation to it, a second after its - /// capture. - fn acquire(&self, name: &str) -> Acquired { + /// Provisions the sandbox `name`, and returns its directory, with its links resolved, as a + /// lease names it. + fn provision(&self, name: &str) -> PathBuf { let dir = self.path("sandboxes", name); let locked = SandboxStore::lock_for_provision(&dir, &files(), None).unwrap(); fs::write(dir.join(CONFIG), b"{}").unwrap(); drop(locked); + filesystem::canonicalize(&dir).unwrap() + } + + /// Provisions the sandbox `name`, and leases the generation to it, a second after its + /// capture. + fn acquire(&self, name: &str) -> Acquired { + let sandbox = self.provision(name); self.store .acquire( self.generation.generation(), - filesystem::canonicalize(&dir).unwrap(), + sandbox, &files(), &after(&self.generation, 1), None, @@ -717,6 +755,188 @@ fn links_and_directories_under_the_names_of_the_sandboxs_store_are_refused() { assert!(fs::symlink_metadata(plan.scratch_path()).unwrap().is_file()); } +#[test] +fn a_scratch_image_that_is_a_file_of_a_generation_s_snapshot_is_refused() { + let fixture = Fixture::new(); + let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + // The RAM of the leased generation's snapshot, and that of another generation's, which the + // store's index need not list, of the template's scratch length: the clone's guest would write + // a snapshot that every clone of its generation restores in place. + let snapshot = fixture.store.snapshot_dir(&fixture.generation).unwrap(); + let leased = snapshot.join("memory.bin"); + stage(&leased, SCRATCH_LEN); + let next = fixture.generation.generation().get().get() + 1; + let other = crate::activation::StoreGeneration::new(std::num::NonZeroU64::new(next).unwrap()); + let other = fixture + .store + .payload_dir(other) + .join(snapshot.file_name().unwrap()); + fs::create_dir_all(&other).unwrap(); + let other = other.join("memory.bin"); + stage(&other, SCRATCH_LEN); + // Through a hard link among the scratch images, another name of the file, and by its own + // path, in the store's directory. + let cases = [ + (&leased, fixture.path("scratches", "leased.raw")), + (&other, fixture.path("scratches", "other.raw")), + (&other, other.clone()), + ]; + for (memory, scratch) in cases { + let linked = scratch != *memory; + if linked { + fs::hard_link(memory, &scratch).unwrap(); + } + let binding = ScratchBinding::bind(&scratch).unwrap(); + let request = scratch_request(&image, &image_binding, &scratch, &binding); + let expected = if linked { + format!( + "the scratch image {} has 2 names, but a clone's scratch image must have a single \ + name: another, a hard link, could make it a file of a snapshot, which every \ + clone of its generation restores in place", + scratch.display() + ) + } else { + format!( + "the scratch image {} is {}, in the directory of the template store {}, where its \ + captures write the snapshots that every clone of a generation restores in place: \ + a clone's scratch image must be outside it", + scratch.display(), + location_of(&File::open(&scratch).unwrap()) + .unwrap() + .display(), + fixture.store.dir().display() + ) + }; + assert_eq!( + failure(fixture.plan(&mut acquired, &request, NO_CONTENT)), + (ErrorKind::Unsupported, expected) + ); + assert_untouched(&acquired); + } +} + +#[test] +fn a_scratch_image_in_the_template_store_is_refused() { + let fixture = Fixture::new(); + let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + // A file in the store's directory, and one of a snapshot that a capture of the store writes, + // which its publication makes a generation's by renaming the capture's transaction, of the + // template's scratch length: by their own paths, and through links on the way to them, which + // the image's location resolves. + let dir = fixture.store.dir().to_owned(); + let loose = dir.join("scratch.raw"); + let transaction = dir.join(CAPTURES_NAME).join("slot").join(TRANSACTION_NAME); + let memory = transaction.join(SNAPSHOT_NAME).join("memory.bin"); + fs::create_dir_all(memory.parent().unwrap()).unwrap(); + for file in [&loose, &memory] { + stage(file, SCRATCH_LEN); + } + let through = |link: &Path| link.join(SNAPSHOT_NAME).join("memory.bin"); + let mut cases = vec![(&loose, loose.clone()), (&memory, memory.clone())]; + let link = fixture.path("links", "transaction"); + fs::create_dir_all(link.parent().unwrap()).unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink(&transaction, &link).unwrap(); + #[cfg(windows)] + junction(&link, &transaction); + cases.push((&memory, through(&link))); + // The transaction held open, as a descriptor keeps a directory whatever renames it. + #[cfg(target_os = "linux")] + let held = File::open(&transaction).unwrap(); + #[cfg(target_os = "linux")] + cases.push(( + &memory, + through( + &Path::new("/proc/self/fd").join(std::os::fd::AsRawFd::as_raw_fd(&held).to_string()), + ), + )); + for (file, scratch) in cases { + let binding = ScratchBinding::bind(&scratch).unwrap(); + let request = scratch_request(&image, &image_binding, &scratch, &binding); + let location = location_of(&File::open(file).unwrap()).unwrap(); + assert_eq!( + failure(fixture.plan(&mut acquired, &request, NO_CONTENT)), + ( + ErrorKind::Unsupported, + format!( + "the scratch image {} is {}, in the directory of the template store {}, \ + where its captures write the snapshots that every clone of a generation \ + restores in place: a clone's scratch image must be outside it", + scratch.display(), + location.display(), + dir.display() + ) + ) + ); + assert_untouched(&acquired); + } +} + +/// On Windows, a file of a volume without a drive letter or a mount point is located by the +/// volume's GUID, above which the store's directory is found all the same. +#[cfg(windows)] +#[test] +fn locations_that_name_their_volume_by_its_guid_are_compared_with_the_store() { + let fixture = Fixture::new(); + let dir = fixture.store.dir().to_owned(); + let inside = dir + .join(CAPTURES_NAME) + .join("slot") + .join(TRANSACTION_NAME) + .join("memory.bin"); + fs::create_dir_all(inside.parent().unwrap()).unwrap(); + stage(&inside, SCRATCH_LEN); + let (outside, _) = fixture.scratch("scratch.ext4", SCRATCH_LEN); + for (file, within) in [(&inside, true), (&outside, false)] { + let location = crate::host::seal::final_volume_path(&File::open(file).unwrap()).unwrap(); + assert!( + location.to_string_lossy().starts_with(r"\\?\Volume{"), + "{}", + location.display() + ); + assert_eq!( + is_within(&location, &dir).unwrap(), + within, + "{}", + location.display() + ); + } +} + +#[test] +fn a_scratch_image_with_another_name_is_refused() { + let fixture = Fixture::new(); + let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + // Another name of the image, outside the template store, as a file of the snapshot of another + // store would be, or one of a snapshot that a capture of this store writes, which neither the + // store's generations nor the image's location show. + let (scratch, _) = fixture.scratch("scratch.ext4", SCRATCH_LEN); + let other = fixture.path("stores", "memory.bin"); + fs::create_dir_all(other.parent().unwrap()).unwrap(); + fs::hard_link(&scratch, &other).unwrap(); + let binding = ScratchBinding::bind(&scratch).unwrap(); + let request = scratch_request(&image, &image_binding, &scratch, &binding); + assert_eq!( + failure(fixture.plan(&mut acquired, &request, NO_CONTENT)), + ( + ErrorKind::Unsupported, + format!( + "the scratch image {} has 2 names, but a clone's scratch image must have a single \ + name: another, a hard link, could make it a file of a snapshot, which every clone \ + of its generation restores in place", + scratch.display() + ) + ) + ); + assert_untouched(&acquired); + // Once its other name went, the image is the clone's alone. + fs::remove_file(&other).unwrap(); + drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); +} + #[test] fn names_that_the_file_system_takes_for_those_of_the_stores_files_are_the_stores() { let fixture = Fixture::new(); diff --git a/nvx-rs/src/host/template/clone/tests/launch.rs b/nvx-rs/src/host/template/clone/tests/launch.rs new file mode 100644 index 000000000..2c728a201 --- /dev/null +++ b/nvx-rs/src/host/template/clone/tests/launch.rs @@ -0,0 +1,1407 @@ +//! Tests of the launches of clones: OpenVMM's restore of the leased snapshot in place with the +//! clone's devices, activation record, and restore-ready endpoint, for which a +//! [`restore_vmm_role`] stands in, listening beside the snapshot on Linux as OpenVMM does; the +//! checks that keep a VMM from starting before the clone's binding is durable, or with a control +//! socket that OpenVMM or the host could not use; the removal of that socket; and the crashes of +//! a launch's caller, which a [`launch_role`] runs. + +use std::cell::RefCell; +use std::env; +use std::ffi::OsString; +use std::io::{Read, Write}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use super::*; +use crate::activation::StoreGeneration; +use crate::host::artifacts; +use crate::host::process::{Process, ProcessRef}; +use crate::host::store::{ + CLONE_ACTIVATION_NAME, CLONE_BINDING_NAME, CLONE_SCRATCH_NAME, JOURNAL_NAME, LaunchState, + LockedStore, OUTCOME_NAME, RESTORE_READY_NAME, Recovery, RuntimeIdentity, +}; +use crate::host::supervisor::{self, Reattached, StopOutcome, VmmCommand}; +use crate::host::template::clone::launch::{Hooks, Point}; +use crate::host::template::store::SNAPSHOT_ARTIFACTS; +use crate::host::template::test_support::artifact; + +/// The environment variable that names the template store of a [`launch_role`]. +const ROLE_STORE: &str = "NVX_CLONE_LAUNCH_TEST_STORE"; +/// The environment variable that names the store generation that a [`launch_role`] leases. +const ROLE_GENERATION: &str = "NVX_CLONE_LAUNCH_TEST_GENERATION"; +/// The environment variable that names the directory of the sandbox of a [`launch_role`]. +const ROLE_SANDBOX: &str = "NVX_CLONE_LAUNCH_TEST_SANDBOX"; +/// The environment variable that names the image of the clone of a [`launch_role`]. +const ROLE_IMAGE: &str = "NVX_CLONE_LAUNCH_TEST_IMAGE"; +/// The environment variable that names where a [`launch_role`] crashes: at the point `prepared` +/// or `begun` of its launch, or once its launch returned, `launched`. +const ROLE_CRASH: &str = "NVX_CLONE_LAUNCH_TEST_CRASH"; +/// The environment variable that names the working directory of the VMM of a [`launch_role`]. +const ROLE_VMM: &str = "NVX_CLONE_LAUNCH_TEST_VMM"; +/// The start of the line in which [`launch_role`] prints the reference of the VMM that it +/// spawned, as JSON. +const REFERENCE: &str = "nvx-clone-launch-test-vmm "; +/// The start of the line that [`launch_role`] prints as it crashes. +const CRASHED: &str = "nvx-clone-launch-test-crashed "; +/// File whose presence in its working directory marks a [`restore_vmm_role`]. Without it, the +/// role returns at once, as it does when `cargo test -- --ignored` runs it. +const VMM_MARKER: &str = "vmm-role"; +/// File whose appearance in its working directory ends a [`restore_vmm_role`]. +const EXIT_MARKER: &str = "exit"; +/// File in which [`fake_vmm`] writes the arguments that a launch gave its VMM, as JSON, for the +/// [`restore_vmm_role`] that runs instead. +const ARGUMENTS: &str = "arguments.json"; +/// File in which a [`restore_vmm_role`] writes, once it has tried to send its restore-ready +/// event, the capability that it read, in hexadecimal, and whether it sent the event. +const STARTED: &str = "started"; +/// The restore-ready event that OpenVMM sends. +const READY_EVENT: &[u8] = b"OPENVMM_RESTORE_READY_V1\n"; +/// The OpenVMM executable that the tests' launches name, which [`fake_vmm`] replaces. +const OPENVMM: &str = "openvmm"; +/// Longest wait for a VMM that is expected to start or exit, for a sandbox's lock, or for a +/// reattach to become determinate. +const TIMEOUT: Duration = Duration::from_secs(60); +/// Longest life of a [`restore_vmm_role`] that nothing ends, so that a failed test leaves no +/// process. +const VMM_LIFETIME: Duration = Duration::from_secs(180); + +/// Returns `bytes` in lowercase hexadecimal. +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Returns the arguments that run only the role `name` of this test executable, on libtest's main +/// thread, so that libtest prints the role's name before the role's own output. +fn role(name: &str) -> Vec { + [ + "--exact", + &format!("host::template::clone::tests::launch::{name}"), + "--ignored", + "--nocapture", + "--test-threads=1", + ] + .map(OsString::from) + .to_vec() +} + +/// Gives the snapshot of the generation of `fixture` its RAM and saved state beside its manifest, +/// as OpenVMM publishes them, and returns the snapshot's directory, and the content of each of +/// its files, by name. +fn complete_snapshot(fixture: &Fixture) -> (PathBuf, Vec<(&'static str, Vec)>) { + let snapshot = fixture.store.snapshot_dir(&fixture.generation).unwrap(); + let mut contents = Vec::new(); + for name in SNAPSHOT_ARTIFACTS { + let path = snapshot.join(name); + if !path.exists() { + fs::write(&path, format!("the {name} of the template")).unwrap(); + } + contents.push((name, fs::read(&path).unwrap())); + } + (snapshot, contents) +} + +/// Returns the seals of the files of the snapshot in `snapshot`, by name, which change with their +/// content, their length, their modification time, and on Linux their change time, which a +/// restore checks. +fn seals(snapshot: &Path) -> Vec<(&'static str, FileSeal)> { + SNAPSHOT_ARTIFACTS + .map(|name| { + let file = File::open(snapshot.join(name)).unwrap(); + (name, FileSeal::of(&file).unwrap()) + }) + .to_vec() +} + +/// Returns the runtime identity that captured the template of `fixture`, under which its clones +/// launch. +fn runtime_of(fixture: &Fixture) -> RuntimeIdentity { + fixture + .store + .read_key(&fixture.generation) + .unwrap() + .runtime() + .clone() +} + +/// Returns whether a socket, or another file, is at `path`. +fn exists(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok() +} + +/// Asserts that the payload of the generation of `fixture` holds no control socket of a clone's +/// VMM, and that the files of the generation's snapshot in `snapshot` have the contents +/// `contents` and the seals `sealed` still. +fn assert_untouched( + fixture: &Fixture, + snapshot: &Path, + contents: &[(&str, Vec)], + sealed: &[(&str, FileSeal)], +) { + let payload = fixture.store.payload_dir(fixture.generation.generation()); + for entry in fs::read_dir(payload).unwrap() { + let name = entry.unwrap().file_name(); + assert!(!name.to_string_lossy().ends_with(".sock"), "{name:?}"); + } + for (name, content) in contents { + assert_eq!(&fs::read(snapshot.join(name)).unwrap(), content, "{name}"); + } + assert_eq!(seals(snapshot), sealed); +} + +/// Asserts that no launch of the clone of `acquired` began or left a file: no journal, activation +/// record, restore-ready socket, or control socket of its VMM, and that the lease still reserves +/// its launch. +fn assert_unlaunched(fixture: &Fixture, acquired: &Acquired) { + let locked = &acquired.locked; + assert!(locked.journal().unwrap().is_none()); + for path in [ + locked.clone_activation_path(), + locked.restore_ready_path(), + fixture.store.clone_socket_path(&acquired.lease), + ] { + assert!(!exists(&path), "{}", path.display()); + } + assert_eq!(locked.reserved(), Some(acquired.lease.launch())); +} + +/// Asserts that the sandbox in `sandbox` holds none of the files that the launch of a clone +/// leaves for its release. +fn assert_released(sandbox: &Path) { + for name in [ + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, + ] { + assert!(!exists(&sandbox.join(name)), "{name}"); + } +} + +/// The seam of the command of a launch that must start no VMM. +fn never_spawned(_: VmmCommand<'_>) -> VmmCommand<'_> { + panic!("a launch that is refused made the command of a VMM") +} + +/// The command that a launch made for its VMM, before [`fake_vmm`] replaced it, and what the +/// sandbox held as the VMM was about to start. +#[derive(Debug)] +struct Made { + /// The VMM's executable. + program: PathBuf, + /// The VMM's arguments. + arguments: Vec, + /// The VMM's working directory. + working_dir: PathBuf, + /// How many descriptors or handles the VMM inherits. + inherited: usize, + /// Whether the sandbox held its clone binding. + bound: bool, + /// Whether the sandbox held the launch's journal. + begun: bool, +} + +/// Returns the seam of the command of a launch whose VMM is a [`restore_vmm_role`] in `vmm`: it +/// keeps the command that the launch made in `made`, writes its arguments in [`ARGUMENTS`] in +/// `vmm`, and runs the role there instead, with the descriptors or handles that the command +/// passes. +fn fake_vmm<'a>( + vmm: &'a Path, + made: &'a RefCell>, +) -> impl for<'h> Fn(VmmCommand<'h>) -> VmmCommand<'h> + 'a { + move |command| { + let arguments: Vec = command + .arguments + .iter() + .map(|argument| argument.to_str().unwrap().to_owned()) + .collect(); + fs::write(vmm.join(ARGUMENTS), serde_json::to_vec(&arguments).unwrap()).unwrap(); + made.replace(Some(Made { + program: command.program.clone(), + arguments, + working_dir: command.working_dir.clone(), + inherited: command.inherit.len(), + bound: command.working_dir.join(CLONE_BINDING_NAME).is_file(), + begun: command.working_dir.join(JOURNAL_NAME).is_file(), + })); + VmmCommand { + program: env::current_exe().unwrap(), + arguments: role("restore_vmm_role"), + working_dir: vmm.to_owned(), + breakaway_from_job: command.breakaway_from_job, + inherit: command.inherit, + } + } +} + +/// The working directory of a [`restore_vmm_role`], which it marks. Dropping it ends the VMM, and +/// waits for it to exit, so that a failed test leaves no process behind. +struct VmmDir { + /// The reference of the VMM, once the test knows it. + reference: Option, + /// The directory, which is removed after the VMM exits. + dir: tempfile::TempDir, +} + +impl VmmDir { + /// Creates the directory, and marks it. + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join(VMM_MARKER), b"").unwrap(); + Self { + reference: None, + dir, + } + } + + /// Returns the path of the directory. + fn path(&self) -> &Path { + self.dir.path() + } + + /// Waits until the VMM has tried to send its restore-ready event, and returns what it wrote + /// in [`STARTED`]. + fn started(&self) -> String { + let path = self.path().join(STARTED); + let deadline = Instant::now() + TIMEOUT; + loop { + match fs::read_to_string(&path) { + Ok(started) => return started, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + assert!(Instant::now() < deadline, "the VMM did not start"); + thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("cannot read {}: {error}", path.display()), + } + } + } + + /// Asks the VMM to exit. + fn end(&self) { + fs::write(self.path().join(EXIT_MARKER), b"").unwrap(); + } + + /// Asks the VMM, which runs, to exit, and waits until it has. It opens the VMM's process + /// before it asks, since a process that has exited no longer opens. + fn end_and_wait(&self) { + let reference = self.reference.as_ref().expect("the test knows the VMM"); + let process = Process::open(reference) + .unwrap() + .expect("the VMM runs until the test ends it"); + self.end(); + assert!(process.wait(Instant::now() + TIMEOUT).unwrap()); + } +} + +impl Drop for VmmDir { + fn drop(&mut self) { + let _ = fs::write(self.path().join(EXIT_MARKER), b""); + if let Some(reference) = &self.reference + && let Ok(Some(process)) = Process::open(reference) + && !process.wait(Instant::now() + TIMEOUT).unwrap_or(false) + { + let _ = process.kill(); + let _ = process.wait(Instant::now() + TIMEOUT); + } + } +} + +/// The VMM of a clone, in a process of its own, in a directory that [`VMM_MARKER`] marks: reads +/// its capability as OpenVMM does, finds the endpoints of its control console and of the +/// restore-ready event, and the snapshot that it restores, among the arguments that its launch +/// made, which [`ARGUMENTS`] holds; holds the snapshot as OpenVMM does, which on Linux listens at +/// the control endpoint beside the snapshot, and on Windows opens the snapshot's directory and +/// files sharing nothing but reading; sends the restore-ready event as OpenVMM does; writes +/// [`STARTED`]; then runs until a file named [`EXIT_MARKER`] appears, or for [`VMM_LIFETIME`] at +/// most, and removes its control socket as it exits, as OpenVMM does at a normal exit. +/// +/// Where OpenVMM would refuse its arguments, this VMM writes why in [`STARTED`] after its +/// capability and whether it sent its event. Where OpenVMM would abort a restore whose event it +/// cannot send, as when the launch's caller crashed, this VMM runs on, so that the test ends it as +/// it ends any VMM. +#[test] +#[ignore = "a clone's VMM that the launch tests run in a process of its own"] +fn restore_vmm_role() { + let directory = env::current_dir().unwrap(); + if !directory.join(VMM_MARKER).exists() { + return; + } + let mut capability = [0; CAPABILITY_LEN]; + std::io::stdin().read_exact(&mut capability).unwrap(); + let arguments: Vec = + serde_json::from_slice(&fs::read(directory.join(ARGUMENTS)).unwrap()).unwrap(); + let (held, problem) = match hold_snapshot(&arguments) { + Ok(held) => (Some(held), String::new()), + Err(problem) => (None, format!(" {problem}")), + }; + let ready = argument(&arguments, "--restore-ready-path").unwrap(); + let sent = send_ready(Path::new(&ready)).is_ok(); + let staged = directory.join(format!(".{STARTED}")); + fs::write(&staged, format!("{} {sent}{problem}", hex(&capability))).unwrap(); + fs::rename(&staged, directory.join(STARTED)).unwrap(); + let started = Instant::now(); + while !directory.join(EXIT_MARKER).exists() && started.elapsed() < VMM_LIFETIME { + thread::sleep(Duration::from_millis(10)); + } + if let Some(held) = held { + held.release(); + } +} + +/// Returns the value of the option `name` among `arguments`, if they name it. +fn argument(arguments: &[String], name: &str) -> Option { + arguments + .iter() + .position(|argument| argument == name) + .and_then(|at| arguments.get(at + 1).cloned()) +} + +/// What a [`restore_vmm_role`] holds of its restore while it runs. +struct Held { + /// The listener at the control endpoint, on Linux, and the path at which it listens. + #[cfg(unix)] + listener: (std::os::unix::net::UnixListener, PathBuf), + /// The snapshot's directory and files, open, on Windows. + #[cfg(windows)] + _files: Vec, +} + +impl Held { + /// Lets go of the restore, as OpenVMM does at a normal exit, which removes its control socket. + fn release(self) { + #[cfg(unix)] + { + let (listener, path) = self.listener; + drop(listener); + fs::remove_file(path).unwrap(); + } + } +} + +/// Holds the snapshot that `arguments` restore as OpenVMM does, or returns why OpenVMM would +/// refuse them. On Linux, OpenVMM requires the parent of its control endpoint to be a directory +/// of its user with mode 0700, not a link, whose path with its links resolved is that of the +/// snapshot's parent, and listens at the endpoint's path with its parent's links resolved. +#[cfg(unix)] +fn hold_snapshot(arguments: &[String]) -> std::result::Result { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let endpoint = argument(arguments, "--microvm-control-console") + .and_then(|console| console.strip_prefix("listen=").map(PathBuf::from)) + .ok_or("no control endpoint")?; + let snapshot = PathBuf::from(argument(arguments, "--restore-snapshot").ok_or("no snapshot")?); + let (Some(parent), Some(name), Some(snapshot_parent)) = + (endpoint.parent(), endpoint.file_name(), snapshot.parent()) + else { + return Err("the endpoint or the snapshot names no directory".to_owned()); + }; + let metadata = fs::symlink_metadata(parent).map_err(|error| error.to_string())?; + if !metadata.is_dir() + || metadata.uid() != crate::host::sys::effective_ids().0 + || metadata.permissions().mode() & 0o7777 != 0o700 + { + return Err(format!( + "the endpoint's directory {} is not one that OpenVMM takes", + parent.display() + )); + } + let canonical = fs::canonicalize(parent).map_err(|error| error.to_string())?; + if fs::canonicalize(snapshot_parent).map_err(|error| error.to_string())? != canonical { + return Err(format!( + "the endpoint {} is not beside the snapshot {}", + endpoint.display(), + snapshot.display() + )); + } + let path = canonical.join(name); + let listener = std::os::unix::net::UnixListener::bind(&path) + .map_err(|error| format!("cannot listen at {}: {error}", path.display()))?; + Ok(Held { + listener: (listener, path), + }) +} + +/// Holds the snapshot that `arguments` restore as OpenVMM does, or returns why OpenVMM would +/// refuse them: on Windows, OpenVMM opens the snapshot's directory and files for reading, sharing +/// nothing but reading, so that a restore of another clone, which shares reading, reads them too, +/// while no one changes them. +#[cfg(windows)] +fn hold_snapshot(arguments: &[String]) -> std::result::Result { + use std::os::windows::fs::OpenOptionsExt; + + /// `FILE_SHARE_READ`. + const SHARE_READ: u32 = 0x1; + /// `FILE_FLAG_BACKUP_SEMANTICS`, which opens a directory. + const BACKUP_SEMANTICS: u32 = 0x0200_0000; + let snapshot = PathBuf::from(argument(arguments, "--restore-snapshot").ok_or("no snapshot")?); + let mut files = Vec::new(); + let paths = SNAPSHOT_ARTIFACTS.map(|name| (snapshot.join(name), 0)); + for (path, flags) in std::iter::once((snapshot.clone(), BACKUP_SEMANTICS)).chain(paths) { + let file = fs::OpenOptions::new() + .read(true) + .share_mode(SHARE_READ) + .custom_flags(flags) + .open(&path) + .map_err(|error| format!("cannot open {}: {error}", path.display()))?; + files.push(file); + } + Ok(Held { _files: files }) +} + +/// Sends the restore-ready event to the endpoint at `path`, as OpenVMM does: through a Unix +/// socket, or a named pipe on Windows. +fn send_ready(path: &Path) -> io::Result<()> { + #[cfg(unix)] + let mut sink = std::os::unix::net::UnixStream::connect(path)?; + #[cfg(windows)] + let mut sink = fs::OpenOptions::new().write(true).open(path)?; + sink.write_all(READY_EVENT) +} + +#[test] +fn a_clone_restores_its_leased_snapshot_with_its_devices_and_activation_record() { + let fixture = Fixture::new(); + let (snapshot, contents) = complete_snapshot(&fixture); + let sealed = seals(&snapshot); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let request = tmpfs_request(&image, &binding); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let (key, activation) = (plan.key().clone(), plan.activation().clone()); + let capability = *plan.capability(); + let sandbox = acquired.locked.dir().to_owned(); + let lease = acquired.lease.clone(); + let mut vmm = VmmDir::new(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(&mut acquired, Path::new(OPENVMM), key.runtime(), hooks) + .unwrap(); + let reference = restoring.starting().process().reference().clone(); + vmm.reference = Some(reference.clone()); + // The VMM read the plan's capability, took its arguments, and sent its event to the + // restore-ready endpoint, which listened before the VMM started. + assert_eq!(vmm.started(), format!("{} true", hex(&capability))); + // The VMM started once the sandbox held the clone binding and the launch's journal, in the + // sandbox's directory, with nothing of the caller's besides its standard streams. + let made = made.take().unwrap(); + assert!(made.bound && made.begun, "{made:?}"); + assert_eq!( + made.program, + artifacts::absolute(Path::new(OPENVMM)).unwrap() + ); + assert_eq!(made.working_dir, sandbox); + assert_eq!(made.inherited, 0); + // It restores the generation's snapshot in place, with the clone's devices: its image, and + // the sparse file of its `scratch` block device. On Linux, its control console listens beside + // the snapshot, at the socket that the lease reserved, named after the launch. + let endpoint = restoring.starting().endpoint().to_owned(); + let ready = restoring.ready_endpoint().to_owned(); + let socket = fixture.store.clone_socket_path(&lease); + if cfg!(target_os = "linux") { + assert_eq!( + socket, + snapshot + .parent() + .unwrap() + .join(format!("{}.sock", lease.launch())) + ); + assert_eq!(endpoint, socket.to_str().unwrap()); + assert!(exists(&socket)); + assert_eq!(ready, sandbox.join(RESTORE_READY_NAME).to_str().unwrap()); + } else { + assert!( + endpoint.starts_with("//./pipe/openvmm-microvm-"), + "{endpoint}" + ); + assert!(!exists(&socket)); + assert!(ready.starts_with("//./pipe/nvx-restore-ready-"), "{ready}"); + } + assert_eq!(restoring.starting().journal().endpoint(), endpoint); + let distro = format!("distro:file:{},ro", shown(&image)); + let scratch = format!("scratch:file:{}", shown(&sandbox.join(CLONE_SCRATCH_NAME))); + let (profile, processors) = (key.cpu_profile(), key.processors().to_string()); + let listen = format!("listen={endpoint}"); + let report = sandbox.join(OUTCOME_NAME); + let expected = [ + "--machine", + "microvm", + "--microvm-sandbox-block", + &distro, + "--microvm-sandbox-block", + &scratch, + "--single-process", + "--hypervisor", + key.hypervisor().as_str(), + "--cpu-profile", + &profile, + "--processors", + &processors, + "--restore-snapshot", + snapshot.to_str().unwrap(), + "--restore-memory", + "512M", + "--restore-ready-path", + &ready, + "--virtio-console", + "none", + "--microvm-control-console", + &listen, + "--microvm-control-auth-stdin", + "--microvm-report", + report.to_str().unwrap(), + ]; + assert_eq!(made.arguments, expected); + // The sandbox holds the clone's activation record, in its canonical encoding, which the + // launch keeps. + let recorded = fs::read(sandbox.join(CLONE_ACTIVATION_NAME)).unwrap(); + assert_eq!(recorded, activation.encode()); + assert_eq!(ActivationRecord::decode(&recorded).unwrap(), activation); + assert_eq!(restoring.activation(), &activation); + assert_eq!(restoring.capability(), &capability); + assert_eq!(restoring.image().path().to_str().unwrap(), shown(&image)); + // The launch never shows its capability. + let shown_launch = format!("{restoring:?}"); + assert!(shown_launch.starts_with("Restoring {"), "{shown_launch}"); + assert!(!shown_launch.contains("capability"), "{shown_launch}"); + assert!( + !shown_launch.contains(&format!("{capability:?}")), + "{shown_launch}" + ); + // Abandoning the launch ends its VMM, which leaves its control socket, and the end of the + // launch removes it. The release then removes the clone's files, and nothing of the + // generation, whose snapshot the restore left as it was. + assert_eq!( + restoring.abandon().unwrap(), + Recovery::Terminated(reference) + ); + assert!(!exists(&socket)); + assert_eq!( + fixture.store.release(&mut acquired.locked).unwrap(), + Some(acquired.lease.clone()) + ); + assert_released(&sandbox); + assert_untouched(&fixture, &snapshot, &contents, &sealed); +} + +#[test] +fn clones_of_one_generation_restore_its_snapshot_in_place_at_once() { + let fixture = Fixture::new(); + let (snapshot, contents) = complete_snapshot(&fixture); + let sealed = seals(&snapshot); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let request = tmpfs_request(&image, &binding); + let mut clones = [fixture.acquire("first"), fixture.acquire("second")]; + let leases = clones.each_ref().map(|acquired| acquired.lease.clone()); + let mut vmms = [VmmDir::new(), VmmDir::new()]; + // Both clones plan and launch at once, each under its own sandbox's lock. + let launched: Vec<(Restoring<'_>, Made)> = thread::scope(|scope| { + let launches: Vec<_> = clones + .iter_mut() + .zip(&vmms) + .map(|(acquired, vmm)| { + let (fixture, request, dir) = (&fixture, &request, vmm.path().to_owned()); + scope.spawn(move || { + let plan = fixture.plan(acquired, request, NO_CONTENT).unwrap(); + let runtime = plan.key().runtime().clone(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + (restoring, made.take().unwrap()) + }) + }) + .collect(); + launches + .into_iter() + .map(|launch| launch.join().unwrap()) + .collect() + }); + for ((restoring, _), vmm) in launched.iter().zip(&mut vmms) { + vmm.reference = Some(restoring.starting().process().reference().clone()); + } + // Both VMMs run, restoring the one snapshot in place, each listening at its own endpoint. + for ((restoring, made), vmm) in launched.iter().zip(&vmms) { + let capability = hex(restoring.capability()); + assert_eq!(vmm.started(), format!("{capability} true")); + let restored = argument(&made.arguments, "--restore-snapshot").unwrap(); + assert_eq!(restored, snapshot.to_str().unwrap()); + } + let endpoints: Vec<&str> = launched + .iter() + .map(|(restoring, _)| restoring.starting().endpoint()) + .collect(); + assert_ne!(endpoints[0], endpoints[1]); + if cfg!(target_os = "linux") { + for (endpoint, lease) in endpoints.iter().zip(&leases) { + let socket = fixture.store.clone_socket_path(lease); + assert_eq!(*endpoint, socket.to_str().unwrap()); + assert!(exists(&socket)); + } + } + // Each VMM exits as OpenVMM does, removing its socket, and each lease is released. + for ((restoring, _), vmm) in launched.into_iter().zip(&vmms) { + vmm.end_and_wait(); + assert_eq!(restoring.abandon().unwrap(), Recovery::Exited); + } + for (acquired, lease) in clones.iter_mut().zip(leases) { + assert_eq!( + fixture.store.release(&mut acquired.locked).unwrap(), + Some(lease) + ); + } + assert_untouched(&fixture, &snapshot, &contents, &sealed); +} + +/// Launches the clone of `acquired`, whose plan `fixture` makes, with a [`restore_vmm_role`] in +/// `vmm`, and commits the launch once the VMM runs, as a clone's restore does. +fn committed( + fixture: &Fixture, + acquired: &mut Acquired, + image: &(PathBuf, DistroBinding), + vmm: &mut VmmDir, +) -> supervisor::Vmm { + let plan = fixture + .plan(acquired, &tmpfs_request(&image.0, &image.1), NO_CONTENT) + .unwrap(); + let runtime = plan.key().runtime().clone(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + vmm.reference = Some(restoring.starting().process().reference().clone()); + let capability = hex(restoring.capability()); + assert_eq!(vmm.started(), format!("{capability} true")); + restoring.into_starting().commit().unwrap() +} + +#[test] +fn the_control_socket_of_a_clone_goes_once_its_vm_has_ended_and_for_good_with_its_lease() { + let fixture = Fixture::new(); + let (snapshot, contents) = complete_snapshot(&fixture); + let sealed = seals(&snapshot); + let image = fixture.image("distro.erofs", IMAGE_LEN); + // A clone's VM whose VMM a termination ends, which leaves its control socket: the teardown + // removes it with the runtime record. + let mut acquired = fixture.acquire("terminated"); + let mut vmm = VmmDir::new(); + let running = committed(&fixture, &mut acquired, &image, &mut vmm); + let socket = fixture.store.clone_socket_path(&acquired.lease); + assert_eq!( + running.endpoint(), + acquired.locked.runtime().unwrap().unwrap().endpoint() + ); + assert_eq!(exists(&socket), cfg!(target_os = "linux")); + assert_eq!( + running.terminate(&mut acquired.locked, TIMEOUT).unwrap(), + StopOutcome::Forced + ); + assert!(acquired.locked.runtime().unwrap().is_none()); + assert!(!exists(&socket)); + // A socket that comes back, as an unsynced removal can after a crash of the host, goes with + // the release, which makes its absence durable before the lease ends. + if cfg!(target_os = "linux") { + fs::write(&socket, b"").unwrap(); + } + assert_eq!( + fixture.store.release(&mut acquired.locked).unwrap(), + Some(acquired.lease.clone()) + ); + assert!(!exists(&socket)); + // A clone whose VMM exits on its own removes its socket itself, and a stop that finds it + // gone ends the VM. + let mut acquired = fixture.acquire("stopped"); + let mut vmm = VmmDir::new(); + let running = committed(&fixture, &mut acquired, &image, &mut vmm); + let socket = fixture.store.clone_socket_path(&acquired.lease); + let deadline = Instant::now() + TIMEOUT; + let stopped = running + .stop(&mut acquired.locked, deadline, None, |_, _| { + vmm.end(); + Ok::<_, ()>(()) + }) + .unwrap(); + assert_eq!(stopped.outcome, StopOutcome::Graceful); + assert!(acquired.locked.runtime().unwrap().is_none()); + assert!(!exists(&socket)); + assert_eq!( + fixture.store.release(&mut acquired.locked).unwrap(), + Some(acquired.lease.clone()) + ); + assert_untouched(&fixture, &snapshot, &contents, &sealed); +} + +#[test] +fn a_clone_of_a_template_with_a_nic_restores_its_policy_without_its_address() { + let fixture = Fixture::with(|class, spec, physical| { + *class = network_class(); + *spec = ManifestSpec::with_network(); + let (distro, scratch) = ( + geometry(IMAGE_LEN, physical), + geometry(SCRATCH_LEN, physical), + ); + set_geometries(class, spec, distro, scratch); + }); + complete_snapshot(&fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let attached = network(NetworkAccess::Allow); + let request = with_network(tmpfs_request(&image, &binding), Some(attached.clone())); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let runtime = plan.key().runtime().clone(); + let mut vmm = VmmDir::new(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + vmm.reference = Some(restoring.starting().process().reference().clone()); + // The devices end with the arguments of the network's policy, without the guest's address, + // which OpenVMM takes from the snapshot, and refuses beside it. + let mut policy = attached.openvmm_arguments().unwrap(); + assert_eq!(policy[..2], ["--net", "10.0.0.2/24"]); + policy.drain(..2); + let arguments = made.take().unwrap().arguments; + let devices = arguments + .iter() + .position(|argument| argument == "--single-process") + .unwrap(); + assert_eq!(arguments[6..devices], policy); + assert!(!arguments.iter().any(|argument| argument == "--net")); +} + +#[test] +fn a_clone_on_a_scratch_image_passes_its_claim_to_its_vmm() { + let fixture = Fixture::new(); + complete_snapshot(&fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); + let mut acquired = fixture.acquire("sandbox"); + let request = scratch_request(&image, &binding, &scratch, &scratch_binding); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let runtime = plan.key().runtime().clone(); + let mut vmm = VmmDir::new(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + let reference = restoring.starting().process().reference().clone(); + vmm.reference = Some(reference.clone()); + vmm.started(); + // The VMM opens the image at its path, as the `scratch` block device, and inherits the one + // descriptor or handle that holds the claim, which no argument names. + let made = made.take().unwrap(); + assert_eq!(made.inherited, 1); + let device = format!("scratch:file:{}", shown(&scratch)); + assert!(made.arguments.contains(&device), "{:?}", made.arguments); + // On Windows, no handle that may write the image is open any more, so OpenVMM's open, which + // shares no writing, succeeds. + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + + /// `FILE_SHARE_READ | FILE_SHARE_DELETE`. + const SHARE_READ_DELETE: u32 = 0x1 | 0x4; + /// `FILE_FLAG_OPEN_REPARSE_POINT`. + const OPEN_REPARSE_POINT: u32 = 0x0020_0000; + let opened = fs::OpenOptions::new() + .read(true) + .write(true) + .share_mode(SHARE_READ_DELETE) + .custom_flags(OPEN_REPARSE_POINT) + .open(&scratch); + assert!(opened.is_ok(), "{opened:?}"); + } + // The VMM holds the claim: another check finds the image claimed. + let error = scratch_binding.check(&scratch, Instant::now()).unwrap_err(); + assert_eq!(error.code(), ErrorCode::ScratchClaimed, "{error}"); + // The claim is the VMM's alone: once the VMM has exited, the image serves another check, + // while the launch lasts. + vmm.end_and_wait(); + assert!(released(|| scratch_binding + .check(&scratch, Instant::now()) + .is_ok())); + assert_eq!(restoring.abandon().unwrap(), Recovery::Exited); +} + +#[test] +fn a_clone_launches_only_as_the_launch_that_its_durable_binding_reserves() { + let fixture = Fixture::new(); + complete_snapshot(&fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let request = tmpfs_request(&image, &binding); + let runtime = runtime_of(&fixture); + let openvmm = Path::new(OPENVMM); + let hooks = Hooks { + command: &never_spawned, + ..Hooks::LAUNCH + }; + // A sandbox that no longer records the clone binding starts no VMM of the clone, and keeps + // nothing of its launch. + let mut unbound = fixture.acquire("unbound"); + let plan = fixture.plan(&mut unbound, &request, NO_CONTENT).unwrap(); + fs::remove_file(unbound.locked.binding_path()).unwrap(); + assert_eq!( + failure(plan.launch_with(&mut unbound, openvmm, &runtime, hooks)), + ( + ErrorKind::Other, + format!( + "the sandbox in {} records no clone binding of the lease {}, which its \ + acquisition makes durable before any VMM of the clone starts", + unbound.locked.dir().display(), + unbound.lease.id() + ) + ) + ); + assert_unlaunched(&fixture, &unbound); + // Nor does an acquisition of another lease. + let mut first = fixture.acquire("first"); + let mut second = fixture.acquire("second"); + let plan = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); + assert_eq!( + failure(plan.launch_with(&mut second, openvmm, &runtime, hooks)), + ( + ErrorKind::Other, + format!( + "a clone is launched under the lock that acquired the lease of its plan, {}, \ + before the launch that the lease reserves begins, but the lock of the sandbox in \ + {} holds the lease {}, and reserves no launch {} of it", + first.lease.id(), + second.locked.dir().display(), + second.lease.id(), + first.lease.launch() + ) + ) + ); + assert_unlaunched(&fixture, &first); + assert_unlaunched(&fixture, &second); + // Nor a runtime other than the one that captured the template. + let other = RuntimeIdentity::new(artifact('d'), artifact('e'), artifact('f')); + assert_ne!(other, runtime); + let plan = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); + let (kind, message) = failure(plan.launch_with(&mut first, openvmm, &other, hooks)); + assert_eq!(kind, ErrorKind::Unsupported, "{message}"); + assert!( + message.starts_with(&format!( + "the template's generation {} restores under the runtime that captured it", + fixture.generation.generation() + )), + "{message}" + ); + assert_unlaunched(&fixture, &first); + // Nor a plan in RAM whose scratch device's file a later plan of the clone, which failed, + // removed: OpenVMM would find no file at the path of its scratch device. + let stale = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); + let failing = Access { + create: &|path: &Path| -> io::Result { + fs::remove_file(path)?; + Err(io::Error::other("the disk failed")) + }, + ..NO_CONTENT + }; + assert!(fixture.plan(&mut first, &request, failing).is_err()); + let sparse = first.locked.clone_scratch_path(); + assert!(!exists(&sparse)); + let (kind, message) = failure(stale.launch_with(&mut first, openvmm, &runtime, hooks)); + assert_eq!(kind, ErrorKind::Unavailable, "{message}"); + assert!( + message.starts_with(&format!( + "the scratch device of the clone {}, which the clone's plan created, cannot be opened", + sparse.display() + )), + "{message}" + ); + assert_unlaunched(&fixture, &first); + // On Linux, a sandbox whose restore-ready socket's path would not fit a Unix socket's + // address is refused. + #[cfg(target_os = "linux")] + { + let mut deep = fixture.acquire(&"d".repeat(100)); + let plan = fixture.plan(&mut deep, &request, NO_CONTENT).unwrap(); + assert_eq!( + failure(plan.launch_with(&mut deep, openvmm, &runtime, hooks)), + ( + ErrorKind::Unsupported, + too_long(&deep.locked.restore_ready_path()) + ) + ); + assert_unlaunched(&fixture, &deep); + } + // The acquisition launches once its plan is its own. + let plan = fixture.plan(&mut second, &request, NO_CONTENT).unwrap(); + let mut vmm = VmmDir::new(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(&mut second, openvmm, &runtime, hooks) + .unwrap(); + vmm.reference = Some(restoring.starting().process().reference().clone()); +} + +/// Returns the message of the refusal of a clone's launch whose socket at `path` would not fit a +/// Unix socket's address. +#[cfg(target_os = "linux")] +fn too_long(path: &Path) -> String { + format!( + "the path of the socket {} of a clone's launch must be shorter than 108 bytes, as a Unix \ + socket's address requires", + path.display() + ) +} + +#[cfg(target_os = "linux")] +#[test] +fn a_clone_whose_control_socket_openvmm_or_the_host_cannot_use_starts_nothing() { + use std::os::unix::fs::PermissionsExt; + + /// Returns a directory of 60 characters in `directory`, private to the current user. + fn deep(directory: &Path) -> PathBuf { + let deep = directory.join("d".repeat(60)); + assert_eq!(platform::create_owner_only_dir(&deep).unwrap(), None); + deep + } + + let hooks = Hooks { + command: &never_spawned, + ..Hooks::LAUNCH + }; + let refused = |fixture: &Fixture, sandbox: &str| { + complete_snapshot(fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire(sandbox); + let plan = fixture + .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) + .unwrap(); + let runtime = runtime_of(fixture); + let refusal = failure(plan.launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks)); + assert_unlaunched(fixture, &acquired); + (refusal, fixture.store.clone_socket_path(&acquired.lease)) + }; + // A store whose path leaves no room for the socket of a clone's VMM in a payload. + let fixture = Fixture::in_store(|directory| deep(directory).join("templates"), |_, _, _| {}); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert!(socket.as_os_str().len() >= 108); + assert_eq!((kind, message), (ErrorKind::Unsupported, too_long(&socket))); + // A store whose path is short, but long once its links are resolved, at which OpenVMM listens. + let fixture = Fixture::in_store( + |directory| { + let link = directory.join("l"); + std::os::unix::fs::symlink(deep(directory), &link).unwrap(); + link.join("templates") + }, + |_, _, _| {}, + ); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert!(socket.as_os_str().len() < 108, "{}", socket.display()); + let resolved = filesystem::canonicalize(socket.parent().unwrap()) + .unwrap() + .join(socket.file_name().unwrap()); + assert_eq!( + (kind, message), + (ErrorKind::Unsupported, too_long(&resolved)) + ); + // A store whose path holds a comma or an equals sign, which OpenVMM's option `listen=` + // takes for the end of the path. + for name in ["a,b", "a=b"] { + let fixture = Fixture::in_store( + |directory| { + let parent = directory.join(name); + assert_eq!(platform::create_owner_only_dir(&parent).unwrap(), None); + parent.join("templates") + }, + |_, _, _| {}, + ); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert_eq!( + (kind, message), + ( + ErrorKind::Unsupported, + format!( + "the path of the control socket {} of a clone's VMM must hold no ',' or '=', \ + which end the path in OpenVMM's option `listen=`", + socket.display() + ) + ) + ); + } + // A store whose path is not valid UTF-8, as OpenVMM takes the paths of its sockets. + let fixture = Fixture::in_store( + |directory| { + use std::os::unix::ffi::OsStrExt; + + let parent = directory.join(std::ffi::OsStr::from_bytes(b"a\xffb")); + assert_eq!(platform::create_owner_only_dir(&parent).unwrap(), None); + parent.join("templates") + }, + |_, _, _| {}, + ); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert!(socket.to_str().is_none(), "{}", socket.display()); + assert_eq!( + (kind, message), + ( + ErrorKind::Unsupported, + format!( + "the path of the socket {} of a clone's launch must be valid UTF-8, as OpenVMM \ + takes it", + socket.display() + ) + ) + ); + // A store whose path is valid UTF-8, but not once its links are resolved, as OpenVMM takes + // the path at which it listens. + let fixture = Fixture::in_store( + |directory| { + use std::os::unix::ffi::OsStrExt; + + let target = directory.join(std::ffi::OsStr::from_bytes(b"a\xffb")); + assert_eq!(platform::create_owner_only_dir(&target).unwrap(), None); + let link = directory.join("u"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + link.join("templates") + }, + |_, _, _| {}, + ); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert!(socket.to_str().is_some(), "{}", socket.display()); + let resolved = filesystem::canonicalize(socket.parent().unwrap()) + .unwrap() + .join(socket.file_name().unwrap()); + assert!(resolved.to_str().is_none(), "{}", resolved.display()); + assert_eq!( + (kind, message), + ( + ErrorKind::Unsupported, + format!( + "the path of the socket {} of a clone's launch must be valid UTF-8, as OpenVMM \ + takes it", + resolved.display() + ) + ) + ); + // A payload private to the current user, as a plan requires, but whose mode is not 0700, as + // OpenVMM requires of the directory of its control socket. + let fixture = Fixture::new(); + let payload = fixture.store.payload_dir(fixture.generation.generation()); + fs::set_permissions(&payload, fs::Permissions::from_mode(0o500)).unwrap(); + let ((kind, message), _) = refused(&fixture, "sandbox"); + fs::set_permissions(&payload, fs::Permissions::from_mode(0o700)).unwrap(); + assert_eq!( + (kind, message), + ( + ErrorKind::Io, + format!( + "the payload of the generation {} has mode 500, not the mode 700 that the \ + directory of a control socket must have", + payload.display() + ) + ) + ); +} + +/// A clone's launch in a process of its own, which [`run_launch_role`] runs: it leases the store +/// generation [`ROLE_GENERATION`] of the store in [`ROLE_STORE`] to the sandbox in +/// [`ROLE_SANDBOX`], plans its clone of the image in [`ROLE_IMAGE`] in RAM, and launches it with a +/// [`restore_vmm_role`] in [`ROLE_VMM`] as its VMM; and it exits without unwinding where +/// [`ROLE_CRASH`] says, as a crash would, once it has printed the reference of the VMM that it +/// spawned, if any. +#[test] +#[ignore = "a clone's launch that the launch tests run in a process of its own"] +fn launch_role() { + let (Some(dir), Ok(generation), Some(sandbox), Some(image), Ok(step), Some(vmm)) = ( + env::var_os(ROLE_STORE), + env::var(ROLE_GENERATION), + env::var_os(ROLE_SANDBOX).map(PathBuf::from), + env::var_os(ROLE_IMAGE).map(PathBuf::from), + env::var(ROLE_CRASH), + env::var_os(ROLE_VMM).map(PathBuf::from), + ) else { + return; + }; + let store = Store::open(dir).unwrap(); + let generation = StoreGeneration::new(generation.parse().unwrap()); + let indexed = store.index().unwrap().get(generation).unwrap().clone(); + let mut acquired = store + .acquire(generation, sandbox, &files(), &after(&indexed, 1), None) + .unwrap(); + let binding = DistroBinding::import(&layer(&image), PolicyVerdict::ACCEPTED).unwrap(); + let plan = ClonePlan::new_with( + &store, + &mut acquired, + &tmpfs_request(&image, &binding), + Instant::now() + CLAIM_RELEASE_WAIT, + NO_CONTENT, + ) + .unwrap(); + let runtime = plan.key().runtime().clone(); + let made = RefCell::new(None); + let command = fake_vmm(&vmm, &made); + let interrupt = |point: Point| -> Result<()> { + if format!("{point:?}").to_lowercase() == step { + crash(&step); + } + Ok(()) + }; + let hooks = Hooks { + command: &command, + interrupt: &interrupt, + }; + let restoring = plan + .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + println!( + "\n{REFERENCE}{}", + serde_json::to_string(restoring.starting().process().reference()).unwrap() + ); + if step == "launched" { + crash(&step); + } + panic!("the clone's launch did not crash at {step}"); +} + +/// Ends this process at once, once it has printed that it crashes at `step`, without unwinding +/// or running destructors, as a crash would. +fn crash(step: &str) -> ! { + println!("\n{CRASHED}{step}"); + std::io::stdout().flush().unwrap(); + std::process::exit(0) +} + +/// Runs [`launch_role`] for the generation of `fixture`, the sandbox in `sandbox`, the image +/// `image`, and a VMM in `vmm`, until it crashes at `step`, and returns the reference of the VMM +/// that it spawned, if it spawned one. +fn run_launch_role( + step: &str, + fixture: &Fixture, + sandbox: &Path, + image: &Path, + vmm: &Path, +) -> Option { + let output = Command::new(env::current_exe().unwrap()) + .args(role("launch_role")) + .env(ROLE_STORE, fixture.store.dir()) + .env(ROLE_GENERATION, fixture.generation.generation().to_string()) + .env(ROLE_SANDBOX, sandbox) + .env(ROLE_IMAGE, image) + .env(ROLE_CRASH, step) + .env(ROLE_VMM, vmm) + .stdin(Stdio::null()) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + let crashed = format!("{CRASHED}{step}"); + assert!( + output.status.success() && stdout.lines().any(|line| line == crashed), + "{}: {stdout}{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + stdout + .lines() + .find_map(|line| line.strip_prefix(REFERENCE)) + .map(|reference| serde_json::from_str(reference).unwrap()) +} + +/// A clone whose launch's caller, a [`launch_role`], crashed, in a fresh fixture: the VMM's +/// working directory, the sandbox, and the lease that the sandbox holds. +struct Crashed { + /// The working directory of the clone's VMM, which ends the VMM as it drops, before the rest. + vmm: VmmDir, + /// The template store, its generation, and the clone's image. + fixture: Fixture, + /// The sandbox's directory. + sandbox: PathBuf, + /// The lease that the sandbox holds. + lease: Lease, + /// The runtime identity under which the clone launched. + runtime: RuntimeIdentity, +} + +impl Crashed { + /// Leases a fresh fixture's generation to a new sandbox, whose clone a [`launch_role`] + /// launches until it crashes at `step`. + fn at(step: &str) -> Self { + let fixture = Fixture::new(); + complete_snapshot(&fixture); + let (image, _) = fixture.image("distro.erofs", IMAGE_LEN); + let sandbox = fixture.provision("sandbox"); + let mut vmm = VmmDir::new(); + vmm.reference = run_launch_role(step, &fixture, &sandbox, &image, vmm.path()); + let lease = Lease::of(&SandboxStore::open(&sandbox).unwrap()) + .unwrap() + .expect("the crash kept the clone's lease"); + let runtime = runtime_of(&fixture); + Self { + vmm, + fixture, + sandbox, + lease, + runtime, + } + } + + /// Takes the lock of the clone's sandbox, as the restarted caller does. + fn lock(&self) -> LockedStore { + SandboxStore::lock_existing(&self.sandbox, &files(), Some(Instant::now() + TIMEOUT)) + .unwrap() + } + + /// Checks that the sandbox holds the clone's activation record, which names the restore of + /// the leased generation, and, on Linux, the restore-ready socket. + fn kept_its_files(&self, locked: &LockedStore) { + let recorded = fs::read(locked.clone_activation_path()).unwrap(); + let record = ActivationRecord::decode(&recorded).unwrap(); + assert_eq!(record.restore().unwrap().template, self.lease.template()); + if cfg!(target_os = "linux") { + assert!(exists(&locked.restore_ready_path())); + } + } + + /// Returns the path of the control socket of the clone's VMM, beside the snapshot that it + /// restores. + fn socket(&self) -> PathBuf { + self.fixture.store.clone_socket_path(&self.lease) + } + + /// Reattaches to the VM of the sandbox of `locked` until the result is determinate, as a + /// caller retries an indeterminate reattach, and returns it. + fn reattach(&self, locked: &mut LockedStore) -> Reattached { + let deadline = Instant::now() + TIMEOUT; + loop { + match supervisor::reattach(locked, &self.runtime).unwrap() { + Reattached::Indeterminate(reason) if Instant::now() < deadline => { + eprintln!("reattach is indeterminate: {reason}"); + thread::sleep(Duration::from_millis(100)); + } + reattached => return reattached, + } + } + } + + /// Checks that the store refuses to release the lease of the sandbox of `locked`, whose + /// launch has not ended, and that the sandbox keeps it. + fn refuses_release(&self, locked: &mut LockedStore) { + let error = self.fixture.store.release(locked).unwrap_err(); + assert_eq!(error.code(), ErrorCode::SandboxRunning, "{error}"); + assert_eq!(Lease::of(locked).unwrap(), Some(self.lease.clone())); + } + + /// Checks that the store releases the lease of the sandbox of `locked`, which removes the + /// clone's files, and nothing of the generation, whose payload holds no socket of the clone. + fn releases(&self, locked: &mut LockedStore) { + assert_eq!( + self.fixture.store.release(locked).unwrap(), + Some(self.lease.clone()) + ); + assert_released(&self.sandbox); + assert!(!exists(&self.socket())); + let snapshot = self.fixture.store.snapshot_dir(&self.fixture.generation); + for name in SNAPSHOT_ARTIFACTS { + assert!(snapshot.as_ref().unwrap().join(name).is_file(), "{name}"); + } + } +} + +#[test] +fn a_launch_whose_caller_crashes_before_it_begins_leaves_files_that_the_release_removes() { + let crashed = Crashed::at("prepared"); + assert_eq!(crashed.vmm.reference, None); + let mut locked = crashed.lock(); + // The launch never began: the sandbox records no launch, and keeps the clone's files. + assert!(locked.journal().unwrap().is_none()); + crashed.kept_its_files(&locked); + match crashed.reattach(&mut locked) { + Reattached::Exited { + record: None, + recovery: Recovery::Clean, + } => {} + other => panic!("unexpected reattach: {other:?}"), + } + crashed.releases(&mut locked); +} + +#[test] +fn a_launch_whose_caller_crashes_once_it_began_ends_as_reattach_recovers_it() { + let crashed = Crashed::at("begun"); + assert_eq!(crashed.vmm.reference, None); + let mut locked = crashed.lock(); + // The journal names the launch that the lease reserved, which spawned nothing, and keeps the + // lease until a reattach ends it. + let journal = locked.journal().unwrap().unwrap(); + assert_eq!(journal.launch(), crashed.lease.launch()); + assert_eq!(journal.state(), LaunchState::Prepared); + crashed.kept_its_files(&locked); + crashed.refuses_release(&mut locked); + match crashed.reattach(&mut locked) { + Reattached::Exited { + record: None, + recovery: Recovery::Exited, + } => {} + other => panic!("unexpected reattach: {other:?}"), + } + crashed.releases(&mut locked); +} + +#[test] +fn a_launch_whose_caller_crashes_once_its_vmm_started_ends_the_vmm_as_reattach_recovers_it() { + let crashed = Crashed::at("launched"); + let reference = crashed.vmm.reference.clone().unwrap(); + let process = Process::open(&reference).unwrap().unwrap(); + // The VMM listens beside the snapshot on Linux, as it has once it tried to send its event. + crashed.vmm.started(); + assert_eq!(exists(&crashed.socket()), cfg!(target_os = "linux")); + let mut locked = crashed.lock(); + // The journal records the VMM, which never committed: reattach ends it through its verified + // reference, and the lease lasts until then. The VMM, which the termination ends, leaves its + // control socket, which the end of the launch removes. + let journal = locked.journal().unwrap().unwrap(); + assert_eq!(journal.launch(), crashed.lease.launch()); + assert_eq!(journal.state(), LaunchState::Spawned); + assert_eq!(journal.process(), Some(&reference)); + if cfg!(target_os = "linux") { + assert_eq!(journal.endpoint(), crashed.socket().to_str().unwrap()); + } + crashed.kept_its_files(&locked); + crashed.refuses_release(&mut locked); + match crashed.reattach(&mut locked) { + Reattached::Exited { + record: None, + recovery: Recovery::Terminated(terminated), + } => assert_eq!(terminated, reference), + other => panic!("unexpected reattach: {other:?}"), + } + assert!(process.wait(Instant::now() + TIMEOUT).unwrap()); + assert!(!exists(&crashed.socket())); + crashed.releases(&mut locked); +} diff --git a/nvx-rs/src/host/template/mod.rs b/nvx-rs/src/host/template/mod.rs index f0c8ae886..3feb60cd3 100644 --- a/nvx-rs/src/host/template/mod.rs +++ b/nvx-rs/src/host/template/mod.rs @@ -37,6 +37,9 @@ //! its memory target against the template, and refuses a request that the template cannot //! serve, such as an image of another length or block geometry than the template's placeholder, //! before any VMM starts. +//! - [`ClonePlan::launch`] launches the clone, as the launch that its lease reserves: OpenVMM +//! restores the generation's snapshot in place with the clone's devices, and the clone's +//! activation record waits in its sandbox, as a [`Restoring`] describes. //! //! # Canonical encoding //! @@ -68,7 +71,7 @@ mod store; mod test_support; mod wire; -pub use self::clone::{ClonePlan, CloneRequest, CloneUpper}; +pub use self::clone::{ClonePlan, CloneRequest, CloneUpper, Restoring}; pub use self::eligibility::{Downtime, DowntimeSource, Eligibility, HostTime, MAX_DOWNTIME_NS}; pub use self::key::{CaptureClass, ClassIdentity, Invariants, KeyIdentity, TemplateKey}; pub use self::manifest::{ diff --git a/nvx-rs/src/host/template/store.rs b/nvx-rs/src/host/template/store.rs index 8dd9cf4dd..05ff211e9 100644 --- a/nvx-rs/src/host/template/store.rs +++ b/nvx-rs/src/host/template/store.rs @@ -26,9 +26,12 @@ use crate::host::process::BootId; use crate::host::store::files::{open_lock, write_private}; use crate::host::store::lease::{ID_LEN, hex, read_private, read_record, store_error}; +#[cfg(all(test, any(target_os = "linux", windows)))] +pub(super) use self::capture::SNAPSHOT_ARTIFACTS; +pub(super) use self::capture::SOCKET_PATH_LIMIT; pub use self::capture::{CAPTURE_RECORD_NAME, CAPTURES_NAME, Capture, Created, TRANSACTION_NAME}; pub use self::leases::{Acquired, LEASES_NAME, Retirement, TOMBSTONES_NAME}; -pub use crate::host::store::lease::{Lease, LeaseId, STORE_FORMAT, StoreId}; +pub use crate::host::store::lease::{GENERATIONS_NAME, Lease, LeaseId, STORE_FORMAT, StoreId}; /// Name of the store's identity record. pub const STORE_RECORD_NAME: &str = "store.json"; @@ -36,8 +39,6 @@ pub const STORE_RECORD_NAME: &str = "store.json"; pub const INDEX_NAME: &str = "index.json"; /// Name of the lock of the store's index. pub const INDEX_LOCK_NAME: &str = "index.lock"; -/// Name of the directory of the generations' payloads. -pub const GENERATIONS_NAME: &str = "generations"; /// Name of the canonical encoding of a generation's key in its payload. pub const KEY_NAME: &str = "key.bin"; /// Name of a generation's snapshot in its payload. @@ -248,6 +249,8 @@ impl Index { /// capture.json the record of the capture that made it /// key.bin the canonical encoding of its compatibility key /// snapshot/ its snapshot, as OpenVMM published it +/// .sock on Linux, the control socket of the VMM of a clone that leases it, +/// named after the clone's launch, until the lease's release /// leases/ the records of the leases that the store granted /// .json the record of the lease L /// tombstones/ the tombstones of the generations that take no new leases @@ -279,7 +282,9 @@ impl Index { /// of its key and of its capture class, its storage generation, and what its manifest records of /// the capture itself, its creation time, its time contract, and the lengths of its artifacts. /// The payload of a generation never changes once the index lists it: the record of its capture, -/// the canonical encoding of its key, which [`Store::read_key`] reads, and its snapshot. The index +/// the canonical encoding of its key, which [`Store::read_key`] reads, and its snapshot, which its +/// clones restore in place. Only the control sockets of the VMMs of the clones that lease it come +/// and go beside them, on Linux, as [`Store::acquire`] describes. The index /// changes as generations are published and retired, and the records of leases and tombstones as /// clones lease generations, and as generations are quarantined and retired. /// @@ -422,6 +427,14 @@ impl Index { /// verifies through the VM's recorded process, also after a reattach. A caller that reads a /// generation's payload, as a restore does, holds a lease of it. /// +/// On Linux, a payload holds the control socket of the VMM of each clone that leases it, +/// `.sock`, named after the launch that the lease reserves, beside the snapshot that the +/// clone restores, and of no clone whose lease has ended: the release makes the socket's absence +/// durable before it ends the lease. So a payload holds entries other than its capture's while +/// its generation has leases, which nothing that reads an indexed payload may refuse, and a +/// retirement deletes the payload with any socket that a crash, or a removal of a sandbox that +/// never released its lease, left in it. +/// /// A tombstone, which [`Store::tombstone`] writes, quarantines a generation: it takes no new /// lease, and selection skips it, while the leases that it granted keep working. /// [`Store::retire`] tombstones a generation, and deletes it once its leases have drained: its diff --git a/nvx-rs/src/host/template/store/capture.rs b/nvx-rs/src/host/template/store/capture.rs index 204ecb438..11b6d95f1 100644 --- a/nvx-rs/src/host/template/store/capture.rs +++ b/nvx-rs/src/host/template/store/capture.rs @@ -52,7 +52,8 @@ const STATE_NAME: &str = "state.bin"; const MEMORY_NAME: &str = "memory.bin"; /// The artifacts of a platform-tier snapshot, in the order of their names: its manifest, its RAM, /// and its saved state. -const SNAPSHOT_ARTIFACTS: [&str; 3] = [MANIFEST_NAME, MEMORY_NAME, STATE_NAME]; +pub(in crate::host::template) const SNAPSHOT_ARTIFACTS: [&str; 3] = + [MANIFEST_NAME, MEMORY_NAME, STATE_NAME]; /// Prefix of the name of the file that backs the RAM of a capture's VM, which OpenVMM creates /// beside the snapshot's destination, and removes as the VM ends. @@ -67,7 +68,7 @@ const STAGING_PREFIX: &str = ".snapshot.staging-"; const SLOT_NAME_LEN: usize = 16; /// Most bytes of a Unix socket's path on Linux, `sun_path`, with its terminating NUL. -const SOCKET_PATH_LIMIT: usize = 108; +pub(in crate::host::template) const SOCKET_PATH_LIMIT: usize = 108; /// Longest record of a capture, in bytes. const MAX_CAPTURE_RECORD_LEN: u64 = 4 * 1024; diff --git a/nvx-rs/src/host/template/store/leases.rs b/nvx-rs/src/host/template/store/leases.rs index e0114c979..7700448de 100644 --- a/nvx-rs/src/host/template/store/leases.rs +++ b/nvx-rs/src/host/template/store/leases.rs @@ -24,10 +24,12 @@ use crate::host::filesystem; use crate::host::platform; use crate::host::store::files::{remove_if_present, sync_dir}; use crate::host::store::lease::{ - Lease, LeaseId, LeaseRecord, MAX_LEASE_RECORD_LEN, read_binding, read_lease, + Lease, LeaseId, LeaseRecord, MAX_LEASE_RECORD_LEN, clone_socket_name, read_binding, read_lease, }; use crate::host::store::regular::was_removed; -use crate::host::store::{CLONE_BINDING_NAME, CallerFiles, LaunchId, LockedStore, SandboxStore}; +use crate::host::store::{ + CLONE_BINDING_NAME, CallerFiles, LaunchId, LockedStore, Reservation, SandboxStore, +}; use crate::host::template::eligibility::HostTime; use crate::host::template::key::KeyIdentity; @@ -143,6 +145,16 @@ impl Store { /// one under which the caller launches the clone, once, as the launch that the lease reserves, /// [`Lease::launch`], and releases the store's lock. /// + /// The launch that the lease reserves restores the generation's snapshot in place, and its VMM + /// listens beside the snapshot, as OpenVMM requires of a restore on Linux: at the control + /// socket `.sock`, in the generation's payload, after the launch's identity, rather + /// than at the sandbox's own socket. So a payload holds the sockets of the clones of its + /// generation, each named after a launch, while they lease it, and none once its leases have + /// drained: the VMM removes its socket as it exits, a stop or a recovery of the sandbox + /// removes the one that an abrupt exit leaves, and a [release](Self::release) makes the + /// socket's absence durable before the lease ends. On Windows, the VMM listens at a named + /// pipe, as every launch does. + /// /// Fails, and leases nothing: /// /// - with [`ErrorKind::NotFound`] if the store does not index the generation; @@ -307,10 +319,29 @@ impl Store { return Err(error); } drop(lock); - locked.reserve(Some(lease.launch().clone())); + locked.reserve(Some(Reservation { + socket: self.clone_socket_path(&lease), + launch: lease.launch().clone(), + })); Ok(Acquired { lease, locked }) } + /// Returns the path of the Linux socket at which the VMM of the clone that holds `lease` + /// listens, its control endpoint: `.sock`, after the launch that the lease reserves, + /// in the payload of the leased generation, beside the snapshot that the VMM restores in + /// place, since OpenVMM requires a restore's control socket in the snapshot's parent. The + /// clone's launch refuses a path that a Unix socket, or OpenVMM's option `listen=`, + /// cannot take. + /// + /// A payload holds the sockets of the clones of its generation only while they lease it: the + /// VMM removes its socket as it exits, a stop or a recovery removes what an abrupt exit + /// leaves, and a [release](Self::release) makes the socket's absence durable before the lease + /// ends. A [retirement](Self::retire) removes the payload with whatever a crash left in it. + pub(crate) fn clone_socket_path(&self, lease: &Lease) -> PathBuf { + self.payload_dir(lease.generation()) + .join(clone_socket_name(lease.launch())) + } + /// Releases the lease that the sandbox of `locked` holds, and returns it, or returns `None` if /// the sandbox holds none. /// @@ -318,14 +349,19 @@ impl Store { /// [`ErrorCode::SandboxRunning`] while the sandbox records a VM or a launch, which may still /// run. The sandbox's store clears those records only once it has verified that the VM's /// process exited, through its recorded [`ProcessRef`](crate::host::process::ProcessRef), as - /// a stop or a [`reattach`](crate::host::supervisor::reattach) does. The release removes what - /// the clone kept of its launch: the file of the scratch device of a clone whose upper is in - /// RAM, [`CLONE_SCRATCH_NAME`](crate::host::store::CLONE_SCRATCH_NAME), which the clone's + /// a stop or a [`reattach`](crate::host::supervisor::reattach) does. The release first ends + /// the reservation of the launch that `locked` holds, if it acquired the lease, so that one + /// that fails part of the way, once it removed files that a launch of the clone's plan needs, + /// leaves no launch that the lock may begin. It then removes what the clone kept of its + /// launch: the file of the scratch device of a clone whose upper is in RAM, + /// [`CLONE_SCRATCH_NAME`](crate::host::store::CLONE_SCRATCH_NAME), which the clone's /// [plan](crate::host::template::ClonePlan) created, the clone's activation record, /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME), and the socket of /// its restore-ready event, [`RESTORE_READY_NAME`](crate::host::store::RESTORE_READY_NAME); - /// then the clone binding, and once their removal is durable, the lease's record in the store, - /// so that an interrupted release leaves a record that no binding names, which a + /// then, on Linux, the control socket of the clone's VMM, `.sock` in the leased + /// generation's payload, after the launch that the lease reserved, whose absence it makes + /// durable; then the clone binding, and once their removal is durable, the lease's record in + /// the store, so that an interrupted release leaves a record that no binding names, which a /// [retirement](Self::retire) removes, and never a binding without its record. It takes no /// lock of the store. /// @@ -366,19 +402,46 @@ impl Store { ))); } let recorded = self.check_records_dir(LEASES_NAME, LEASES_WHAT)?; + // A plan of the clone launches under the reservation that the lease's acquisition left in + // the lock, which the release ends before it removes anything, so that a release that + // fails once it removed the clone's files, such as the scratch device's file of an upper + // in RAM, leaves no launch that the lock may begin. + locked.reserve(None); // The clone's VM has exited, and the sandbox launches nothing else under the lease, so no - // VM holds the scratch device's file of a RAM upper. The binding's removal makes the - // removal of the clone's files durable too, since all are entries of the sandbox's - // directory. + // VM holds the scratch device's file of a RAM upper, or listens at the clone's control + // socket. The binding's removal makes the removal of the clone's files durable too, since + // all are entries of the sandbox's directory, but not that of the socket, an entry of the + // leased generation's payload, which must be durable before the lease ends, so that a + // generation whose leases have drained holds no clone's socket. locked.remove_clone_files()?; + self.remove_clone_socket(&lease, sync)?; unbind(locked.dir(), sync)?; - locked.reserve(None); if recorded { remove_lease_record(&self.lease_path(lease.id()))?; } Ok(Some(lease)) } + /// Removes the control socket of the VMM of the clone that holds `lease`, beside the snapshot + /// that the VMM restored, if it is there, and makes its absence durable with `sync`. The VMM + /// removes its socket as it exits, and a stop or a recovery removes the one that an abrupt + /// exit leaves, but neither makes the removal durable. A payload that is gone holds no socket. + fn remove_clone_socket(&self, lease: &Lease, sync: SyncDir<'_>) -> Result<()> { + if !cfg!(unix) { + return Ok(()); + } + let payload = self.payload_dir(lease.generation()); + remove_if_present(&payload.join(clone_socket_name(lease.launch())))?; + match fs::symlink_metadata(&payload) { + Ok(_) => sync(&payload), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(store_error(format!( + "cannot examine the payload of the generation {}", + payload.display() + ))(error)), + } + } + /// Returns the leases of the generation `generation` that the store's records list, in the /// order of their identities, including those that an interrupted release or removal of /// their sandboxes left, which a [retirement](Self::retire) removes. diff --git a/nvx-rs/src/host/template/store/leases/tests.rs b/nvx-rs/src/host/template/store/leases/tests.rs index 5018935ca..3e2b19e55 100644 --- a/nvx-rs/src/host/template/store/leases/tests.rs +++ b/nvx-rs/src/host/template/store/leases/tests.rs @@ -787,8 +787,10 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco .dir() .join(LEASES_NAME) .join(format!("{}.json", lease.id())); + let payload = store.payload_dir(generation); + let socket = store.clone_socket_path(&lease); // Each sync notes its directory, and whether any file that the clone kept of its launch, the - // binding, and the record existed then. + // control socket of its VMM included, the binding, and the record existed then. let kept = [ CLONE_SCRATCH_NAME, CLONE_ACTIVATION_NAME, @@ -798,7 +800,8 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco let sync = |dir: &Path| -> Result<()> { let clone = kept .iter() - .any(|name| fs::symlink_metadata(sandbox.join(name)).is_ok()); + .any(|name| fs::symlink_metadata(sandbox.join(name)).is_ok()) + || fs::symlink_metadata(&socket).is_ok(); let bound = sandbox.join(CLONE_BINDING_NAME).exists(); synced .borrow_mut() @@ -808,18 +811,68 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco for name in kept { fs::write(sandbox.join(name), b"").unwrap(); } + // A socket that a VMM on Linux left in the payload, which a file stands for. + if cfg!(unix) { + fs::write(&socket, b"").unwrap(); + } let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); assert_eq!(store.release_with(&mut locked, &sync).unwrap(), Some(lease)); drop(locked); - assert_eq!(*synced.borrow(), [(sandbox.clone(), false, false, true)]); + let mut expected = vec![(sandbox.clone(), false, false, true)]; + if cfg!(unix) { + expected.insert(0, (payload.clone(), false, true, true)); + } + assert_eq!(*synced.borrow(), expected); assert!(!record.exists()); + assert!(fs::symlink_metadata(&socket).is_err()); + // A release under the lock that acquired the lease ends the lock's reservation before it + // removes the clone's files, such as the scratch device's file of an upper in RAM, which a + // launch of the clone's plan needs, so that one that fails after their removal leaves no + // launch to begin. + let Acquired { + lease: again, + mut locked, + } = store + .acquire( + generation, + sandbox.as_path(), + &files(), + &after(&published, 1), + None, + ) + .unwrap(); + assert_eq!(locked.reserved(), Some(again.launch())); + let scratch = sandbox.join(CLONE_SCRATCH_NAME); + fs::write(&scratch, b"").unwrap(); + // A release that cannot make the socket's removal durable keeps the binding, and the lease. + if cfg!(unix) { + let error = store + .release_with(&mut locked, &|dir: &Path| { + if dir == payload { + Err(Error::io("the disk failed")) + } else { + Ok(()) + } + }) + .unwrap_err(); + assert_eq!(error.message(), "the disk failed"); + assert_eq!(binding(&sandbox), Some(again.clone())); + assert_eq!(locked.reserved(), None); + assert!(fs::symlink_metadata(&scratch).is_err()); + } // A release that cannot make the binding's removal durable keeps the record, which a // retirement removes once no binding names it. - let again = leased(&store, &published, &sandbox); - let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); let error = store - .release_with(&mut locked, &|_: &Path| Err(Error::io("the disk failed"))) + .release_with(&mut locked, &|dir: &Path| { + if dir == sandbox { + Err(Error::io("the disk failed")) + } else { + Ok(()) + } + }) .unwrap_err(); + assert_eq!(locked.reserved(), None); + assert!(fs::symlink_metadata(&scratch).is_err()); drop(locked); assert_eq!(error.message(), "the disk failed"); assert_eq!(binding(&sandbox), None); From b3dfa2cbb3bd1f7dda5a2afb44ecac0d25ec6dd4 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Tue, 27 Oct 2026 20:50:49 -0700 Subject: [PATCH 4/7] nvx-rs: refuse a clone's plan of another guest network than its template's OpenVMM restores a clone's NIC, with the guest's address, from the template's snapshot, and a clone's launch drops the --net of the adapter's plan, so a plan that gave the guest another address or prefix than the template's NIC planned a clone that silently ran on the template's. ClonePlan::new now refuses such a plan with ErrorKind::Unsupported: it compares the plan's network with the guest network of the template key's NIC, TemplateKey::guest_network, in the canonical form that both write. The plan's policy must be the NIC's too, which OpenVMM checks as it restores the NIC, refusing another one once the launch began, so a plan of another policy fails closed. The key holds only OpenVMM's identity of the policy, which nvx-rs does not compute, so the docs of CloneRequest::adapter, of the restore's device arguments, and the README say so. Which policy a template serves stays the network templates' concern, one template per policy, whose key holds its identity. Part of #844. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a424884c-018a-45f5-a7fe-c57548a873f0 --- nvx-rs/README.md | 16 ++++++++------- nvx-rs/src/host/template/clone.rs | 25 ++++++++++++++++++++---- nvx-rs/src/host/template/clone/launch.rs | 6 +++++- nvx-rs/src/host/template/clone/tests.rs | 20 +++++++++++++++++++ nvx-rs/src/host/template/key.rs | 18 +++++++++++++---- 5 files changed, 69 insertions(+), 16 deletions(-) diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 938ba8b97..0311c2294 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -635,13 +635,15 @@ and no launch. the launch that the lease reserves, from metadata alone: the leased generation, its key, which must speak the current contracts, the boot and control contracts of the runtime that captured it included, and its snapshot; a network exactly -when the template has a NIC; a memory target in whole blocks of -`MEMORY_BLOCK_SIZE` from the template's RAM to its capacity; the clone's image, -which the check of its `DistroBinding` opens; its upper, either the caller's -scratch image, which the check of its `ScratchBinding` claims, and which nothing -formats, copies, or reads, or for an upper in RAM a fresh sparse file, -`clone-scratch.raw` in the sandbox's directory, which the release removes; and a -fresh capability. +when the template has a NIC, on the guest network of that NIC, whose address +OpenVMM restores from the snapshot, and under the policy of that NIC, which +OpenVMM checks as it restores it, since the key names the policy only by +OpenVMM's identity of it; a memory target in whole blocks of `MEMORY_BLOCK_SIZE` +from the template's RAM to its capacity; the clone's image, which the check of +its `DistroBinding` opens; its upper, either the caller's scratch image, which +the check of its `ScratchBinding` claims, and which nothing formats, copies, or +reads, or for an upper in RAM a fresh sparse file, `clone-scratch.raw` in the +sandbox's directory, which the release removes; and a fresh capability. It builds the clone's activation record, `ClonePlan::activation`, which the clone's guest applies in place of the configuration that the template captured, diff --git a/nvx-rs/src/host/template/clone.rs b/nvx-rs/src/host/template/clone.rs index 07a20a326..8178cfa1a 100644 --- a/nvx-rs/src/host/template/clone.rs +++ b/nvx-rs/src/host/template/clone.rs @@ -59,7 +59,11 @@ pub struct CloneRequest { /// `distro`, an EROFS layer, over its upper: in RAM, a tmpfs, as /// [`SandboxUpper::RAM`](crate::host::sandbox::SandboxUpper::RAM) plans it, or on the /// caller's scratch image. Its network is the one of the template's NIC, which every restore - /// of the template attaches, or none for a template without a NIC. + /// of the template attaches, or none for a template without a NIC: on the NIC's guest + /// network, which [`ClonePlan::new`] checks, and under the NIC's policy, which OpenVMM checks + /// as it restores the NIC, since the template's key names the policy only by OpenVMM's + /// identity of it. OpenVMM refuses to restore a clone under another policy, which ends its + /// launch once it began. pub adapter: SandboxPlan, /// The binding of the image. pub binding: DistroBinding, @@ -297,9 +301,10 @@ impl ClonePlan { /// - with [`ErrorKind::Unsupported`] for a request that the template cannot serve: a template /// whose key names other contracts than this build speaks, the boot and control contracts /// of the runtime that captured it included; a plan with a network for a template without a - /// NIC, or without one for a template with a NIC; and a memory target that is not whole - /// blocks of [`MEMORY_BLOCK_SIZE`] and of the template's memory blocks, from the template's - /// RAM to its RAM capacity; + /// NIC, without one for a template with a NIC, or with another guest network than the + /// template's NIC, whose address OpenVMM restores from the snapshot; and a memory target + /// that is not whole blocks of [`MEMORY_BLOCK_SIZE`] and of the template's memory blocks, + /// from the template's RAM to its RAM capacity; /// - with [`ErrorKind::Io`] for a template whose command line names the DNS server of its /// guest's resolver in another form than OpenVMM writes: more than once, without a NIC, or /// other than as one IP address in its canonical spelling; @@ -635,6 +640,18 @@ fn check_template( ) })); } + // OpenVMM restores the guest's address on the NIC from the snapshot, and a clone's launch + // drops the plan's, so the plan must name the template's. + if let Some(network) = request.adapter.network().filter(|_| attached) + && let Some(template) = key.guest_network() + && network.guest_network() != template + { + return Err(Error::unsupported(format!( + "the adapter's plan of a clone gives its guest the network {}, but the template's \ + generation {name} restores its NIC on {template}, from its snapshot", + network.guest_network() + ))); + } let memory = request.memory; let block = key.memory_block_size(); if memory < key.memory() diff --git a/nvx-rs/src/host/template/clone/launch.rs b/nvx-rs/src/host/template/clone/launch.rs index f1ba7bd3c..bb6b9d399 100644 --- a/nvx-rs/src/host/template/clone/launch.rs +++ b/nvx-rs/src/host/template/clone/launch.rs @@ -418,7 +418,11 @@ fn check_sparse(path: &Path, scratch: BlockGeometry) -> Result<()> { /// Returns OpenVMM's arguments of the devices of a clone's restore, `devices`: those that /// [`SandboxPlan::openvmm_arguments`] renders, without `--net` and the guest's address, which -/// OpenVMM takes from the snapshot, and refuses beside it. +/// OpenVMM takes from the snapshot, and refuses beside it. The options of the network's policy +/// stay: OpenVMM compares the policy that they name with the one of the snapshot's NIC, and +/// refuses to restore under another, so the plan must keep its template NIC's policy, as +/// [`CloneRequest::adapter`](super::CloneRequest::adapter) says, and a plan of another one fails +/// closed, ending the launch. fn device_arguments(devices: &SandboxPlan) -> Result> { let mut arguments = devices.openvmm_arguments()?; if let Some(at) = arguments.iter().position(|argument| argument == "--net") { diff --git a/nvx-rs/src/host/template/clone/tests.rs b/nvx-rs/src/host/template/clone/tests.rs index fc686ce6c..063f7cc2b 100644 --- a/nvx-rs/src/host/template/clone/tests.rs +++ b/nvx-rs/src/host/template/clone/tests.rs @@ -1591,6 +1591,26 @@ fn networks_that_the_template_does_not_have_are_refused() { ) ); assert_untouched(&acquired); + // OpenVMM restores the guest's address on the NIC from the snapshot, so a plan of another + // address, or of another prefix, is refused. + for other in ["10.0.0.3/24", "10.0.0.2/16"] { + let moved = with_network( + request.clone(), + Some(SandboxNetwork::new(other, NetworkPolicy::new(NetworkAccess::Allow)).unwrap()), + ); + assert_eq!( + failure(fixture.plan(&mut acquired, &moved, NO_CONTENT)), + ( + ErrorKind::Unsupported, + format!( + "the adapter's plan of a clone gives its guest the network {other}, but the \ + template's generation {generation} restores its NIC on 10.0.0.2/24, from its \ + snapshot" + ) + ) + ); + assert_untouched(&acquired); + } let attached = with_network(request, Some(network(NetworkAccess::Allow))); drop(fixture.plan(&mut acquired, &attached, NO_CONTENT).unwrap()); } diff --git a/nvx-rs/src/host/template/key.rs b/nvx-rs/src/host/template/key.rs index 0c64ed6e1..419bb658f 100644 --- a/nvx-rs/src/host/template/key.rs +++ b/nvx-rs/src/host/template/key.rs @@ -366,6 +366,12 @@ fn cpu_profile_problem(profile: &str) -> Option { }) } +/// Returns the guest's network on `nic`, its IPv4 address and prefix length, in the canonical form +/// of a [`SandboxNetwork`]. +fn guest_network_of(nic: &Nic) -> String { + format!("{}/{}", nic.ipv4, nic.prefix_length) +} + /// The invariant inputs of a template's capture, known before it: what a template's adapter /// requests, and the class of the templates that the capture makes. /// @@ -664,10 +670,7 @@ impl TemplateKey { .filter(|_| attached) .map(|network| network.guest_network().to_owned()); // A network that `SandboxNetwork` accepts has the canonical form that this writes. - let found_network = machine - .nic - .as_ref() - .map(|nic| format!("{}/{}", nic.ipv4, nic.prefix_length)); + let found_network = machine.nic.as_ref().map(guest_network_of); if found_network != requested_network { let describe = |network: Option| network.unwrap_or_else(|| "none".to_owned()); return mismatch( @@ -763,6 +766,13 @@ impl TemplateKey { self.machine.nic.is_some() } + /// Returns the guest's network on the template's NIC, its IPv4 address and prefix length, in + /// the canonical form of a [`SandboxNetwork`], such as `10.0.0.2/24`, which OpenVMM restores + /// from the snapshot with the NIC, or `None` for a template without a NIC. + pub(crate) fn guest_network(&self) -> Option { + self.machine.nic.as_ref().map(guest_network_of) + } + /// Returns the SHA-256 digest of the canonical encoding of the network policy that OpenVMM /// enforces on the template's NIC, which OpenVMM computed and the manifest records, or `None` /// for a template without a NIC. From 7def871e1709610185dbc01a7a4ed4ff8aaddaa1 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 28 Oct 2026 07:06:51 -0700 Subject: [PATCH 5/7] nvx-rs: release a clone's lease through a handle on its sandbox's directory A release checked a sandbox, read its clone binding, and removed the clone's files and the binding by the sandbox's path. A rename of one of the path's directories, or a change of a link in it, as the release ran could make it remove the files and the binding of another sandbox, whose lock it does not hold, and end the lease that this one holds still. The release now opens one handle of the directory that the path names, and checks through it that the directory holds the lock's file, as a removal of the sandbox does. Through that handle, it checks that the sandbox records no VM and no launch, reads the binding, removes the clone's files and the binding, and makes their absence durable. A path that names another directory than the lock's is refused, and nothing is removed. It reads the records and the binding through the handle as their readers do at their paths: regular files that are not links, and on Windows a file whose deletion is pending is removed already. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a424884c-018a-45f5-a7fe-c57548a873f0 --- nvx-rs/src/host/platform/linux.rs | 19 ++ nvx-rs/src/host/platform/other.rs | 5 + nvx-rs/src/host/platform/windows.rs | 22 ++ nvx-rs/src/host/store/lease.rs | 72 +++++- nvx-rs/src/host/store/lifecycle/tests.rs | 4 +- nvx-rs/src/host/store/mod.rs | 60 +++-- nvx-rs/src/host/store/regular.rs | 49 +++-- nvx-rs/src/host/store/tests.rs | 7 +- nvx-rs/src/host/template/store/leases.rs | 67 ++++-- .../src/host/template/store/leases/tests.rs | 208 +++++++++++++++--- 10 files changed, 424 insertions(+), 89 deletions(-) diff --git a/nvx-rs/src/host/platform/linux.rs b/nvx-rs/src/host/platform/linux.rs index 39f0c80a2..da60d4de7 100644 --- a/nvx-rs/src/host/platform/linux.rs +++ b/nvx-rs/src/host/platform/linux.rs @@ -673,6 +673,25 @@ impl Directory { } } + /// Opens its entry `name` for reading, not following a link, and without waiting, as an open + /// of a FIFO without a writer would, or returns `None` if it has no such entry. The caller + /// checks what the entry is. + pub(crate) fn open_file(&self, name: &str) -> io::Result> { + use nix::fcntl::{OFlag, openat}; + use nix::sys::stat::Mode; + + match openat( + &self.0, + name, + OFlag::O_RDONLY | OFlag::O_NOFOLLOW | OFlag::O_NONBLOCK | OFlag::O_CLOEXEC, + Mode::empty(), + ) { + Ok(file) => Ok(Some(File::from(file))), + Err(Errno::ENOENT) => Ok(None), + Err(errno) => Err(errno.into()), + } + } + /// Removes its file `name`, if it has one. pub(crate) fn remove_file(&self, name: &str) -> io::Result<()> { use nix::unistd::{UnlinkatFlags, unlinkat}; diff --git a/nvx-rs/src/host/platform/other.rs b/nvx-rs/src/host/platform/other.rs index 537e71959..8923ad434 100644 --- a/nvx-rs/src/host/platform/other.rs +++ b/nvx-rs/src/host/platform/other.rs @@ -116,6 +116,11 @@ impl Directory { match *self {} } + /// Never called: no directory is open. + pub(crate) fn open_file(&self, _name: &str) -> io::Result> { + match *self {} + } + /// Never called: no directory is open. pub(crate) fn remove_file(&self, _name: &str) -> io::Result<()> { match *self {} diff --git a/nvx-rs/src/host/platform/windows.rs b/nvx-rs/src/host/platform/windows.rs index 5099d7533..1b14972b0 100644 --- a/nvx-rs/src/host/platform/windows.rs +++ b/nvx-rs/src/host/platform/windows.rs @@ -1446,6 +1446,28 @@ impl Directory { } } + /// Opens its entry `name` for reading, not following a link or other reparse point, or + /// returns `None` if it has no such entry, or one whose deletion is pending, which a file + /// system without POSIX deletion keeps under its name until its last handle closes, and which + /// is removed. The caller checks what the entry is. + pub(crate) fn open_file(&self, name: &str) -> io::Result> { + use windows_sys::Wdk::Storage::FileSystem::{ + FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, + }; + use windows_sys::Win32::Foundation::{STATUS_DELETE_PENDING, STATUS_OBJECT_NAME_NOT_FOUND}; + use windows_sys::Win32::Storage::FileSystem::FILE_GENERIC_READ; + + match self.open_entry( + OsStr::new(name), + FILE_GENERIC_READ, + FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, + ) { + Ok(entry) => Ok(Some(fs::File::from(entry))), + Err(STATUS_OBJECT_NAME_NOT_FOUND | STATUS_DELETE_PENDING) => Ok(None), + Err(status) => Err(status_error(status)), + } + } + /// Removes its file `name`, if it has one. A file whose deletion is pending already, which a /// file system without POSIX deletion keeps under its name until its last handle closes, /// counts as removed. diff --git a/nvx-rs/src/host/store/lease.rs b/nvx-rs/src/host/store/lease.rs index 73096bb21..73aa01448 100644 --- a/nvx-rs/src/host/store/lease.rs +++ b/nvx-rs/src/host/store/lease.rs @@ -8,6 +8,7 @@ use std::ffi::OsStr; use std::fmt; +use std::fs::File; use std::io; use std::num::NonZeroU64; use std::path::{Path, PathBuf}; @@ -16,7 +17,7 @@ use std::str::FromStr; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use super::regular::{open_regular, read_at_most, was_removed}; +use super::regular::{open_regular, open_regular_in, read_at_most, was_removed}; use super::{CLONE_BINDING_NAME, LaunchId, SandboxStore}; use crate::activation::{Identity, StoreGeneration, TemplateReference}; use crate::error::{Error, ErrorCode, ErrorKind, Result}; @@ -277,10 +278,35 @@ pub(crate) fn read_binding(dir: &Path) -> Result> { read_lease(&dir.join(CLONE_BINDING_NAME), BINDING_DESCRIPTION) } +/// Reads the clone binding of the sandbox whose directory `directory` holds open, in `dir`, as +/// [`read_binding`] reads it at its path, or returns `None` if the directory holds none, or, on +/// Windows, one whose deletion is pending, which is removed: through the handle, so that it reads +/// the binding of the directory that the handle holds, whatever `dir` names meanwhile. +pub(crate) fn read_binding_in( + directory: &platform::Directory, + dir: &Path, +) -> Result> { + read_lease_with(&dir.join(CLONE_BINDING_NAME), BINDING_DESCRIPTION, || { + open_regular_in(directory, CLONE_BINDING_NAME) + }) +} + /// Reads the record of a lease at `path`, its `description`, or returns `None` if it does not /// exist. pub(crate) fn read_lease(path: &Path, description: &str) -> Result> { - let Some(record) = read_record::(path, description, MAX_LEASE_RECORD_LEN)? else { + read_lease_with(path, description, || open_existing(path)) +} + +/// Reads the record of a lease, its `description` at `path`, from the file that `open` opens, or +/// returns `None` if `open` finds none. +fn read_lease_with( + path: &Path, + description: &str, + open: impl FnOnce() -> io::Result>, +) -> Result> { + let Some(record) = + read_record_with::(path, description, MAX_LEASE_RECORD_LEN, open)? + else { return Ok(None); }; record.into_lease().map(Some).map_err(|problem| { @@ -387,13 +413,32 @@ pub(crate) fn store_error(context: String) -> impl FnOnce(io::Error) -> Error { /// Windows, a file's own access control list decides who may open it, whatever the store's /// directory allows. pub(crate) fn read_private(path: &Path, description: &str, limit: u64) -> Result>> { - let failed = || store_error(format!("cannot read the {description} {}", path.display())); - let file = match open_regular(path) { - Ok(file) => file, + read_private_with(path, description, limit, || open_existing(path)) +} + +/// Opens the regular file at `path`, as [`open_regular`] does, or returns `None` if it does not +/// exist, or was removed as this opened it, as [`was_removed`] tells. +fn open_existing(path: &Path) -> io::Result> { + match open_regular(path) { + Ok(file) => Ok(Some(file)), Err(error) if error.kind() == io::ErrorKind::NotFound || was_removed(path, &error) => { - return Ok(None); + Ok(None) } - Err(error) => return Err(failed()(error)), + Err(error) => Err(error), + } +} + +/// Reads the private regular file that `open` opens, the store's `description` at `path`, of at +/// most `limit` bytes, as [`read_private`] reads it, or returns `None` if `open` finds none. +fn read_private_with( + path: &Path, + description: &str, + limit: u64, + open: impl FnOnce() -> io::Result>, +) -> Result>> { + let failed = || store_error(format!("cannot read the {description} {}", path.display())); + let Some(file) = open().map_err(failed())? else { + return Ok(None); }; if let Some(problem) = platform::private_file_problem(&file).map_err(failed())? { return Err(Error::io(format!( @@ -413,7 +458,18 @@ pub(crate) fn read_record( description: &str, limit: u64, ) -> Result> { - let Some(text) = read_private(path, description, limit)? else { + read_record_with(path, description, limit, || open_existing(path)) +} + +/// Reads the store's record that `open` opens, its `description` at `path`, as [`read_record`] +/// reads it, or returns `None` if `open` finds none. +fn read_record_with( + path: &Path, + description: &str, + limit: u64, + open: impl FnOnce() -> io::Result>, +) -> Result> { + let Some(text) = read_private_with(path, description, limit, open)? else { return Ok(None); }; let malformed = || format!("the {description} {} is malformed", path.display()); diff --git a/nvx-rs/src/host/store/lifecycle/tests.rs b/nvx-rs/src/host/store/lifecycle/tests.rs index 8ca75742b..f78bd7457 100644 --- a/nvx-rs/src/host/store/lifecycle/tests.rs +++ b/nvx-rs/src/host/store/lifecycle/tests.rs @@ -1207,7 +1207,9 @@ mod windows { .map(|name| delete_pending(&dir.join(name))) .collect(); let locked = SandboxStore::lock_existing(&dir, &files(), None).unwrap(); - locked.remove_clone_files().unwrap(); + locked + .remove_clone_files(&locked.open_dir().unwrap()) + .unwrap(); drop(locked); drop(pending); let names = entries_of(&dir); diff --git a/nvx-rs/src/host/store/mod.rs b/nvx-rs/src/host/store/mod.rs index 2ae8d97cb..faecc57ab 100644 --- a/nvx-rs/src/host/store/mod.rs +++ b/nvx-rs/src/host/store/mod.rs @@ -777,27 +777,49 @@ impl LockedStore { Ok(owned) } + /// Opens the sandbox's directory through its path, as one handle through which a release of + /// the lease of its clone names the directory's entries, and checks through that handle that + /// the directory holds this lock's file, as a [removal](self#removal) checks it: no change of + /// the path meanwhile, of a link in it or of one of its directories, then makes the release + /// reach the entries of another sandbox, whose lock it does not hold. + /// + /// Fails with [`ErrorKind::Other`] if the directory that the path names does not hold this + /// lock's file. + pub(crate) fn open_dir(&self) -> Result { + let dir = self.dir(); + let directory = platform::Directory::open(dir).map_err(io_error(format!( + "cannot open the directory {}", + dir.display() + )))?; + let lock = self.path(LOCK_NAME); + let examine = || io_error(format!("cannot examine {}", lock.display())); + let held = platform::opened_file_identity(&self.lock).map_err(examine())?; + if directory.entry_identity(LOCK_NAME).map_err(examine())? != Some(held) { + return Err(Error::other(format!( + "the directory that {} names does not hold the lock of its sandbox: the path names \ + another directory than when the lock was taken", + dir.display() + ))); + } + Ok(directory) + } + /// Removes what the clone of the sandbox kept of its launch in the sandbox's directory, whose - /// VM has exited: the file of its scratch device, [`CLONE_SCRATCH_NAME`], its activation - /// record, [`CLONE_ACTIVATION_NAME`], and the socket of its restore-ready event, - /// [`RESTORE_READY_NAME`]. A release, as a [removal](self#removal) does, removes them before - /// the clone binding, whose removal, once durable, makes theirs durable too, since all are - /// entries of the sandbox's directory. A file whose deletion is pending, as a Windows file - /// system without POSIX deletion leaves a removed file that another handle holds, is removed - /// already, as [`was_removed`](regular::was_removed) tells, so that a release that failed once - /// it removed the file can be retried. - pub(crate) fn remove_clone_files(&self) -> Result<()> { + /// VM has exited, through `directory`, the handle of the directory that + /// [`open_dir`](Self::open_dir) returned: the file of its scratch device, + /// [`CLONE_SCRATCH_NAME`], its activation record, [`CLONE_ACTIVATION_NAME`], and the socket of + /// its restore-ready event, [`RESTORE_READY_NAME`]. A release, as a [removal](self#removal) + /// does, removes them before the clone binding, whose removal, once durable, makes theirs + /// durable too, since all are entries of the sandbox's directory. A file whose deletion is + /// pending, as a Windows file system without POSIX deletion leaves a removed file that another + /// handle holds, is removed already, so that a release that failed once it removed the file can + /// be retried. + pub(crate) fn remove_clone_files(&self, directory: &platform::Directory) -> Result<()> { for name in CLONE_FILE_NAMES { - let path = self.path(name); - match std::fs::remove_file(&path) { - Err(error) - if error.kind() != io::ErrorKind::NotFound - && !regular::was_removed(&path, &error) => - { - return Err(io_error(format!("cannot remove {}", path.display()))(error)); - } - _ => {} - } + directory.remove_file(name).map_err(io_error(format!( + "cannot remove {}", + self.path(name).display() + )))?; } Ok(()) } diff --git a/nvx-rs/src/host/store/regular.rs b/nvx-rs/src/host/store/regular.rs index 771c18ab0..e85f2845b 100644 --- a/nvx-rs/src/host/store/regular.rs +++ b/nvx-rs/src/host/store/regular.rs @@ -50,19 +50,42 @@ pub(crate) fn open_regular(path: &Path) -> io::Result { const OPEN_REPARSE_POINT: u32 = 0x0020_0000; options.custom_flags(OPEN_REPARSE_POINT); } - let file = match options.open(path) { - Ok(file) => file, - // `O_NOFOLLOW` refuses a link, and an open that does not wait refuses a socket. - #[cfg(unix)] - Err(error) - if [nix::errno::Errno::ELOOP, nix::errno::Errno::ENXIO] - .into_iter() - .any(|errno| error.raw_os_error() == Some(errno as i32)) => - { - return Err(not_regular()); - } - Err(error) => return Err(error), - }; + let file = options.open(path).map_err(refused_open)?; + regular(file) +} + +/// Opens the file `name` of `directory` for reading, as [`open_regular`] opens the file at a +/// path, if it is a regular file, or returns `None` if the directory has no such entry, or, on +/// Windows, one whose deletion is pending, which is removed. The open names the entry through the +/// handle, so that it reaches the entry of the directory that the handle holds, whatever the +/// directory's path names meanwhile. +pub(crate) fn open_regular_in( + directory: &platform::Directory, + name: &str, +) -> io::Result> { + match directory.open_file(name) { + Ok(Some(file)) => regular(file).map(Some), + Ok(None) => Ok(None), + Err(error) => Err(refused_open(error)), + } +} + +/// Returns `error`, with which an open failed, as the error of a file that is not a regular file if +/// the open refused one: on Unix, a link, which `O_NOFOLLOW` refuses, or a socket, which an open +/// that does not wait refuses. +fn refused_open(error: io::Error) -> io::Error { + #[cfg(unix)] + if [nix::errno::Errno::ELOOP, nix::errno::Errno::ENXIO] + .into_iter() + .any(|errno| error.raw_os_error() == Some(errno as i32)) + { + return not_regular(); + } + error +} + +/// Returns `file`, which an open that follows no link opened, if it is a regular file. +fn regular(file: File) -> io::Result { let metadata = file.metadata()?; if !metadata.file_type().is_file() || filesystem::is_reparse_point(&metadata) { return Err(not_regular()); diff --git a/nvx-rs/src/host/store/tests.rs b/nvx-rs/src/host/store/tests.rs index abba2a806..f9d4157ca 100644 --- a/nvx-rs/src/host/store/tests.rs +++ b/nvx-rs/src/host/store/tests.rs @@ -401,6 +401,8 @@ fn a_lock_file_that_is_a_link_fails_at_once() { assert_eq!(fs::read(&target).unwrap(), CAPABILITY); } +// Other hosts open no handle of a sandbox's directory, through which a clone's files go. +#[cfg(any(target_os = "linux", windows))] #[test] fn the_files_that_a_clone_kept_of_its_launch_go_and_nothing_else() { let (_directory, store) = store(); @@ -414,7 +416,8 @@ fn the_files_that_a_clone_kept_of_its_launch_go_and_nothing_else() { ] { fs::write(path, b"left").unwrap(); } - locked.remove_clone_files().unwrap(); + let directory = locked.open_dir().unwrap(); + locked.remove_clone_files(&directory).unwrap(); let mut left: Vec = fs::read_dir(store.dir()) .unwrap() .map(|entry| entry.unwrap().file_name().into_string().unwrap()) @@ -425,7 +428,7 @@ fn the_files_that_a_clone_kept_of_its_launch_go_and_nothing_else() { expected.sort_unstable(); assert_eq!(left, expected); // Nothing to remove is no failure. - locked.remove_clone_files().unwrap(); + locked.remove_clone_files(&directory).unwrap(); } #[cfg(unix)] diff --git a/nvx-rs/src/host/template/store/leases.rs b/nvx-rs/src/host/template/store/leases.rs index 7700448de..0fe9692d8 100644 --- a/nvx-rs/src/host/template/store/leases.rs +++ b/nvx-rs/src/host/template/store/leases.rs @@ -24,11 +24,13 @@ use crate::host::filesystem; use crate::host::platform; use crate::host::store::files::{remove_if_present, sync_dir}; use crate::host::store::lease::{ - Lease, LeaseId, LeaseRecord, MAX_LEASE_RECORD_LEN, clone_socket_name, read_binding, read_lease, + Lease, LeaseId, LeaseRecord, MAX_LEASE_RECORD_LEN, clone_socket_name, read_binding, + read_binding_in, read_lease, }; -use crate::host::store::regular::was_removed; +use crate::host::store::regular::{open_regular_in, was_removed}; use crate::host::store::{ - CLONE_BINDING_NAME, CallerFiles, LaunchId, LockedStore, Reservation, SandboxStore, + CLONE_BINDING_NAME, CallerFiles, JOURNAL_NAME, LaunchId, LockedStore, RUNTIME_NAME, + Reservation, SandboxStore, }; use crate::host::template::eligibility::HostTime; use crate::host::template::key::KeyIdentity; @@ -366,23 +368,44 @@ impl Store { /// lock of the store. /// /// The caller takes the sandbox's lock through the lease's path, as [`Lease::sandbox`] - /// returns it, and as [`acquire`](Self::acquire) takes it, which has no links: a link in the - /// path of the lock could change as the release runs, and make it remove another sandbox's - /// binding, whose lock it does not hold. + /// returns it, and as [`acquire`](Self::acquire) takes it, with its links resolved. The + /// release names the entries of the sandbox's directory through one handle of the directory + /// that the path names, once it checked through that handle that the directory holds the + /// lock's file: through it, it checks that the sandbox records no VM and no launch, reads the + /// binding, removes the clone's files and the binding, and makes their absence durable. No + /// change of the path as the release runs, of a link in it or of one of its directories, then + /// makes it check or remove the files of another sandbox, whose lock it does not hold. /// - /// Fails with [`ErrorKind::Other`] if the sandbox holds a lease of another store, or if the - /// lock does not name the sandbox by its lease's path, and then removes nothing. + /// Fails with [`ErrorKind::Other`] if the sandbox holds a lease of another store, if the lock + /// does not name the sandbox by its lease's path, or if the directory that the path names does + /// not hold the lock's file, and then removes nothing. pub fn release(&self, locked: &mut LockedStore) -> Result> { - self.release_with(locked, &sync_dir) + self.release_with(locked, &|| {}, &sync_dir, &platform::Directory::sync) } - /// Releases a lease as [`release`](Self::release) does, making the removal of its binding - /// durable with `sync`. - fn release_with(&self, locked: &mut LockedStore, sync: SyncDir<'_>) -> Result> { - if locked.journal()?.is_some() || locked.runtime()?.is_some() { - return Err(may_run(locked.dir())); + /// Releases a lease as [`release`](Self::release) does, running `checked` once its checks + /// pass, before it removes anything, and making the removal of the clone's socket durable with + /// `sync`, and that of the binding with `sync_sandbox`, through the handle of the sandbox's + /// directory. + fn release_with( + &self, + locked: &mut LockedStore, + checked: &dyn Fn(), + sync: SyncDir<'_>, + sync_sandbox: &dyn Fn(&platform::Directory) -> io::Result<()>, + ) -> Result> { + let directory = locked.open_dir()?; + // A record whose deletion is pending, on Windows, is removed, as its readers take it. + for name in [JOURNAL_NAME, RUNTIME_NAME] { + let present = open_regular_in(&directory, name).map_err(store_error(format!( + "cannot examine {}", + locked.dir().join(name).display() + )))?; + if present.is_some() { + return Err(may_run(locked.dir())); + } } - let Some(lease) = read_binding(locked.dir())? else { + let Some(lease) = read_binding_in(&directory, locked.dir())? else { return Ok(None); }; if lease.store() != self.id { @@ -402,6 +425,7 @@ impl Store { ))); } let recorded = self.check_records_dir(LEASES_NAME, LEASES_WHAT)?; + checked(); // A plan of the clone launches under the reservation that the lease's acquisition left in // the lock, which the release ends before it removes anything, so that a release that // fails once it removed the clone's files, such as the scratch device's file of an upper @@ -413,9 +437,18 @@ impl Store { // all are entries of the sandbox's directory, but not that of the socket, an entry of the // leased generation's payload, which must be durable before the lease ends, so that a // generation whose leases have drained holds no clone's socket. - locked.remove_clone_files()?; + locked.remove_clone_files(&directory)?; self.remove_clone_socket(&lease, sync)?; - unbind(locked.dir(), sync)?; + let binding = locked.dir().join(CLONE_BINDING_NAME); + directory + .remove_file(CLONE_BINDING_NAME) + .map_err(store_error(format!("cannot remove {}", binding.display())))?; + // The binding's absence is durable before its record goes: a binding that outlasted the + // record through a crash would let a retirement delete the generation that it names. + sync_sandbox(&directory).map_err(store_error(format!( + "cannot sync the directory {}", + locked.dir().display() + )))?; if recorded { remove_lease_record(&self.lease_path(lease.id()))?; } diff --git a/nvx-rs/src/host/template/store/leases/tests.rs b/nvx-rs/src/host/template/store/leases/tests.rs index 3e2b19e55..bfafe04e3 100644 --- a/nvx-rs/src/host/template/store/leases/tests.rs +++ b/nvx-rs/src/host/template/store/leases/tests.rs @@ -18,13 +18,14 @@ use crate::activation::TemplateReference; use crate::host::process::{Process, ProcessRef}; use crate::host::store::files::write_private; use crate::host::store::{ - CLONE_ACTIVATION_NAME, CLONE_SCRATCH_NAME, FORMAT, LaunchState, RESTORE_READY_NAME, + CLONE_ACTIVATION_NAME, CLONE_SCRATCH_NAME, FORMAT, LOCK_NAME, LaunchState, RESTORE_READY_NAME, RUNTIME_NAME, Recovery, RuntimeIdentity, }; use crate::host::supervisor::{self, Reattached, StopOutcome, VmmCommand}; use crate::host::template::eligibility::MAX_DOWNTIME_NS; use crate::host::template::store::tests::{ - link_dir, link_file, publish, publish_with, remove, storage, store as new_store, unlink_dir, + cause, link_dir, link_file, publish, publish_with, remove, storage, store as new_store, + unlink_dir, }; use crate::host::template::test_support::{ManifestSpec, artifact, network_class}; use crate::host::test_support::released; @@ -797,7 +798,7 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco RESTORE_READY_NAME, ]; let synced = RefCell::new(Vec::new()); - let sync = |dir: &Path| -> Result<()> { + let note = |dir: &Path| { let clone = kept .iter() .any(|name| fs::symlink_metadata(sandbox.join(name)).is_ok()) @@ -806,6 +807,16 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco synced .borrow_mut() .push((dir.to_owned(), clone, bound, record.exists())); + }; + let sync = |dir: &Path| -> Result<()> { + note(dir); + Ok(()) + }; + // The release syncs the sandbox's directory through its handle, which holds the lock's file. + let lock = platform::file_identity(&sandbox.join(LOCK_NAME)).unwrap(); + let sync_sandbox = |directory: &platform::Directory| -> io::Result<()> { + assert_eq!(directory.entry_identity(LOCK_NAME).unwrap(), Some(lock)); + note(&sandbox); Ok(()) }; for name in kept { @@ -816,7 +827,12 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco fs::write(&socket, b"").unwrap(); } let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); - assert_eq!(store.release_with(&mut locked, &sync).unwrap(), Some(lease)); + assert_eq!( + store + .release_with(&mut locked, &|| {}, &sync, &sync_sandbox) + .unwrap(), + Some(lease) + ); drop(locked); let mut expected = vec![(sandbox.clone(), false, false, true)]; if cfg!(unix) { @@ -847,13 +863,18 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco // A release that cannot make the socket's removal durable keeps the binding, and the lease. if cfg!(unix) { let error = store - .release_with(&mut locked, &|dir: &Path| { - if dir == payload { - Err(Error::io("the disk failed")) - } else { - Ok(()) - } - }) + .release_with( + &mut locked, + &|| {}, + &|dir: &Path| { + if dir == payload { + Err(Error::io("the disk failed")) + } else { + Ok(()) + } + }, + &|_| Ok(()), + ) .unwrap_err(); assert_eq!(error.message(), "the disk failed"); assert_eq!(binding(&sandbox), Some(again.clone())); @@ -863,18 +884,18 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco // A release that cannot make the binding's removal durable keeps the record, which a // retirement removes once no binding names it. let error = store - .release_with(&mut locked, &|dir: &Path| { - if dir == sandbox { - Err(Error::io("the disk failed")) - } else { - Ok(()) - } + .release_with(&mut locked, &|| {}, &|_: &Path| Ok(()), &|_| { + Err(io::Error::other("the disk failed")) }) .unwrap_err(); assert_eq!(locked.reserved(), None); assert!(fs::symlink_metadata(&scratch).is_err()); drop(locked); - assert_eq!(error.message(), "the disk failed"); + assert_eq!( + error.message(), + format!("cannot sync the directory {}", sandbox.display()) + ); + assert_eq!(cause(&error), "the disk failed"); assert_eq!(binding(&sandbox), None); assert_eq!(store.leases(generation).unwrap(), [again]); assert_eq!(store.retire(generation, None).unwrap(), Retirement::Retired); @@ -1151,21 +1172,20 @@ fn a_release_takes_the_lock_of_its_sandbox_through_the_path_of_its_lease() { let link = root.join("link"); link_dir(&first, &link); // A lock taken through a link refuses a release, which removes nothing, also once the link - // names another sandbox, whose lease a release would otherwise end without its lock. + // names another sandbox, whose lease a release would otherwise end without its lock: the + // directory that the link names then holds another lock file than the one locked. let mut locked = SandboxStore::lock_existing(link.join("sandbox"), &files(), None).unwrap(); - for retargeted in [false, true] { + for (retargeted, refusal) in [ + (false, "does not name it by the path of its lease"), + (true, "does not hold the lock of its sandbox"), + ] { if retargeted { unlink_dir(&link); link_dir(&second, &link); } let error = store.release(&mut locked).unwrap_err(); assert_eq!(error.kind(), ErrorKind::Other, "{error}"); - assert!( - error - .message() - .contains("does not name it by the path of its lease"), - "{error}" - ); + assert!(error.message().contains(refusal), "{error}"); assert_eq!(binding(&leased_sandbox), Some(lease.clone())); assert_eq!(binding(&other), Some(other_lease.clone())); assert_eq!(store.leases(generation).unwrap(), both); @@ -1177,6 +1197,101 @@ fn a_release_takes_the_lock_of_its_sandbox_through_the_path_of_its_lease() { assert_eq!(store.leases(generation).unwrap(), [other_lease]); } +/// A release whose path comes to name another sandbox, once a directory in the path is renamed, +/// and a link to the other sandbox's parent takes its place: only on Unix, since Windows refuses +/// to rename a directory that holds an open file, such as the lock's. +#[cfg(unix)] +#[test] +fn a_release_ends_the_lease_of_the_sandbox_whose_lock_it_holds_whatever_its_path_names_meanwhile() { + let (directory, store) = new_store(); + let root = filesystem::canonicalize(directory.path()).unwrap(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let (first, second) = (root.join("first"), root.join("second")); + fs::create_dir(&first).unwrap(); + fs::create_dir(&second).unwrap(); + let leased_sandbox = provision(&first, "sandbox"); + let other = provision(&second, "sandbox"); + let lease = leased(&store, &published, &leased_sandbox); + let other_lease = leased(&store, &published, &other); + let kept = [ + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, + ]; + for sandbox in [&leased_sandbox, &other] { + for name in kept { + fs::write(sandbox.join(name), b"").unwrap(); + } + } + // Moves `parent` to `to`, and links `parent` to the other sandbox's parent. + let swap = |parent: &Path, to: &Path| { + fs::rename(parent, to).unwrap(); + link_dir(&second, parent); + }; + // The other sandbox's files, and its lease, stay. + let other_stays = || { + for name in kept { + assert!(other.join(name).exists(), "{name}"); + } + assert_eq!(binding(&other), Some(other_lease.clone())); + }; + // Once the release has checked the sandbox, its path comes to name the other sandbox, from + // the removal of its files on. + let moved = root.join("moved"); + let mut locked = SandboxStore::lock_existing(&leased_sandbox, &files(), None).unwrap(); + assert_eq!( + store + .release_with( + &mut locked, + &|| swap(&first, &moved), + &sync_dir, + &platform::Directory::sync + ) + .unwrap(), + Some(lease) + ); + drop(locked); + // The release removed the files and the binding of the sandbox whose lock it held. + let released = moved.join("sandbox"); + for name in kept { + assert!(fs::symlink_metadata(released.join(name)).is_err(), "{name}"); + } + assert_eq!(binding(&released), None); + other_stays(); + assert_eq!( + store.leases(generation).unwrap(), + std::slice::from_ref(&other_lease) + ); + // A release whose path names the other sandbox once the lock is taken, before the release + // begins, refuses, and removes nothing. + let again = leased(&store, &published, &released); + for name in kept { + fs::write(released.join(name), b"").unwrap(); + } + let mut locked = SandboxStore::lock_existing(&released, &files(), None).unwrap(); + let away = root.join("away"); + swap(&moved, &away); + let error = store.release(&mut locked).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Other, "{error}"); + assert!( + error + .message() + .contains("does not hold the lock of its sandbox"), + "{error}" + ); + drop(locked); + let kept_sandbox = away.join("sandbox"); + for name in kept { + assert!(kept_sandbox.join(name).exists(), "{name}"); + } + assert_eq!(binding(&kept_sandbox), Some(again.clone())); + other_stays(); + let mut both = vec![again, other_lease]; + both.sort_by_key(Lease::id); + assert_eq!(store.leases(generation).unwrap(), both); +} + #[test] fn a_retirement_that_finds_its_generation_unindexed_makes_the_index_durable_first() { let (_directory, store) = new_store(); @@ -1807,10 +1922,11 @@ fn clone_role() { }; let store = Store::open(dir).unwrap(); let generation = StoreGeneration::new(generation.parse().unwrap()); - // The step's transaction crashes as it makes its write in `target` durable. + // The step's transaction crashes as it makes its write in `target` durable, or, for a + // release, the removal of the binding, in the sandbox's directory. let target = match step.as_str() { "after-record" => Some(store.dir().join(LEASES_NAME)), - "after-binding" | "releasing" => Some(sandbox.clone()), + "after-binding" => Some(sandbox.clone()), _ => None, }; let sync = |dir: &Path| -> Result<()> { @@ -1821,7 +1937,9 @@ fn clone_role() { }; if step == "releasing" { let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); - store.release_with(&mut locked, &sync).unwrap(); + store + .release_with(&mut locked, &|| {}, &sync, &|_| crash(&step)) + .unwrap(); panic!("the release did not make the removal of its binding durable"); } let now = after(store.index().unwrap().get(generation).unwrap(), 1); @@ -2446,4 +2564,36 @@ mod windows { drop(held_record); assert!(store.leases(generation).unwrap().is_empty()); } + + #[test] + fn a_release_takes_the_files_of_its_sandbox_whose_deletion_is_pending_for_removed() { + let (directory, store) = new_store(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let sandbox = provision(directory.path(), "sandbox"); + let lease = leased(&store, &published, &sandbox); + // A stop removed the runtime record of the clone's VM, and the journal of its launch, + // which a scanner, say, holds: the release, which reads them through its handle of the + // sandbox's directory, finds that the sandbox records no VM and no launch. + let held: Vec = [JOURNAL_NAME, RUNTIME_NAME] + .into_iter() + .map(|name| { + fs::write(sandbox.join(name), b"{}").unwrap(); + delete_pending(&sandbox.join(name)) + }) + .collect(); + assert_eq!(release(&store, &sandbox).unwrap(), Some(lease)); + assert_eq!(binding(&sandbox), None); + assert!(store.leases(generation).unwrap().is_empty()); + drop(held); + // A release that removed the binding, which a reader holds, and failed before its record + // went, leaves a binding whose deletion is pending, which a release retried finds removed. + let again = leased(&store, &published, &sandbox); + let held_binding = delete_pending(&sandbox.join(CLONE_BINDING_NAME)); + assert_eq!(release(&store, &sandbox).unwrap(), None); + drop(held_binding); + assert_eq!(binding(&sandbox), None); + assert_eq!(store.leases(generation).unwrap(), [again]); + assert_eq!(store.retire(generation, None).unwrap(), Retirement::Retired); + } } From e9d753144528fbaf5d71bcd5e6a9ff223f204158 Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 28 Oct 2026 08:04:28 -0700 Subject: [PATCH 6/7] nvx-rs: tell the files of a store apart by their 128-bit IDs on Windows The stores check that a path, or an entry of a directory that they hold open, names the file whose lock they hold: as a lock is taken, as a sandbox's store is removed, as a release opens the sandbox's directory, and as a holder removes its lock file. A clone's plan finds whether its scratch image is in the template store by comparing the directories above the image with the store's. On Windows, all of these compared the volume serial number and the 64-bit file index, which a file system such as ReFS does not keep unique, so another file could pass for the lock's, or a directory for the store's. They now compare the identity that a file's seal holds, through platform::file_id, opened_file_id and Directory::entry_id: the device and inode on Linux, and on Windows the volume serial number and the 128-bit file ID of FILE_ID_INFO, which ReFS keeps unique too. The 64-bit identity stays where it is recorded: a launch's log claim, and the identities of host paths. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a424884c-018a-45f5-a7fe-c57548a873f0 --- nvx-rs/src/host/platform/linux.rs | 22 +++++++++-- nvx-rs/src/host/platform/mod.rs | 26 +++++++++++++ nvx-rs/src/host/platform/other.rs | 12 +++++- nvx-rs/src/host/platform/windows.rs | 38 +++++++++++++++---- nvx-rs/src/host/store/lifecycle.rs | 11 +++--- nvx-rs/src/host/store/lifecycle/tests.rs | 4 +- nvx-rs/src/host/store/mod.rs | 8 ++-- nvx-rs/src/host/store/tests.rs | 4 +- nvx-rs/src/host/template/clone.rs | 4 +- nvx-rs/src/host/template/store.rs | 4 +- .../src/host/template/store/leases/tests.rs | 4 +- 11 files changed, 104 insertions(+), 33 deletions(-) diff --git a/nvx-rs/src/host/platform/linux.rs b/nvx-rs/src/host/platform/linux.rs index da60d4de7..d562f1bb9 100644 --- a/nvx-rs/src/host/platform/linux.rs +++ b/nvx-rs/src/host/platform/linux.rs @@ -639,6 +639,20 @@ pub(crate) fn opened_file_identity(file: &File) -> io::Result<(u64, u64)> { Ok((metadata.dev(), metadata.ino())) } +/// Returns the identity by which the stores tell a file or directory apart from every other, +/// following its links: its device and inode, as [`opened_file_id`] gives them. +pub(crate) fn file_id(path: &Path) -> io::Result<(u64, u128)> { + let metadata = fs::metadata(path)?; + Ok((metadata.dev(), u128::from(metadata.ino()))) +} + +/// Returns the identity by which the stores tell the file that `file` holds open apart from every +/// other: its device and inode, as its [`FileSeal`](crate::host::seal::FileSeal) holds them. +pub(crate) fn opened_file_id(file: &File) -> io::Result<(u64, u128)> { + let metadata = file.metadata()?; + Ok((metadata.dev(), u128::from(metadata.ino()))) +} + /// Returns the number of names, its hard links, of the file that `file` holds open. pub(crate) fn link_count(file: &File) -> io::Result { Ok(file.metadata()?.nlink()) @@ -660,14 +674,14 @@ impl Directory { .map(Self) } - /// Returns the device and inode of its entry `name`, not following a link, as - /// [`opened_file_identity`] gives them, or `None` if it has no such entry. - pub(crate) fn entry_identity(&self, name: &str) -> io::Result> { + /// Returns the identity of its entry `name`, not following a link, as [`opened_file_id`] gives + /// it, or `None` if it has no such entry. + pub(crate) fn entry_id(&self, name: &str) -> io::Result> { use nix::fcntl::AtFlags; use nix::sys::stat::fstatat; match fstatat(&self.0, name, AtFlags::AT_SYMLINK_NOFOLLOW) { - Ok(status) => Ok(Some((status.st_dev, status.st_ino))), + Ok(status) => Ok(Some((status.st_dev, u128::from(status.st_ino)))), Err(Errno::ENOENT) => Ok(None), Err(errno) => Err(errno.into()), } diff --git a/nvx-rs/src/host/platform/mod.rs b/nvx-rs/src/host/platform/mod.rs index 3243f86ca..c1886f7dd 100644 --- a/nvx-rs/src/host/platform/mod.rs +++ b/nvx-rs/src/host/platform/mod.rs @@ -171,4 +171,30 @@ mod tests { Err(error) => panic!("{error}"), } } + + /// The stores tell files apart by the identities that their seals hold, with the 128-bit file + /// IDs of `FILE_ID_INFO` on Windows, whether they examine a path, an open file, or an entry of + /// an open directory. + #[cfg(any(target_os = "linux", windows))] + #[test] + fn the_stores_tell_files_apart_by_the_identities_that_their_seals_hold() { + use crate::host::seal::FileSeal; + + let directory = tempfile::tempdir().unwrap(); + let opened = Directory::open(directory.path()).unwrap(); + let mut ids = Vec::new(); + for name in ["first", "second"] { + let path = directory.path().join(name); + std::fs::write(&path, b"").unwrap(); + let file = std::fs::File::open(&path).unwrap(); + let seal = FileSeal::of(&file).unwrap(); + let id = (seal.device(), seal.file_id()); + assert_eq!(opened_file_id(&file).unwrap(), id, "{name}"); + assert_eq!(file_id(&path).unwrap(), id, "{name}"); + assert_eq!(opened.entry_id(name).unwrap(), Some(id), "{name}"); + ids.push(id); + } + assert_ne!(ids[0], ids[1]); + assert_eq!(opened.entry_id("missing").unwrap(), None); + } } diff --git a/nvx-rs/src/host/platform/other.rs b/nvx-rs/src/host/platform/other.rs index 8923ad434..b3ef3b66d 100644 --- a/nvx-rs/src/host/platform/other.rs +++ b/nvx-rs/src/host/platform/other.rs @@ -98,6 +98,16 @@ pub(crate) fn opened_file_identity(_file: &File) -> io::Result<(u64, u64)> { Err(unsupported()) } +/// Fails: this host keeps no store. +pub(crate) fn file_id(_path: &Path) -> io::Result<(u64, u128)> { + Err(unsupported()) +} + +/// Fails: this host keeps no store. +pub(crate) fn opened_file_id(_file: &File) -> io::Result<(u64, u128)> { + Err(unsupported()) +} + pub(crate) fn link_count(_file: &File) -> io::Result { Err(unsupported()) } @@ -112,7 +122,7 @@ impl Directory { } /// Never called: no directory is open. - pub(crate) fn entry_identity(&self, _name: &str) -> io::Result> { + pub(crate) fn entry_id(&self, _name: &str) -> io::Result> { match *self {} } diff --git a/nvx-rs/src/host/platform/windows.rs b/nvx-rs/src/host/platform/windows.rs index 1b14972b0..c75216fe8 100644 --- a/nvx-rs/src/host/platform/windows.rs +++ b/nvx-rs/src/host/platform/windows.rs @@ -1366,7 +1366,8 @@ pub(crate) fn set_access(path: &Path, sddl: &str) -> io::Result<()> { result } -/// Returns the volume serial number and file index of a file or directory. +/// Returns the volume serial number and 64-bit file index of a file or directory, which a file +/// system such as ReFS does not keep unique. /// /// The object is opened without access rights, which succeeds even while OpenVMM holds it open. pub fn file_identity(path: &Path) -> io::Result<(u64, u64)> { @@ -1377,6 +1378,18 @@ pub fn file_identity(path: &Path) -> io::Result<(u64, u64)> { handle_identity(file.as_raw_handle() as HANDLE) } +/// Returns the identity by which the stores tell a file or directory apart from every other, +/// following its links, as [`opened_file_id`] gives it. +/// +/// The object is opened without access rights, which succeeds even while OpenVMM holds it open. +pub(crate) fn file_id(path: &Path) -> io::Result<(u64, u128)> { + let file = OpenOptions::new() + .access_mode(0) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(path)?; + opened_file_id(&file) +} + /// Returns whether the file at `path` is deleted: whether an open of it finds no file, or one /// whose deletion is pending, which a file system without POSIX deletion keeps under its name, /// refusing every open of it, until the last handle to it closes. A file that opens is not, nor is @@ -1425,10 +1438,9 @@ impl Directory { .map(Self) } - /// Returns the volume serial number and file index of its entry `name`, not following a link - /// or other reparse point, as [`opened_file_identity`] gives them, or `None` if it has no such - /// entry. - pub(crate) fn entry_identity(&self, name: &str) -> io::Result> { + /// Returns the identity of its entry `name`, not following a link or other reparse point, as + /// [`opened_file_id`] gives it, or `None` if it has no such entry. + pub(crate) fn entry_id(&self, name: &str) -> io::Result> { use windows_sys::Wdk::Storage::FileSystem::{ FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, }; @@ -1440,7 +1452,7 @@ impl Directory { FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, ) { - Ok(entry) => handle_identity(raw(&entry)).map(Some), + Ok(entry) => opened_file_id(&fs::File::from(entry)).map(Some), Err(STATUS_OBJECT_NAME_NOT_FOUND) => Ok(None), Err(status) => Err(status_error(status)), } @@ -1611,11 +1623,21 @@ fn status_error(status: windows_sys::Win32::Foundation::NTSTATUS) -> io::Error { io::Error::from_raw_os_error(code as i32) } -/// Returns the volume serial number and file index of the file that `file` holds open. +/// Returns the volume serial number and 64-bit file index of the file that `file` holds open, +/// which a file system such as ReFS does not keep unique: [`opened_file_id`] tells files apart. pub(crate) fn opened_file_identity(file: &fs::File) -> io::Result<(u64, u64)> { handle_identity(file.as_raw_handle() as HANDLE) } +/// Returns the identity by which the stores tell the file that `file` holds open apart from every +/// other, as its [`FileSeal`](crate::host::seal::FileSeal) holds it: the volume serial number and +/// the 128-bit file ID of `FILE_ID_INFO`, which a file system such as ReFS keeps unique where the +/// 64-bit file index is not. +pub(crate) fn opened_file_id(file: &fs::File) -> io::Result<(u64, u128)> { + let seal = crate::host::seal::FileSeal::of(file)?; + Ok((seal.device(), seal.file_id())) +} + /// Returns the number of names, its hard links, of the file that `file` holds open. pub(crate) fn link_count(file: &fs::File) -> io::Result { Ok(u64::from( @@ -1623,7 +1645,7 @@ pub(crate) fn link_count(file: &fs::File) -> io::Result { )) } -/// Returns the volume serial number and file index of the file that `handle` holds open. +/// Returns the volume serial number and 64-bit file index of the file that `handle` holds open. fn handle_identity(handle: HANDLE) -> io::Result<(u64, u64)> { let information = handle_information(handle)?; Ok(( diff --git a/nvx-rs/src/host/store/lifecycle.rs b/nvx-rs/src/host/store/lifecycle.rs index f28adf52f..0dedfb0c8 100644 --- a/nvx-rs/src/host/store/lifecycle.rs +++ b/nvx-rs/src/host/store/lifecycle.rs @@ -246,9 +246,8 @@ impl LockedStore { return; } let path = self.path(LOCK_NAME); - let held = platform::opened_file_identity(&self.lock); - if matches!((held, platform::file_identity(&path)), (Ok(held), Ok(current)) if held == current) - { + let held = platform::opened_file_id(&self.lock); + if matches!((held, platform::file_id(&path)), (Ok(held), Ok(current)) if held == current) { let _ = fs::remove_file(&path); } } @@ -386,9 +385,9 @@ impl LockedStore { move |error: io::Error| RemovalError::Io { path, error } }; let directory = platform::Directory::open(dir).map_err(failed)?; - let held = platform::opened_file_identity(&self.lock).map_err(entry_failed(LOCK_NAME))?; + let held = platform::opened_file_id(&self.lock).map_err(entry_failed(LOCK_NAME))?; if directory - .entry_identity(LOCK_NAME) + .entry_id(LOCK_NAME) .map_err(entry_failed(LOCK_NAME))? != Some(held) { @@ -398,7 +397,7 @@ impl LockedStore { } for name in [JOURNAL_NAME, RUNTIME_NAME] { if directory - .entry_identity(name) + .entry_id(name) .map_err(entry_failed(name))? .is_some() { diff --git a/nvx-rs/src/host/store/lifecycle/tests.rs b/nvx-rs/src/host/store/lifecycle/tests.rs index f78bd7457..ee248e2f6 100644 --- a/nvx-rs/src/host/store/lifecycle/tests.rs +++ b/nvx-rs/src/host/store/lifecycle/tests.rs @@ -435,7 +435,7 @@ fn a_removal_makes_the_absence_of_the_clones_files_durable_before_it_removes_any locked .remove_contents_with(&|| {}, &|directory| { // The directory that the sync makes durable holds the sandbox's lock file. - let lock = directory.entry_identity(LOCK_NAME).unwrap(); + let lock = directory.entry_id(LOCK_NAME).unwrap(); synced.borrow_mut().push((lock, entries_of(&dir))); Ok(()) }) @@ -446,7 +446,7 @@ fn a_removal_makes_the_absence_of_the_clones_files_durable_before_it_removes_any assert_eq!( synced.into_inner(), [( - Some(platform::file_identity(&dir.join(LOCK_NAME)).unwrap()), + Some(platform::file_id(&dir.join(LOCK_NAME)).unwrap()), sorted(&[BOOT, CONFIG, LOCK_NAME, LOG_NAME]) )] ); diff --git a/nvx-rs/src/host/store/mod.rs b/nvx-rs/src/host/store/mod.rs index faecc57ab..5d0c9ebe3 100644 --- a/nvx-rs/src/host/store/mod.rs +++ b/nvx-rs/src/host/store/mod.rs @@ -639,8 +639,8 @@ impl SandboxStore { } }, } - let locked = platform::opened_file_identity(&file).map_err(failed)?; - match platform::file_identity(&path) { + let locked = platform::opened_file_id(&file).map_err(failed)?; + match platform::file_id(&path) { Ok(current) if current == locked => Ok(Taken::Locked(file)), Ok(_) => Ok(Taken::Replaced), Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Taken::Replaced), @@ -793,8 +793,8 @@ impl LockedStore { )))?; let lock = self.path(LOCK_NAME); let examine = || io_error(format!("cannot examine {}", lock.display())); - let held = platform::opened_file_identity(&self.lock).map_err(examine())?; - if directory.entry_identity(LOCK_NAME).map_err(examine())? != Some(held) { + let held = platform::opened_file_id(&self.lock).map_err(examine())?; + if directory.entry_id(LOCK_NAME).map_err(examine())? != Some(held) { return Err(Error::other(format!( "the directory that {} names does not hold the lock of its sandbox: the path names \ another directory than when the lock was taken", diff --git a/nvx-rs/src/host/store/tests.rs b/nvx-rs/src/host/store/tests.rs index f9d4157ca..ef11598a0 100644 --- a/nvx-rs/src/host/store/tests.rs +++ b/nvx-rs/src/host/store/tests.rs @@ -323,8 +323,8 @@ fn a_lock_file_replaced_while_waiting_locks_nothing() { move || { let locked = store.lock(None).unwrap(); // The waiter holds the lock of the file that the path names now. - let current = platform::file_identity(&locked.path(LOCK_NAME)).unwrap(); - let held = platform::opened_file_identity(&locked.lock).unwrap(); + let current = platform::file_id(&locked.path(LOCK_NAME)).unwrap(); + let held = platform::opened_file_id(&locked.lock).unwrap(); assert_eq!(current, held); } }); diff --git a/nvx-rs/src/host/template/clone.rs b/nvx-rs/src/host/template/clone.rs index 8178cfa1a..ac78de75c 100644 --- a/nvx-rs/src/host/template/clone.rs +++ b/nvx-rs/src/host/template/clone.rs @@ -777,9 +777,9 @@ fn check_outside_store(checked: &CheckedScratch, store: &Store, path: &Path) -> /// those that went from their paths meanwhile, as a publication renames a capture's transaction, /// which `dir`, the store's own directory, outlives. fn is_within(location: &Path, dir: &Path) -> io::Result { - let dir = platform::file_identity(dir)?; + let dir = platform::file_id(dir)?; for ancestor in location.ancestors().skip(1) { - match platform::file_identity(ancestor) { + match platform::file_id(ancestor) { Ok(identity) if identity == dir => return Ok(true), Ok(_) => {} Err(error) diff --git a/nvx-rs/src/host/template/store.rs b/nvx-rs/src/host/template/store.rs index 05ff211e9..c5cf41938 100644 --- a/nvx-rs/src/host/template/store.rs +++ b/nvx-rs/src/host/template/store.rs @@ -891,8 +891,8 @@ fn take_lock(dir: &Path, deadline: Option) -> Result { } }, } - let locked = platform::opened_file_identity(&file).map_err(failed)?; - match platform::file_identity(&path) { + let locked = platform::opened_file_id(&file).map_err(failed)?; + match platform::file_id(&path) { Ok(current) if current == locked => return Ok(file), Ok(_) => {} Err(error) if error.kind() == io::ErrorKind::NotFound => {} diff --git a/nvx-rs/src/host/template/store/leases/tests.rs b/nvx-rs/src/host/template/store/leases/tests.rs index bfafe04e3..335a96618 100644 --- a/nvx-rs/src/host/template/store/leases/tests.rs +++ b/nvx-rs/src/host/template/store/leases/tests.rs @@ -813,9 +813,9 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco Ok(()) }; // The release syncs the sandbox's directory through its handle, which holds the lock's file. - let lock = platform::file_identity(&sandbox.join(LOCK_NAME)).unwrap(); + let lock = platform::file_id(&sandbox.join(LOCK_NAME)).unwrap(); let sync_sandbox = |directory: &platform::Directory| -> io::Result<()> { - assert_eq!(directory.entry_identity(LOCK_NAME).unwrap(), Some(lock)); + assert_eq!(directory.entry_id(LOCK_NAME).unwrap(), Some(lock)); note(&sandbox); Ok(()) }; From df208b3a1ff846483144949773e0ab84e0aa1a8f Mon Sep 17 00:00:00 2001 From: Enrique Saurez Date: Wed, 28 Oct 2026 08:09:54 -0700 Subject: [PATCH 7/7] nvx-rs: remove a clone's socket through a handle on its generation's payload On Linux, the control socket of a clone's VMM is in the payload of the leased generation, generations/ in the template store. A release, a recovery of the clone's launch, and the clearing of its runtime record removed the socket by its path, which follows a link that took the place of the store's directory, of the directory of the generations, or of the payload, so they could remove a file of another directory, and the release then synced that directory. They now open the payload through a handle of the store's directory, following no link at the store's directory, at the directory of the generations, or at the payload, and remove the socket through that handle. A link, or a file, in the place of one of them is refused. Through the handle of the store's directory, they first read the store's identity record, and take a directory that holds another store, or none, as one that holds no socket of the lease's: its files stay. The identity record's reader moves to the module of the leases, beside the directory of the generations, so that the sandbox's store reads it without the template store's module. The store's directory and the directory of the generations open with the right to search them alone, which an owner-only directory may grant without the right to read it, so that such a directory does not block the removal; the payload opens for reading, so that its handle can make the removal durable. A release opens the payload before it removes anything, so that a refused link refuses it and nothing goes, and makes the socket's absence durable through the payload's handle, whatever the payload's path names meanwhile. A release needs the payload to be a directory of the current user with mode 0700, as a launch does: it opens the payload for reading, to make the removal durable, and removes an entry from it. An acquisition now refuses, on Linux, a payload that is not, so that it grants no lease that its release could not end. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a424884c-018a-45f5-a7fe-c57548a873f0 --- nvx-rs/src/host/platform/linux.rs | 64 ++++ nvx-rs/src/host/platform/other.rs | 15 + nvx-rs/src/host/platform/windows.rs | 61 ++++ nvx-rs/src/host/store/lease.rs | 97 +++++ nvx-rs/src/host/store/mod.rs | 30 +- .../src/host/template/clone/tests/launch.rs | 19 +- nvx-rs/src/host/template/store.rs | 40 +- nvx-rs/src/host/template/store/leases.rs | 133 +++++-- .../src/host/template/store/leases/tests.rs | 343 +++++++++++++++++- 9 files changed, 712 insertions(+), 90 deletions(-) diff --git a/nvx-rs/src/host/platform/linux.rs b/nvx-rs/src/host/platform/linux.rs index d562f1bb9..1157a689f 100644 --- a/nvx-rs/src/host/platform/linux.rs +++ b/nvx-rs/src/host/platform/linux.rs @@ -706,6 +706,70 @@ impl Directory { } } + /// Opens the directory `path` with the right to search it alone, without following a link at + /// its own name, though links on the way to it are followed, or returns `None` if it does not + /// exist. Fails if it is a link, or not a directory. + /// + /// The handle names the directory's entries, and needs no right to read the directory, which + /// an owner-only directory may withhold; it cannot [sync](Self::sync) it. + pub(crate) fn search(path: &Path) -> io::Result> { + use nix::fcntl::OFlag; + + match OpenOptions::new() + .read(true) + .custom_flags((OFlag::O_PATH | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW).bits()) + .open(path) + { + Ok(dir) => Ok(Some(Self(dir))), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) + if [Errno::ELOOP, Errno::ENOTDIR] + .map(|errno| Some(errno as i32)) + .contains(&error.raw_os_error()) => + { + Err(io::Error::other(format!( + "{} is a link, or not a directory", + path.display() + ))) + } + Err(error) => Err(error), + } + } + + /// Opens its subdirectory `name` with the right to search it alone, not following a link, as + /// [`search`](Self::search) opens a directory, or returns `None` if it has no such entry. + /// Fails if the entry is a link, or not a directory. + pub(crate) fn search_dir(&self, name: &str) -> io::Result> { + self.open_dir_with(name, nix::fcntl::OFlag::O_PATH) + } + + /// Opens its subdirectory `name` for reading, not following a link, or returns `None` if it + /// has no such entry. Fails if the entry is a link, or not a directory. + pub(crate) fn open_dir(&self, name: &str) -> io::Result> { + self.open_dir_with(name, nix::fcntl::OFlag::O_RDONLY) + } + + /// Opens its subdirectory `name` with the access `access`, not following a link, as + /// [`open_dir`](Self::open_dir) and [`search_dir`](Self::search_dir) do. + fn open_dir_with(&self, name: &str, access: nix::fcntl::OFlag) -> io::Result> { + use nix::fcntl::{OFlag, openat}; + use nix::sys::stat::Mode; + + match openat( + &self.0, + name, + access | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC, + Mode::empty(), + ) { + Ok(dir) => Ok(Some(Self(File::from(dir)))), + Err(Errno::ENOENT) => Ok(None), + Err(Errno::ELOOP | Errno::ENOTDIR) => Err(io::Error::other(format!( + "{name} is a link, or not a directory" + ))), + Err(errno) => Err(errno.into()), + } + } + /// Removes its file `name`, if it has one. pub(crate) fn remove_file(&self, name: &str) -> io::Result<()> { use nix::unistd::{UnlinkatFlags, unlinkat}; diff --git a/nvx-rs/src/host/platform/other.rs b/nvx-rs/src/host/platform/other.rs index b3ef3b66d..443b748db 100644 --- a/nvx-rs/src/host/platform/other.rs +++ b/nvx-rs/src/host/platform/other.rs @@ -131,6 +131,21 @@ impl Directory { match *self {} } + /// Never called: no directory is open. + pub(crate) fn open_dir(&self, _name: &str) -> io::Result> { + match *self {} + } + + /// Fails: this host keeps no store. + pub(crate) fn search(_path: &Path) -> io::Result> { + Err(unsupported()) + } + + /// Never called: no directory is open. + pub(crate) fn search_dir(&self, _name: &str) -> io::Result> { + match *self {} + } + /// Never called: no directory is open. pub(crate) fn remove_file(&self, _name: &str) -> io::Result<()> { match *self {} diff --git a/nvx-rs/src/host/platform/windows.rs b/nvx-rs/src/host/platform/windows.rs index c75216fe8..b5dd0fa6d 100644 --- a/nvx-rs/src/host/platform/windows.rs +++ b/nvx-rs/src/host/platform/windows.rs @@ -1480,6 +1480,67 @@ impl Directory { } } + /// Opens the directory `path` without following a link or other reparse point at its own + /// name, though those on the way to it are followed, and without access rights, which the + /// opens of its entries do not need, or returns `None` if it does not exist. Fails if it is a + /// link, a junction or another reparse point, or not a directory. + pub(crate) fn search(path: &Path) -> io::Result> { + let opened = OpenOptions::new() + .access_mode(0) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path); + let dir = match opened { + Ok(dir) => dir, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let attributes = handle_information(dir.as_raw_handle() as HANDLE)?.dwFileAttributes; + if attributes & FILE_ATTRIBUTE_DIRECTORY == 0 + || attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + { + return Err(io::Error::other(format!( + "{} is a link, or not a directory", + path.display() + ))); + } + Ok(Some(Self(dir))) + } + + /// Opens its subdirectory `name` as [`open_dir`](Self::open_dir) does: Windows needs no right + /// to list a directory to name its entries. + pub(crate) fn search_dir(&self, name: &str) -> io::Result> { + self.open_dir(name) + } + + /// Opens its subdirectory `name`, not following a link or other reparse point, or returns + /// `None` if it has no such entry, or one whose deletion is pending. Fails if the entry is a + /// link, a junction or another reparse point, or not a directory. + pub(crate) fn open_dir(&self, name: &str) -> io::Result> { + use windows_sys::Wdk::Storage::FileSystem::{ + FILE_DIRECTORY_FILE, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, + }; + use windows_sys::Win32::Foundation::{ + STATUS_DELETE_PENDING, STATUS_NOT_A_DIRECTORY, STATUS_OBJECT_NAME_NOT_FOUND, + }; + use windows_sys::Win32::Storage::FileSystem::{FILE_READ_ATTRIBUTES, SYNCHRONIZE}; + + let refused = || io::Error::other(format!("{name} is a link, or not a directory")); + let entry = match self.open_entry( + OsStr::new(name), + FILE_READ_ATTRIBUTES | SYNCHRONIZE, + FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, + ) { + Ok(entry) => entry, + Err(STATUS_OBJECT_NAME_NOT_FOUND | STATUS_DELETE_PENDING) => return Ok(None), + Err(STATUS_NOT_A_DIRECTORY) => return Err(refused()), + Err(status) => return Err(status_error(status)), + }; + if handle_information(raw(&entry))?.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return Err(refused()); + } + Ok(Some(Self(fs::File::from(entry)))) + } + /// Removes its file `name`, if it has one. A file whose deletion is pending already, which a /// file system without POSIX deletion keeps under its name until its last handle closes, /// counts as removed. diff --git a/nvx-rs/src/host/store/lease.rs b/nvx-rs/src/host/store/lease.rs index 73aa01448..cb534e9fe 100644 --- a/nvx-rs/src/host/store/lease.rs +++ b/nvx-rs/src/host/store/lease.rs @@ -29,6 +29,12 @@ pub const STORE_FORMAT: u32 = 1; /// Name of the directory of the generations' payloads. pub const GENERATIONS_NAME: &str = "generations"; +/// Name of the store's identity record. +pub const STORE_RECORD_NAME: &str = "store.json"; + +/// Longest identity record, in bytes. +pub(crate) const MAX_STORE_RECORD_LEN: u64 = 4 * 1024; + /// Bytes of a [`StoreId`] and of a [`StorageGeneration`](crate::host::template::StorageGeneration). pub(crate) const ID_LEN: usize = 16; @@ -272,6 +278,97 @@ pub(crate) fn is_clone_socket(lease: &Lease, path: &Path) -> bool { == Some(OsStr::new(GENERATIONS_NAME)) } +/// Opens the payload of the generation `generation` of the template store `id` in `store`, which +/// holds the control sockets of the VMMs of the generation's clones on Linux: +/// `generations/`, for reading, so that the caller can sync it once it removed a +/// socket from it. +/// +/// It follows no link at the store's own name, at the directory of the generations, or at the +/// payload, and fails if one of them is a link, or not a directory, so that a link that takes the +/// place of one of them never leads a removal to the files of another directory. It reads the +/// store's identity through the handle of the store's directory, failing if it cannot read the +/// identity record, and returns `None` if that directory holds another store, or none, as it does +/// if the store, its directory of the generations, or the payload is gone: the payload of a store +/// that is gone holds no socket. It opens the store's directory and the directory of the +/// generations with the right to search them alone, which an owner-only directory may grant +/// without the right to read it. +pub(crate) fn open_payload( + store: &Path, + id: StoreId, + generation: StoreGeneration, +) -> Result> { + let payload = store.join(GENERATIONS_NAME).join(generation.to_string()); + let failed = || { + store_error(format!( + "cannot open the payload of the generation {}", + payload.display() + )) + }; + let Some(dir) = platform::Directory::search(store).map_err(failed())? else { + return Ok(None); + }; + if read_identity_in(&dir, store)? != Some(id) { + return Ok(None); + } + let Some(generations) = dir.search_dir(GENERATIONS_NAME).map_err(failed())? else { + return Ok(None); + }; + generations + .open_dir(&generation.to_string()) + .map_err(failed()) +} + +/// Reads the identity of the store in `dir` from its identity record, or returns `None` if the +/// directory holds none. +pub(crate) fn read_identity(dir: &Path) -> Result> { + let path = dir.join(STORE_RECORD_NAME); + read_identity_with(&path, || open_existing(&path)) +} + +/// Reads the identity of the store whose directory `directory` holds open, in `dir`, as +/// [`read_identity`] reads it at its path, or returns `None` if the directory holds no identity +/// record, or, on Windows, one whose deletion is pending: through the handle, so that it reads +/// the identity of the directory that the handle holds, whatever `dir` names meanwhile. +pub(crate) fn read_identity_in( + directory: &platform::Directory, + dir: &Path, +) -> Result> { + read_identity_with(&dir.join(STORE_RECORD_NAME), || { + open_regular_in(directory, STORE_RECORD_NAME) + }) +} + +/// Reads the identity of a store from its identity record at `path`, which `open` opens, or +/// returns `None` if `open` finds none. +fn read_identity_with( + path: &Path, + open: impl FnOnce() -> io::Result>, +) -> Result> { + let description = "identity record of the template store"; + let Some(record) = + read_record_with::(path, description, MAX_STORE_RECORD_LEN, open)? + else { + return Ok(None); + }; + record.store.parse().map(Some).map_err(|error: Error| { + Error::io(format!( + "the {description} {} is malformed: {}", + path.display(), + error.message() + )) + }) +} + +/// The identity record of a store, as JSON. +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct StoreRecord { + /// [`STORE_FORMAT`]. + pub(crate) format: u32, + /// The store's identity. + pub(crate) store: String, +} + /// Reads the clone binding of the sandbox in `dir`, or returns `None` if it holds none, or if the /// directory does not exist. pub(crate) fn read_binding(dir: &Path) -> Result> { diff --git a/nvx-rs/src/host/store/mod.rs b/nvx-rs/src/host/store/mod.rs index 5d0c9ebe3..d6a66e3c8 100644 --- a/nvx-rs/src/host/store/mod.rs +++ b/nvx-rs/src/host/store/mod.rs @@ -251,7 +251,7 @@ use self::files::{io_error, remove_if_present}; pub use self::identity::{ ArtifactIdentity, IdentityCheck, IdentityPart, MAX_IDENTITY_LEN, RuntimeIdentity, }; -use self::lease::{Lease, is_clone_socket}; +use self::lease::{Lease, clone_socket_name, is_clone_socket, open_payload}; pub use self::lifecycle::{Lingering, RemovalError}; pub use self::record::{ LaunchId, LaunchJournal, LaunchState, LogClaim, MAX_ENDPOINT_LEN, RuntimeRecord, @@ -1095,7 +1095,9 @@ impl LockedStore { /// Removes the runtime record, the launch capability, and the control socket of a VM that has /// exited, and any outcome report that its VMM staged without publishing it, the runtime /// record last. The control socket of a clone's VMM, beside the snapshot that it restores, goes - /// too. The log, console log, and outcome report outlive the run. + /// too, and the clearing fails, keeping the runtime record, if a link takes the place of the + /// template store's directory, of its directory of the generations, or of the payload of the + /// socket's generation. The log, console log, and outcome report outlive the run. pub fn clear_runtime(&mut self) -> Result<()> { // Only a Linux endpoint is a file, which may be a clone's beside its snapshot. A record that // cannot be read names none, and a release removes a clone's socket all the same. @@ -1128,6 +1130,15 @@ impl LockedStore { /// names it. Any other endpoint stays: the sandbox's own control socket, which the caller /// removes, and a named pipe. Neither reads the binding. /// + /// The socket goes through a handle of the generation's payload, which a handle of the + /// template store's directory opens, as a release opens it: following no link at the store's + /// directory, at its directory of the generations, or at the payload, and once it checked + /// through that handle that the directory holds the lease's store, so that neither a link that + /// takes the place of one of them nor a directory that takes the place of the store ever leads + /// the removal to the files of another directory. Such a link fails the removal; a payload + /// that is gone holds no socket, and neither does a directory that holds another store, or + /// none, in the place of the lease's, whose files stay. + /// /// The removal is not made durable here: /// [`template::Store::release`](crate::host::template::Store::release) makes it durable before /// the lease ends, and a retirement removes whatever a crash brings back with the generation's @@ -1137,9 +1148,18 @@ impl LockedStore { if !cfg!(unix) || socket == self.socket_path() { return Ok(()); } - match Lease::of(self)? { - Some(lease) if is_clone_socket(&lease, socket) => remove_if_present(socket), - _ => Ok(()), + let Some(lease) = Lease::of(self)?.filter(|lease| is_clone_socket(lease, socket)) else { + return Ok(()); + }; + // The socket is `/generations//.sock`, as the check found. + let Some(store) = socket.ancestors().nth(3) else { + return Ok(()); + }; + match open_payload(store, lease.store(), lease.generation())? { + Some(payload) => payload + .remove_file(&clone_socket_name(lease.launch())) + .map_err(store_error(format!("cannot remove {}", socket.display()))), + None => Ok(()), } } } diff --git a/nvx-rs/src/host/template/clone/tests/launch.rs b/nvx-rs/src/host/template/clone/tests/launch.rs index 2c728a201..dc2495364 100644 --- a/nvx-rs/src/host/template/clone/tests/launch.rs +++ b/nvx-rs/src/host/template/clone/tests/launch.rs @@ -1111,14 +1111,27 @@ fn a_clone_whose_control_socket_openvmm_or_the_host_cannot_use_starts_nothing() ) ); // A payload private to the current user, as a plan requires, but whose mode is not 0700, as - // OpenVMM requires of the directory of its control socket. + // OpenVMM requires of the directory of its control socket. An acquisition refuses such a + // payload too, so its mode changes here once the lease is granted. let fixture = Fixture::new(); + complete_snapshot(&fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let plan = fixture + .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) + .unwrap(); let payload = fixture.store.payload_dir(fixture.generation.generation()); fs::set_permissions(&payload, fs::Permissions::from_mode(0o500)).unwrap(); - let ((kind, message), _) = refused(&fixture, "sandbox"); + let refusal = failure(plan.launch_with( + &mut acquired, + Path::new(OPENVMM), + &runtime_of(&fixture), + hooks, + )); fs::set_permissions(&payload, fs::Permissions::from_mode(0o700)).unwrap(); + assert_unlaunched(&fixture, &acquired); assert_eq!( - (kind, message), + refusal, ( ErrorKind::Io, format!( diff --git a/nvx-rs/src/host/template/store.rs b/nvx-rs/src/host/template/store.rs index c5cf41938..762f45e8c 100644 --- a/nvx-rs/src/host/template/store.rs +++ b/nvx-rs/src/host/template/store.rs @@ -24,17 +24,19 @@ use crate::error::{Error, ErrorCode, ErrorKind, Result}; use crate::host::platform::{self, parse_hex_id}; use crate::host::process::BootId; use crate::host::store::files::{open_lock, write_private}; -use crate::host::store::lease::{ID_LEN, hex, read_private, read_record, store_error}; +use crate::host::store::lease::{ + ID_LEN, StoreRecord, hex, read_identity, read_private, read_record, store_error, +}; #[cfg(all(test, any(target_os = "linux", windows)))] pub(super) use self::capture::SNAPSHOT_ARTIFACTS; pub(super) use self::capture::SOCKET_PATH_LIMIT; pub use self::capture::{CAPTURE_RECORD_NAME, CAPTURES_NAME, Capture, Created, TRANSACTION_NAME}; pub use self::leases::{Acquired, LEASES_NAME, Retirement, TOMBSTONES_NAME}; -pub use crate::host::store::lease::{GENERATIONS_NAME, Lease, LeaseId, STORE_FORMAT, StoreId}; +pub use crate::host::store::lease::{ + GENERATIONS_NAME, Lease, LeaseId, STORE_FORMAT, STORE_RECORD_NAME, StoreId, +}; -/// Name of the store's identity record. -pub const STORE_RECORD_NAME: &str = "store.json"; /// Name of the store's index. pub const INDEX_NAME: &str = "index.json"; /// Name of the lock of the store's index. @@ -49,9 +51,6 @@ pub const MAX_GENERATIONS: usize = 1024; /// The store generation that a new store assigns first. const FIRST_GENERATION: StoreGeneration = StoreGeneration::new(NonZeroU64::MIN); -/// Longest identity record, in bytes. -const MAX_STORE_RECORD_LEN: u64 = 4 * 1024; - /// Longest index, in bytes: more than an index of [`MAX_GENERATIONS`] generations takes. const MAX_INDEX_LEN: u64 = 4 * 1024 * 1024; @@ -917,33 +916,6 @@ fn write_record(path: &Path, record: &T, description: &str) -> Res write_private(path, &text) } -/// Reads the identity of the store in `dir` from its identity record, or returns `None` if the -/// directory holds none. -fn read_identity(dir: &Path) -> Result> { - let path = dir.join(STORE_RECORD_NAME); - let description = "identity record of the template store"; - let Some(record) = read_record::(&path, description, MAX_STORE_RECORD_LEN)? else { - return Ok(None); - }; - record.store.parse().map(Some).map_err(|error: Error| { - Error::io(format!( - "the {description} {} is malformed: {}", - path.display(), - error.message() - )) - }) -} - -/// The identity record of a store, as JSON. -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct StoreRecord { - /// [`STORE_FORMAT`]. - format: u32, - /// The store's identity. - store: String, -} - /// The index of a store, as JSON. #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] diff --git a/nvx-rs/src/host/template/store/leases.rs b/nvx-rs/src/host/template/store/leases.rs index 0fe9692d8..e19d73967 100644 --- a/nvx-rs/src/host/template/store/leases.rs +++ b/nvx-rs/src/host/template/store/leases.rs @@ -24,8 +24,8 @@ use crate::host::filesystem; use crate::host::platform; use crate::host::store::files::{remove_if_present, sync_dir}; use crate::host::store::lease::{ - Lease, LeaseId, LeaseRecord, MAX_LEASE_RECORD_LEN, clone_socket_name, read_binding, - read_binding_in, read_lease, + Lease, LeaseId, LeaseRecord, MAX_LEASE_RECORD_LEN, clone_socket_name, open_payload, + read_binding, read_binding_in, read_lease, }; use crate::host::store::regular::{open_regular_in, was_removed}; use crate::host::store::{ @@ -136,16 +136,17 @@ impl Store { /// In one transaction, under the store's lock and then the sandbox's /// [lifecycle lock](crate::host::store#lifecycle-locks), it checks that the store indexes the /// generation, that no tombstone quarantines it, that it is eligible at `now`, as - /// [`Generation::eligibility`] decides it, and that the sandbox is provisioned, records no VM - /// and no launch, and holds no lease; and it writes the lease's record in the store, and then - /// the same record as the sandbox's clone binding, beside its launch journal, each durable - /// before the next step, so that no binding outlasts, through a crash of the host, the record - /// that keeps its generation in the store. It resolves the links of the sandbox's path before - /// it takes the sandbox's lock, and the lease, the binding, and the lock that it returns all - /// name the sandbox by that path, as [`Lease::sandbox`] returns it, which no later change of a - /// link redirects to another directory. It returns the lease, with the sandbox's lock, the only - /// one under which the caller launches the clone, once, as the launch that the lease reserves, - /// [`Lease::launch`], and releases the store's lock. + /// [`Generation::eligibility`] decides it, that its payload, on Linux, is one that a launch of + /// the clone and a release of its lease can use, and that the sandbox is provisioned, records + /// no VM and no launch, and holds no lease; and it writes the lease's record in the store, and + /// then the same record as the sandbox's clone binding, beside its launch journal, each + /// durable before the next step, so that no binding outlasts, through a crash of the host, the + /// record that keeps its generation in the store. It resolves the links of the sandbox's path + /// before it takes the sandbox's lock, and the lease, the binding, and the lock that it + /// returns all name the sandbox by that path, as [`Lease::sandbox`] returns it, which no later + /// change of a link redirects to another directory. It returns the lease, with the sandbox's + /// lock, the only one under which the caller launches the clone, once, as the launch that the + /// lease reserves, [`Lease::launch`], and releases the store's lock. /// /// The launch that the lease reserves restores the generation's snapshot in place, and its VMM /// listens beside the snapshot, as OpenVMM requires of a restore on Linux: at the control @@ -162,6 +163,10 @@ impl Store { /// - with [`ErrorKind::NotFound`] if the store does not index the generation; /// - with [`ErrorCode::TemplateTombstoned`] if a tombstone quarantines it; /// - with [`ErrorCode::TemplateIneligible`] if it is not eligible at `now`; + /// - on Linux, with [`ErrorKind::Io`] if the generation's payload is not a directory of the + /// current user with mode 0700, which a launch of the clone requires, as OpenVMM does of the + /// directory of a control socket, and which a release needs, to remove the clone's socket + /// from the payload and make its absence durable; /// - with [`ErrorCode::SandboxNotProvisioned`] if the directory does not exist, or holds no /// configuration record of the caller, as `files` declares it; /// - with [`ErrorCode::SandboxRunning`] if the sandbox records a VM, or a launch, which may @@ -223,6 +228,12 @@ impl Store { ), )); } + // A release removes the socket of the clone's VMM through a handle of the payload, which + // it opens for reading, so that it can make the removal durable: a payload that it could + // not open, or remove an entry from, would keep the lease and its binding past it. + if cfg!(unix) { + self.check_payload(generation)?; + } let not_provisioned = || { Error::new( ErrorCode::SandboxNotProvisioned, @@ -376,22 +387,38 @@ impl Store { /// change of the path as the release runs, of a link in it or of one of its directories, then /// makes it check or remove the files of another sandbox, whose lock it does not hold. /// + /// Likewise, on Linux, the release opens the leased generation's payload before it removes + /// anything, through a handle of the store's directory: following no link at the store's + /// directory, at its directory of the generations, or at the payload, and once it checked + /// through that handle that the directory holds this store. It removes the clone's socket and + /// makes its absence durable through the payload's handle, so that neither a link that takes + /// the place of one of them nor a directory that takes the place of the store ever leads it to + /// remove the files of another directory. A directory that holds another store, or none, holds + /// no payload of this store's, so no socket goes. + /// /// Fails with [`ErrorKind::Other`] if the sandbox holds a lease of another store, if the lock /// does not name the sandbox by its lease's path, or if the directory that the path names does - /// not hold the lock's file, and then removes nothing. + /// not hold the lock's file, and with [`ErrorKind::Io`] if the store's directory, its + /// directory of the generations, or the payload is a link, or not a directory, and then + /// removes nothing. pub fn release(&self, locked: &mut LockedStore) -> Result> { - self.release_with(locked, &|| {}, &sync_dir, &platform::Directory::sync) + self.release_with( + locked, + &|| {}, + &platform::Directory::sync, + &platform::Directory::sync, + ) } /// Releases a lease as [`release`](Self::release) does, running `checked` once its checks /// pass, before it removes anything, and making the removal of the clone's socket durable with - /// `sync`, and that of the binding with `sync_sandbox`, through the handle of the sandbox's - /// directory. + /// `sync_payload`, through the handle of the leased generation's payload, and that of the + /// binding with `sync_sandbox`, through the handle of the sandbox's directory. fn release_with( &self, locked: &mut LockedStore, checked: &dyn Fn(), - sync: SyncDir<'_>, + sync_payload: &dyn Fn(&platform::Directory) -> io::Result<()>, sync_sandbox: &dyn Fn(&platform::Directory) -> io::Result<()>, ) -> Result> { let directory = locked.open_dir()?; @@ -425,6 +452,8 @@ impl Store { ))); } let recorded = self.check_records_dir(LEASES_NAME, LEASES_WHAT)?; + // The payload opens before anything goes, so that a link in its place refuses the release. + let payload = self.open_clone_payload(&lease)?; checked(); // A plan of the clone launches under the reservation that the lease's acquisition left in // the lock, which the release ends before it removes anything, so that a release that @@ -438,7 +467,9 @@ impl Store { // leased generation's payload, which must be durable before the lease ends, so that a // generation whose leases have drained holds no clone's socket. locked.remove_clone_files(&directory)?; - self.remove_clone_socket(&lease, sync)?; + if let Some(payload) = &payload { + self.remove_clone_socket(&lease, payload, sync_payload)?; + } let binding = locked.dir().join(CLONE_BINDING_NAME); directory .remove_file(CLONE_BINDING_NAME) @@ -455,26 +486,60 @@ impl Store { Ok(Some(lease)) } - /// Removes the control socket of the VMM of the clone that holds `lease`, beside the snapshot - /// that the VMM restored, if it is there, and makes its absence durable with `sync`. The VMM - /// removes its socket as it exits, and a stop or a recovery removes the one that an abrupt - /// exit leaves, but neither makes the removal durable. A payload that is gone holds no socket. - fn remove_clone_socket(&self, lease: &Lease, sync: SyncDir<'_>) -> Result<()> { - if !cfg!(unix) { - return Ok(()); - } - let payload = self.payload_dir(lease.generation()); - remove_if_present(&payload.join(clone_socket_name(lease.launch())))?; - match fs::symlink_metadata(&payload) { - Ok(_) => sync(&payload), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(store_error(format!( - "cannot examine the payload of the generation {}", + /// Refuses the payload of the generation `generation` unless it is a directory of the current + /// user with mode 0700, as a launch of a clone requires it, OpenVMM requiring that mode of the + /// directory of a control socket, and as a release of the clone's lease needs it on Linux: the + /// release opens the payload for reading, and removes the clone's socket from it. + fn check_payload(&self, generation: StoreGeneration) -> Result<()> { + let payload = self.payload_dir(generation); + let problem = platform::check_store_dir(&payload).map_err(store_error(format!( + "cannot open the payload of the generation {}", + payload.display() + )))?; + match problem { + Some(problem) => Err(Error::io(format!( + "the payload of the generation {} {problem}", payload.display() - ))(error)), + ))), + None => Ok(()), } } + /// Opens the payload of the generation that `lease` leases, which on Linux holds the control + /// socket of the clone's VMM, through the store's directory, following no link, as + /// [`open_payload`] opens it, or returns `None` elsewhere, where the VMM listens at a named + /// pipe, or if the payload is gone, or the store's directory holds another store, or none, + /// since neither holds a socket of this store's clones. + fn open_clone_payload(&self, lease: &Lease) -> Result> { + if !cfg!(unix) { + return Ok(None); + } + open_payload(self.dir(), self.id, lease.generation()) + } + + /// Removes the control socket of the VMM of the clone that holds `lease`, beside the snapshot + /// that the VMM restored, from `payload`, the handle of the leased generation's payload, if it + /// is there, and makes its absence durable with `sync`, through that handle. The VMM removes + /// its socket as it exits, and a stop or a recovery removes the one that an abrupt exit + /// leaves, but neither makes the removal durable. + fn remove_clone_socket( + &self, + lease: &Lease, + payload: &platform::Directory, + sync: &dyn Fn(&platform::Directory) -> io::Result<()>, + ) -> Result<()> { + payload + .remove_file(&clone_socket_name(lease.launch())) + .map_err(store_error(format!( + "cannot remove {}", + self.clone_socket_path(lease).display() + )))?; + sync(payload).map_err(store_error(format!( + "cannot sync the directory {}", + self.payload_dir(lease.generation()).display() + ))) + } + /// Returns the leases of the generation `generation` that the store's records list, in the /// order of their identities, including those that an interrupted release or removal of /// their sandboxes left, which a [retirement](Self::retire) removes. diff --git a/nvx-rs/src/host/template/store/leases/tests.rs b/nvx-rs/src/host/template/store/leases/tests.rs index 335a96618..eeee09695 100644 --- a/nvx-rs/src/host/template/store/leases/tests.rs +++ b/nvx-rs/src/host/template/store/leases/tests.rs @@ -23,6 +23,7 @@ use crate::host::store::{ }; use crate::host::supervisor::{self, Reattached, StopOutcome, VmmCommand}; use crate::host::template::eligibility::MAX_DOWNTIME_NS; +use crate::host::template::store::SNAPSHOT_NAME; use crate::host::template::store::tests::{ cause, link_dir, link_file, publish, publish_with, remove, storage, store as new_store, unlink_dir, @@ -284,6 +285,29 @@ fn acquisitions_refuse_what_they_cannot_lease_and_lease_nothing() { assert_eq!(error.code(), ErrorCode::TemplateIneligible, "{error}"); assert!(error.message().contains(condition), "{error}"); } + // On Unix, a payload that is not a directory of the current user with mode 0700, which a + // launch of the clone requires, and through which a release ends the lease: it opens the + // payload for reading, and removes the clone's socket from it. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let payload = store.payload_dir(generation); + for mode in [0o300, 0o500] { + fs::set_permissions(&payload, fs::Permissions::from_mode(mode)).unwrap(); + let error = refusal(&store, generation, &sandbox, &now); + fs::set_permissions(&payload, fs::Permissions::from_mode(0o700)).unwrap(); + assert_eq!(error.kind(), ErrorKind::Io, "{error}"); + assert_eq!( + error.message(), + format!( + "the payload of the generation {} has mode {mode:o}, not the mode 700 that \ + the directory of a control socket must have", + payload.display() + ) + ); + } + } // A sandbox that is not provisioned: its directory is missing, or holds no configuration. let unprovisioned = directory.path().join("unprovisioned"); drop(SandboxStore::lock_for_provision(&unprovisioned, &files(), None).unwrap()); @@ -808,8 +832,11 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco .borrow_mut() .push((dir.to_owned(), clone, bound, record.exists())); }; - let sync = |dir: &Path| -> Result<()> { - note(dir); + // The release syncs the payload through its handle, which holds the generation's snapshot. + let snapshot = platform::file_id(&payload.join(SNAPSHOT_NAME)).unwrap(); + let sync_payload = |directory: &platform::Directory| -> io::Result<()> { + assert_eq!(directory.entry_id(SNAPSHOT_NAME).unwrap(), Some(snapshot)); + note(&payload); Ok(()) }; // The release syncs the sandbox's directory through its handle, which holds the lock's file. @@ -829,7 +856,7 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); assert_eq!( store - .release_with(&mut locked, &|| {}, &sync, &sync_sandbox) + .release_with(&mut locked, &|| {}, &sync_payload, &sync_sandbox) .unwrap(), Some(lease) ); @@ -866,17 +893,15 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco .release_with( &mut locked, &|| {}, - &|dir: &Path| { - if dir == payload { - Err(Error::io("the disk failed")) - } else { - Ok(()) - } - }, + &|_| Err(io::Error::other("the disk failed")), &|_| Ok(()), ) .unwrap_err(); - assert_eq!(error.message(), "the disk failed"); + assert_eq!( + error.message(), + format!("cannot sync the directory {}", payload.display()) + ); + assert_eq!(cause(&error), "the disk failed"); assert_eq!(binding(&sandbox), Some(again.clone())); assert_eq!(locked.reserved(), None); assert!(fs::symlink_metadata(&scratch).is_err()); @@ -884,7 +909,7 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco // A release that cannot make the binding's removal durable keeps the record, which a // retirement removes once no binding names it. let error = store - .release_with(&mut locked, &|| {}, &|_: &Path| Ok(()), &|_| { + .release_with(&mut locked, &|| {}, &|_| Ok(()), &|_| { Err(io::Error::other("the disk failed")) }) .unwrap_err(); @@ -1245,7 +1270,7 @@ fn a_release_ends_the_lease_of_the_sandbox_whose_lock_it_holds_whatever_its_path .release_with( &mut locked, &|| swap(&first, &moved), - &sync_dir, + &platform::Directory::sync, &platform::Directory::sync ) .unwrap(), @@ -1292,6 +1317,294 @@ fn a_release_ends_the_lease_of_the_sandbox_whose_lock_it_holds_whatever_its_path assert_eq!(store.leases(generation).unwrap(), both); } +/// The payload of a generation, which holds the control sockets of the VMMs of its clones on +/// Linux, opens through the store's directory, following no link, or junction on Windows, at the +/// store's directory, at its directory of the generations, or at the payload, once it has found +/// that the store's directory holds the store of the caller's identity. On Unix, a store's +/// directory and its directory of the generations need only the right to be searched, which an +/// owner-only directory may grant without the right to read it. +#[test] +fn the_payload_of_a_generation_opens_through_no_link() { + let (directory, store) = new_store(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let id = store.id(); + let payload = store.payload_dir(generation); + let snapshot = platform::file_id(&payload.join(SNAPSHOT_NAME)).unwrap(); + let opened = open_payload(store.dir(), id, generation).unwrap().unwrap(); + assert_eq!(opened.entry_id(SNAPSHOT_NAME).unwrap(), Some(snapshot)); + drop(opened); + // A generation without a payload has none, nor has a store that is gone, or one of another + // identity. + assert!( + open_payload(store.dir(), id, store_generation(9)) + .unwrap() + .is_none() + ); + let missing = directory.path().join("missing"); + assert!(open_payload(&missing, id, generation).unwrap().is_none()); + let foreign = StoreId::from_bytes([0x5a; 16]); + assert!( + open_payload(store.dir(), foreign, generation) + .unwrap() + .is_none() + ); + // Another store in the store's place, whose generation of the same number has a payload, has + // none of the store's, nor has a directory that holds no store. + let other = Store::create(directory.path().join("other")).unwrap(); + fs::create_dir(other.payload_dir(generation)).unwrap(); + let moved = directory.path().join("moved"); + fs::rename(store.dir(), &moved).unwrap(); + fs::rename(other.dir(), store.dir()).unwrap(); + assert!(open_payload(store.dir(), id, generation).unwrap().is_none()); + assert!( + open_payload(store.dir(), other.id(), generation) + .unwrap() + .is_some() + ); + fs::rename(store.dir(), other.dir()).unwrap(); + fs::create_dir(store.dir()).unwrap(); + assert!(open_payload(store.dir(), id, generation).unwrap().is_none()); + fs::remove_dir(store.dir()).unwrap(); + // A link in the place of the store's directory, even to the store, of the directory of the + // generations, or of the payload, and a file in the place of the store's directory or of the + // payload, are refused. + let generations = store.dir().join(GENERATIONS_NAME); + let refused = |entry: &dyn std::fmt::Display| { + let error = open_payload(store.dir(), id, generation).err().unwrap(); + assert_eq!(error.kind(), ErrorKind::Io, "{error}"); + assert_eq!( + error.message(), + format!( + "cannot open the payload of the generation {}", + payload.display() + ) + ); + assert_eq!( + cause(&error), + format!("{entry} is a link, or not a directory") + ); + }; + link_dir(&moved, store.dir()); + refused(&store.dir().display()); + unlink_dir(store.dir()); + fs::write(store.dir(), b"").unwrap(); + refused(&store.dir().display()); + fs::remove_file(store.dir()).unwrap(); + fs::rename(&moved, store.dir()).unwrap(); + for (path, name, link) in [ + (&payload, generation.to_string(), true), + (&generations, GENERATIONS_NAME.to_owned(), true), + (&payload, generation.to_string(), false), + ] { + fs::rename(path, &moved).unwrap(); + if link { + link_dir(&moved, path); + } else { + fs::write(path, b"").unwrap(); + } + refused(&name); + if link { + unlink_dir(path); + } else { + fs::remove_file(path).unwrap(); + } + fs::rename(&moved, path).unwrap(); + } + assert!(open_payload(store.dir(), id, generation).unwrap().is_some()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mode = |dir: &Path, mode| { + fs::set_permissions(dir, fs::Permissions::from_mode(mode)).unwrap(); + }; + for dir in [&generations, store.dir()] { + mode(dir, 0o300); + } + let opened = open_payload(store.dir(), id, generation); + for dir in [&generations, store.dir()] { + mode(dir, 0o700); + } + let opened = opened.unwrap().unwrap(); + assert_eq!(opened.entry_id(SNAPSHOT_NAME).unwrap(), Some(snapshot)); + opened.sync().unwrap(); + } +} + +/// A release whose generation's payload, or store's directory, a link takes the place of: before +/// the release begins, it refuses, and removes nothing; once it has opened the payload, it removes +/// the socket from the payload that it opened, and the link's target keeps its file. Only on Unix, +/// where the payload holds the socket of the clone's VMM. +#[cfg(unix)] +#[test] +fn a_release_removes_the_socket_from_the_payload_that_it_opened_whatever_its_path_names_meanwhile() +{ + let (directory, store) = new_store(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let sandbox = provision(directory.path(), "sandbox"); + let lease = leased(&store, &published, &sandbox); + let payload = store.payload_dir(generation); + let name = clone_socket_name(lease.launch()); + // A socket that the VMM left in the payload, which a file stands for, and a directory that + // holds a file of its name, to which a link takes the payload's place. + fs::write(payload.join(&name), b"").unwrap(); + let decoy = directory.path().join("decoy"); + fs::create_dir(&decoy).unwrap(); + fs::write(decoy.join(&name), b"").unwrap(); + // A directory of the store's layout, whose generation of the same number holds a file of the + // socket's name, to which a link takes the store's place. + let decoy_store = directory.path().join("decoy-store"); + let decoy_socket = decoy_store + .join(GENERATIONS_NAME) + .join(generation.to_string()) + .join(&name); + fs::create_dir_all(decoy_socket.parent().unwrap()).unwrap(); + fs::write(&decoy_socket, b"").unwrap(); + let moved = directory.path().join("moved"); + let swap = || { + fs::rename(&payload, &moved).unwrap(); + link_dir(&decoy, &payload); + }; + let restore = || { + unlink_dir(&payload); + fs::rename(&moved, &payload).unwrap(); + }; + let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); + let refused = |locked: &mut LockedStore, entry: &dyn std::fmt::Display| { + let error = store.release(locked).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io, "{error}"); + assert_eq!( + error.message(), + format!( + "cannot open the payload of the generation {}", + payload.display() + ) + ); + assert_eq!( + cause(&error), + format!("{entry} is a link, or not a directory") + ); + assert!(decoy.join(&name).exists()); + assert!(decoy_socket.exists()); + assert_eq!(binding(&sandbox), Some(lease.clone())); + }; + swap(); + refused(&mut locked, &generation); + assert!(moved.join(&name).exists()); + restore(); + fs::rename(store.dir(), &moved).unwrap(); + link_dir(&decoy_store, store.dir()); + refused(&mut locked, &store.dir().display()); + unlink_dir(store.dir()); + fs::rename(&moved, store.dir()).unwrap(); + assert!(payload.join(&name).exists()); + assert_eq!( + store + .release_with( + &mut locked, + &swap, + &platform::Directory::sync, + &platform::Directory::sync + ) + .unwrap(), + Some(lease) + ); + drop(locked); + assert!(fs::symlink_metadata(moved.join(&name)).is_err()); + assert!(decoy.join(&name).exists()); + assert!(decoy_socket.exists()); + assert_eq!(binding(&sandbox), None); + restore(); + assert!(store.leases(generation).unwrap().is_empty()); +} + +/// A sandbox's store that clears the runtime record of its clone's VM removes the VMM's control +/// socket through the payload of its generation, as a recovery of its launch does: it refuses a +/// link in the place of the payload or of the template store's directory, keeping the record, and +/// leaves the files of another store in the template store's place. Only on Unix, where the socket +/// is a file. +#[cfg(unix)] +#[test] +fn a_sandbox_s_store_removes_the_socket_of_its_clone_through_no_link() { + let (directory, store) = new_store(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let sandbox = provision(directory.path(), "sandbox"); + let lease = leased(&store, &published, &sandbox); + let payload = store.payload_dir(generation); + let socket = store.clone_socket_path(&lease); + let name = clone_socket_name(lease.launch()); + fs::write(&socket, b"").unwrap(); + let decoy = directory.path().join("decoy"); + fs::create_dir(&decoy).unwrap(); + fs::write(decoy.join(&name), b"").unwrap(); + // Another store, whose generation of the same number holds a file of the socket's name. + let other = Store::create(directory.path().join("other")).unwrap(); + fs::create_dir(other.payload_dir(generation)).unwrap(); + let other_socket = other.payload_dir(generation).join(&name); + fs::write(&other_socket, b"").unwrap(); + // The runtime record of the clone's VM, whose control endpoint is the socket. + let record = json!({ + "format": FORMAT, + "launch": lease.launch().to_string(), + "process": { "pid": 42, "startTime": 4242, "boot": "ab".repeat(16) }, + "endpoint": socket.to_str().unwrap(), + "runtime": serde_json::to_value(runtime()).unwrap(), + }); + let write_record = || { + write_private(&sandbox.join(RUNTIME_NAME), record.to_string().as_bytes()).unwrap(); + }; + write_record(); + let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); + let refused = |locked: &mut LockedStore, entry: &dyn std::fmt::Display| { + let error = locked.clear_runtime().unwrap_err(); + assert_eq!( + error.message(), + format!( + "cannot open the payload of the generation {}", + payload.display() + ) + ); + assert_eq!( + cause(&error), + format!("{entry} is a link, or not a directory") + ); + assert!(decoy.join(&name).exists()); + assert!(other_socket.exists()); + assert!(sandbox.join(RUNTIME_NAME).exists()); + }; + let moved = directory.path().join("moved"); + fs::rename(&payload, &moved).unwrap(); + link_dir(&decoy, &payload); + refused(&mut locked, &generation); + assert!(moved.join(&name).exists()); + unlink_dir(&payload); + fs::rename(&moved, &payload).unwrap(); + fs::rename(store.dir(), &moved).unwrap(); + link_dir(other.dir(), store.dir()); + refused(&mut locked, &store.dir().display()); + unlink_dir(store.dir()); + // Another store in the store's place holds no socket of the clone's, so the record goes, and + // the other store's file stays, as does the socket in the store's own directory. + fs::rename(other.dir(), store.dir()).unwrap(); + locked.clear_runtime().unwrap(); + assert!(!sandbox.join(RUNTIME_NAME).exists()); + fs::rename(store.dir(), other.dir()).unwrap(); + fs::rename(&moved, store.dir()).unwrap(); + assert!(other_socket.exists()); + assert!(socket.exists()); + write_record(); + locked.clear_runtime().unwrap(); + assert!(fs::symlink_metadata(&socket).is_err()); + assert!(!sandbox.join(RUNTIME_NAME).exists()); + assert!(decoy.join(&name).exists()); + assert!(other_socket.exists()); + drop(locked); + assert_eq!(release(&store, &sandbox).unwrap(), Some(lease)); +} + #[test] fn a_retirement_that_finds_its_generation_unindexed_makes_the_index_durable_first() { let (_directory, store) = new_store(); @@ -1938,7 +2251,9 @@ fn clone_role() { if step == "releasing" { let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); store - .release_with(&mut locked, &|| {}, &sync, &|_| crash(&step)) + .release_with(&mut locked, &|| {}, &platform::Directory::sync, &|_| { + crash(&step) + }) .unwrap(); panic!("the release did not make the removal of its binding durable"); }