nvx-rs: reserve the launch of a leased clone, and document the lock order

aci_edge_sandboxes/tests/openvmm_fake.rs

@@ -17,7 +17,9 @@ use std::time::{Duration, Instant};
 use aci_edge_sandboxes::nvx::ErrorCode as NvxCode;
 use aci_edge_sandboxes::nvx::Hypervisor;
 use aci_edge_sandboxes::nvx::host::platform;
-use aci_edge_sandboxes::nvx::host::store::{ArtifactIdentity, RuntimeIdentity, SandboxStore};
+use aci_edge_sandboxes::nvx::host::store::{
+    ArtifactIdentity, LaunchId, RuntimeIdentity, SandboxStore,
+};
 use aci_edge_sandboxes::nvx::host::supervisor::{self, Launched, VmmCommand};
 use aci_edge_sandboxes::nvx::protocol::CAPABILITY_LEN;
 use aci_edge_sandboxes::openvmm::OpenVmmConfig;
@@ -1181,7 +1183,11 @@ fn interrupted_starts_before_an_endpoint_recover_or_retain_the_child() {
         let store = SandboxStore::open(&dir).unwrap();
         let mut locked = store.lock(None).unwrap();
         let mut launch = locked
-            .begin_launch(&runtime_with('1'), &CAPABILITY)
+            .begin_launch(
+                &LaunchId::generate().unwrap(),
+                &runtime_with('1'),
+                &CAPABILITY,
+            )
             .unwrap();
         let listen = arguments
             .iter()
@@ -1254,7 +1260,11 @@ fn unrecorded_launches_end_once_their_log_is_released() {
     let store = SandboxStore::open(&dir).unwrap();
     let mut locked = store.lock(None).unwrap();
     let mut launch = locked
-        .begin_launch(&runtime_with('1'), &CAPABILITY)
+        .begin_launch(
+            &LaunchId::generate().unwrap(),
+            &runtime_with('1'),
+            &CAPABILITY,
+        )
         .unwrap();
     let log = launch.take_log().unwrap();
     drop(launch);

doc/usage.md

@@ -109,7 +109,7 @@ each code for callers of the `nvx` library.
 | `foreign-files` | 1 | `deprovision` found files that NVX does not own in the state directory, which the message names, and kept them with the directory. |
 | `outcome-report-exists` | 1 | A file exists at the `--outcome-report` path. |
 | `scratch-claimed` | 1 | Another sandbox claims the scratch image. |
-| `sandbox-leased` | 1 | The sandbox holds a lease of a template's generation already, which a release ends once its VM has exited. |
+| `sandbox-leased` | 1 | The sandbox holds a lease of a template's generation, which allows no other lease, and one launch of its clone, until a release ends it once its VM has exited. |
 | `template-ineligible` | 1 | The restore window of a template's generation, which the time contract of its capture sets, excludes the host's time, so no clone may restore it now. |
 | `template-tombstoned` | 1 | A tombstone quarantines the template's generation, which takes no new clones. |
 | `exited` | 1 | OpenVMM exited, as when a start fails, whose message names the category and status of OpenVMM's outcome report when it has one. |

nvx-rs/src/error.rs

@@ -154,8 +154,8 @@ pub enum ErrorCode {
     /// `template-tombstoned`: a tombstone quarantines the template's generation, which takes no
     /// new clones, while the clones that lease it keep their leases.
     TemplateTombstoned,
-    /// `sandbox-leased`: the sandbox holds a lease of a template's generation already, which a
-    /// release ends once its VM has exited.
+    /// `sandbox-leased`: the sandbox holds a lease of a template's generation, which allows no
+    /// other lease, and one launch of its clone, until a release ends it once its VM has exited.
     SandboxLeased,
 }
 

nvx-rs/src/host/cli/sandbox/lifecycle.rs

@@ -75,7 +75,7 @@ use crate::host::sandbox::distro::DISTRO_POLICY_REVISION;
 use crate::host::sandbox::scratch::CLAIM_RELEASE_WAIT;
 use crate::host::sandbox::{PreparedLaunch, SandboxLaunch, require_file};
 use crate::host::store::{
-    IdentityPart, LockedStore, OUTCOME_NAME, RUNTIME_NAME, Recovery, SandboxStore,
+    IdentityPart, LaunchId, LockedStore, OUTCOME_NAME, RUNTIME_NAME, Recovery, SandboxStore,
 };
 use crate::host::supervisor::{self, Launched, Reattached, StopOutcome, Vmm, VmmCommand};
 use crate::protocol::{CAPABILITY_LEN, GuestFeatures};
@@ -581,7 +581,8 @@ fn start_locked(
     ];
     let directory = std::env::current_dir().map_err(Exit::from_os_error)?;
     let mut command = Vec::new();
-    let mut starting = supervisor::launch(locked, &runtime, &capability, |endpoint| {
+    let launch = LaunchId::generate().map_err(error_with_cause)?;
+    let mut starting = supervisor::launch(locked, &launch, &runtime, &capability, |endpoint| {
         command = head;
         command.push(format!("listen={endpoint}"));
         command.extend(tail);

nvx-rs/src/host/instance.rs

@@ -68,8 +68,8 @@ use super::process::KILL_TIMEOUT;
 use super::report::{self, Report};
 use super::session::{self, ControlSession, ExecEvent, SessionError};
 use super::store::{
-    CallerFiles, IdentityPart, LockedStore, OUTCOME_NAME, Recovery, RemovalError, RuntimeIdentity,
-    RuntimeRecord, SOCKET_NAME, SandboxStore,
+    CallerFiles, IdentityPart, LaunchId, LockedStore, OUTCOME_NAME, Recovery, RemovalError,
+    RuntimeIdentity, RuntimeRecord, SOCKET_NAME, SandboxStore,
 };
 use super::supervisor::{self, Reattached, Starting, StopOutcome, Vmm, VmmCommand};
 use crate::error::{Error, ErrorCode, ErrorKind, Result};
@@ -706,14 +706,21 @@ impl LockedInstance {
         let endpoint = control_endpoint(&instance.store.socket_path())?;
         spec.fit_process_command_line(&spec.openvmm_arguments(&endpoint, &report)?, 0)?;
         let capability = generate_capability()?;
+        let launch = LaunchId::generate()?;
         let started = Instant::now();
-        let starting = supervisor::launch(&mut self.locked, runtime, &capability, |endpoint| {
-            let arguments = spec.openvmm_arguments(endpoint, &report)?;
-            spec.fit_process_command_line(&arguments, 0)?;
-            let mut command = VmmCommand::new(&spec.openvmm, arguments, instance.dir());
-            command.breakaway_from_job = spec.breakaway_from_job;
-            Ok(command)
-        })?;
+        let starting = supervisor::launch(
+            &mut self.locked,
+            &launch,
+            runtime,
+            &capability,
+            |endpoint| {
+                let arguments = spec.openvmm_arguments(endpoint, &report)?;
+                spec.fit_process_command_line(&arguments, 0)?;
+                let mut command = VmmCommand::new(&spec.openvmm, arguments, instance.dir());
+                command.breakaway_from_job = spec.breakaway_from_job;
+                Ok(command)
+            },
+        )?;
         let (pid, start_time) = {
             let reference = starting.process().reference();
             (reference.pid, reference.start_time)

nvx-rs/src/host/store/mod.rs

@@ -49,10 +49,13 @@
 //!
 //! # Launches
 //!
-//! A launch holds the sandbox's lock throughout:
+//! A launch holds the sandbox's lock throughout, under an identity that the caller reserved
+//! before it began, with [`LaunchId::generate`], a fresh one for each launch, or that a clone's
+//! lease reserved, which records it durably before any VMM of the clone exists, for the one
+//! launch of the clone that the lease allows:
 //!
 //! 1. [`LockedStore::begin_launch`] stores the capability, creates a fresh log and claims it, and
-//!    writes the journal's marker, before anything is spawned.
+//!    writes the journal's marker, which names the launch, before anything is spawned.
 //! 2. The caller spawns the VMM with [`Launch::take_log`] as its standard output and error, and
 //!    [`Launch::record_process`] records the VMM's [`ProcessRef`] right after the spawn.
 //! 3. Once the VM is ready, [`Launch::commit`] replaces the journal with the runtime record. A
@@ -65,7 +68,7 @@
 //! use std::time::{Duration, Instant};
 //!
 //! use nvx::host::launch::LaunchSpec;
-//! use nvx::host::store::{RuntimeIdentity, SandboxStore};
+//! use nvx::host::store::{LaunchId, RuntimeIdentity, SandboxStore};
 //! use nvx::host::supervisor::{self, VmmCommand};
 //! use nvx::protocol::CAPABILITY_LEN;
 //!
@@ -77,7 +80,7 @@
 //!     let store = SandboxStore::create("/var/lib/example/sandbox")?;
 //!     let mut locked = store.lock(Some(Instant::now() + Duration::from_secs(10)))?;
 //!     locked.recover()?;
-//!     let mut launch = locked.begin_launch(runtime, capability)?;
+//!     let mut launch = locked.begin_launch(&LaunchId::generate()?, runtime, capability)?;
 //!     let arguments = spec.openvmm_arguments(launch.endpoint(), &store.outcome_path())?;
 //!     let command = VmmCommand::new(&spec.openvmm, arguments, store.dir());
 //!     let log = launch.take_log()?;
@@ -94,15 +97,15 @@
 //! transition of the sandbox overlaps it, not even through the same guard:
 //!
 //! ```compile_fail
-//! # use nvx::host::store::{RuntimeIdentity, SandboxStore};
+//! # use nvx::host::store::{LaunchId, RuntimeIdentity, SandboxStore};
 //! # use nvx::protocol::CAPABILITY_LEN;
 //! # fn overlap(
 //! #     store: &SandboxStore,
 //! #     runtime: &RuntimeIdentity,
 //! #     capability: &[u8; CAPABILITY_LEN],
 //! # ) -> nvx::Result<()> {
 //! let mut locked = store.lock(None)?;
-//! let launch = locked.begin_launch(runtime, capability)?;
+//! let launch = locked.begin_launch(&LaunchId::generate()?, runtime, capability)?;
 //! locked.recover()?; // `locked` is still borrowed by `launch`.
 //! launch.commit()?;
 //! # Ok(())
@@ -246,6 +249,7 @@ use super::filesystem::FileIdentity;
 use super::platform;
 use super::process::{BootId, Process, ProcessRef};
 use super::supervisor::Launched;
+use super::template::Lease;
 use crate::error::{Error, ErrorCode, ErrorKind, Result};
 use crate::protocol::CAPABILITY_LEN;
 
@@ -623,6 +627,9 @@ pub struct LockedStore {
     files: CallerFiles,
     /// Whether this holder removed its lock file, after which the path may name another's.
     lock_removed: bool,
+    /// The launch of the sandbox's clone that the lease acquired under this lock reserves, until
+    /// [`begin_launch`](Self::begin_launch) begins it, or a release ends the lease.
+    reserved: Option<LaunchId>,
 }
 
 impl Deref for LockedStore {
@@ -642,29 +649,44 @@ impl LockedStore {
             lock,
             files,
             lock_removed: false,
+            reserved: None,
         }
     }
 
+    /// Lets [`begin_launch`](Self::begin_launch) begin `launch`, once, as the launch of the
+    /// sandbox's clone that the lease acquired under this lock reserves, or, given `None`, no
+    /// launch of a clone, once a release ends the lease.
+    pub(crate) fn reserve(&mut self, launch: Option<LaunchId>) {
+        self.reserved = launch;
+    }
+
     /// Returns the caller's files that the lock knows: those that a [lifecycle
     /// lock](self#lifecycle-locks) was taken with, and none for [`SandboxStore::lock`].
     pub fn files(&self) -> &CallerFiles {
         &self.files
     }
 
-    /// Begins a launch of the runtime `runtime`, whose VMM receives `capability`.
+    /// Begins the launch `launch`, which the caller reserved with [`LaunchId::generate`], of the
+    /// runtime `runtime`, whose VMM receives `capability`.
     ///
     /// It stores the capability, replaces the log of the previous launch with a fresh one, which
     /// it claims, removes the outcome report, console log, and control socket of the previous
     /// launch, and any outcome report that its VMM staged without publishing it, chooses a control
-    /// endpoint, and writes the journal's marker. It keeps the console log of a caller that keeps
-    /// the console of every launch, as [`CallerFiles::keeping_console`] declares. Nothing is
-    /// spawned yet: the caller spawns the VMM with [`Launch::take_log`].
+    /// endpoint, and writes the journal's marker, which names the launch. It keeps the console log
+    /// of a caller that keeps the console of every launch, as [`CallerFiles::keeping_console`]
+    /// declares. Nothing is spawned yet: the caller spawns the VMM with [`Launch::take_log`].
     ///
     /// Fails if the sandbox's VM runs, or a launch was interrupted and is not recovered yet, and
-    /// with [`ErrorKind::Unsupported`] for an all-zero capability
-    /// or an invalid runtime identity.
+    /// with [`ErrorKind::Unsupported`] for an all-zero capability or an invalid runtime identity.
+    /// A sandbox whose clone holds a lease of a template's generation, which its
+    /// [clone binding](CLONE_BINDING_NAME) records, launches once, as the launch that the lease
+    /// reserves, under the lock that acquired the lease, as
+    /// [`template::Store::acquire`](crate::host::template::Store::acquire) returns it: any other
+    /// launch, and any launch under another lock or after that one began, fails with
+    /// [`ErrorCode::SandboxLeased`] until a release ends the lease.
     pub fn begin_launch(
         &mut self,
+        launch: &LaunchId,
         runtime: &RuntimeIdentity,
         capability: &[u8; CAPABILITY_LEN],
     ) -> Result<Launch<'_>> {
@@ -686,6 +708,26 @@ impl LockedStore {
                 self.dir.display()
             )));
         }
+        if let Some(lease) = Lease::of(self)?
+            && (lease.launch() != launch || self.reserved.as_ref() != Some(launch))
+        {
+            let reservation = if lease.launch() == launch {
+                format!("whose launch {launch} begins once, under the lock that acquired the lease")
+            } else {
+                format!("which reserves the launch {}, not {launch}", lease.launch())
+            };
+            return Err(Error::new(
+                ErrorCode::SandboxLeased,
+                format!(
+                    "the sandbox in {} holds the lease {} of the generation {} of the template \
+                     store {}, {reservation}",
+                    self.dir.display(),
+                    lease.id(),
+                    lease.generation(),
+                    lease.store()
+                ),
+            ));
+        }
         let socket = self.socket_path();
         for path in [&socket, &self.outcome_path()] {
             remove_if_present(path)?;
@@ -698,12 +740,13 @@ impl LockedStore {
             .map_err(|error| Error::io("cannot choose a control endpoint").with_source(error))?;
         let (log, claim) = self.create_log()?;
         files::write_private(&self.capability_path(), capability)?;
-        let journal =
-            LaunchJournal::prepared(LaunchId::generate()?, endpoint, runtime.clone(), claim);
+        let journal = LaunchJournal::prepared(launch.clone(), endpoint, runtime.clone(), claim);
         if let Err(error) = record::write(&self.path(JOURNAL_NAME), &journal) {
             let _ = remove_if_present(&self.capability_path());
             return Err(error);
         }
+        // A lease reserves one launch, which has begun.
+        self.reserved = None;
         Ok(Launch {
             locked: self,
             journal,

nvx-rs/src/host/store/record.rs

@@ -36,15 +36,20 @@ fn is_hex_id(text: &str) -> bool {
             .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
 }
 
-/// Identity of one launch: 32 lowercase hexadecimal digits, the 16 random bytes that the store
-/// draws for every launch, and which its journal and runtime record share.
+/// Identity of one launch: 32 lowercase hexadecimal digits, the 16 random bytes that
+/// [`generate`](Self::generate) draws for every launch, and which its journal and runtime record
+/// share.
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(try_from = "String", into = "String")]
 pub struct LaunchId(String);
 
 impl LaunchId {
-    /// Draws a fresh launch identity.
-    pub(super) fn generate() -> Result<Self> {
+    /// Draws a fresh launch identity, which a caller reserves for one launch before it begins it
+    /// with [`LockedStore::begin_launch`](super::LockedStore::begin_launch), or
+    /// [`supervisor::launch`](crate::host::supervisor::launch), and never uses for another. A
+    /// clone's lease reserves one, as
+    /// [`template::Store::acquire`](crate::host::template::Store::acquire) draws it.
+    pub fn generate() -> Result<Self> {
         let mut bytes = [0u8; LAUNCH_ID_BYTES];
         getrandom::fill(&mut bytes)
             .map_err(|error| Error::other("cannot draw a launch identity").with_source(error))?;