nvx-rs: attribute a clone's failures, record them, and reattach it

nvx-rs/src/host/platform/linux.rs

@@ -784,6 +784,17 @@ impl Directory {
     pub(crate) fn sync(&self) -> io::Result<()> {
         self.0.sync_all()
     }
+
+    /// Returns why the directory cannot hold a sandbox store, as [`check_store_dir`] returns it for
+    /// a path, if it cannot: unless it belongs to the current user with mode 0700.
+    pub(crate) fn store_dir_problem(&self) -> io::Result<Option<String>> {
+        let metadata = self.0.metadata()?;
+        Ok(store_dir_problem(
+            metadata.uid(),
+            metadata.mode(),
+            sys::effective_ids().0,
+        ))
+    }
 }
 
 /// Claims a launch's fresh log, which `file` holds open.
@@ -868,7 +879,8 @@ pub(crate) struct ReadyListener {
 
 /// Listens at the Unix socket `socket_path`, in place of a socket that an earlier launch left
 /// there, as the restore-ready endpoint of a clone's launch, and returns the endpoint, as
-/// OpenVMM's `--restore-ready-path` takes it, and its listener.
+/// OpenVMM's `--restore-ready-path` takes it, and its listener. The socket has mode 0600,
+/// whatever the umask, as [`set_ready_socket_mode`] gives it.
 pub(crate) fn listen_ready(socket_path: &Path) -> io::Result<(String, ReadyListener)> {
     let endpoint = socket_path
         .to_str()
@@ -879,6 +891,10 @@ pub(crate) fn listen_ready(socket_path: &Path) -> io::Result<(String, ReadyListe
         _ => {}
     }
     let socket = UnixListener::bind(socket_path)?;
+    // The umask may clear bits of the mode that the bind gives the socket, even its owner's
+    // permission to write it, without which a VMM that is not root cannot connect: the mode is
+    // set again before any VMM may connect.
+    set_ready_socket_mode(socket_path)?;
     // The host polls the socket before it accepts, so an accept never blocks.
     socket.set_nonblocking(true)?;
     let listener = ReadyListener {
@@ -965,6 +981,35 @@ impl ReadyListener {
     }
 }
 
+/// Gives the restore-ready socket at `socket_path`, which the caller bound, the mode 0600, whatever
+/// the umask: its owner, the user of the clone's VMM, may connect to it, which takes the
+/// permission to write it, and no one else may.
+///
+/// The socket is opened only to locate it, following no link at its name, and refused unless it
+/// is a socket.
+fn set_ready_socket_mode(socket_path: &Path) -> io::Result<()> {
+    use std::os::unix::fs::FileTypeExt;
+
+    use nix::fcntl::OFlag;
+
+    let socket = OpenOptions::new()
+        .read(true)
+        .custom_flags((OFlag::O_PATH | OFlag::O_NOFOLLOW).bits())
+        .open(socket_path)?;
+    if !socket.metadata()?.file_type().is_socket() {
+        return Err(io::Error::other(format!(
+            "{} is not the restore-ready socket that the launch bound",
+            socket_path.display()
+        )));
+    }
+    // A descriptor that only locates the socket cannot change its mode itself, but its link in
+    // /proc names exactly the socket that it opened.
+    fs::set_permissions(
+        Path::new("/proc/self/fd").join(socket.as_raw_fd().to_string()),
+        Permissions::from_mode(0o600),
+    )
+}
+
 /// Connects to a control endpoint served by the process `expected_pid`.
 ///
 /// The connection attempt never blocks: a listener whose backlog is full yields
@@ -1110,4 +1155,65 @@ mod tests {
         assert!(elapsed < Duration::from_secs(1), "{elapsed:?}");
         assert!(process.wait(Instant::now()).unwrap().unwrap());
     }
+
+    #[test]
+    fn a_restore_ready_socket_lets_its_owner_connect_whatever_the_umask() {
+        let directory = tempfile::tempdir().unwrap();
+        let path = directory.path().join("ready.sock");
+        // A umask that clears even the owner's permission to write what it creates, in a thread
+        // of the test's own, whose umask no other thread shares.
+        let (endpoint, _listener) = thread::scope(|scope| {
+            scope
+                .spawn(|| {
+                    nix::sched::unshare(nix::sched::CloneFlags::CLONE_FS).unwrap();
+                    nix::sys::stat::umask(nix::sys::stat::Mode::from_bits_truncate(0o277));
+                    listen_ready(&path)
+                })
+                .join()
+                .unwrap()
+        })
+        .unwrap();
+        let mode = fs::symlink_metadata(&path).unwrap().permissions().mode();
+        assert_eq!(mode & PERMISSION_BITS, 0o600, "{mode:o}");
+        UnixStream::connect(&endpoint).unwrap();
+    }
+
+    #[test]
+    fn a_restore_ready_client_that_closed_before_its_connection_was_accepted_is_read_to_its_end() {
+        let pid = std::process::id();
+        let receive_all = |listener: &mut ReadyListener| {
+            let mut received = Vec::new();
+            let deadline = Instant::now() + Duration::from_secs(10);
+            while listener
+                .receive(pid, &mut received, 64, Duration::from_millis(50))
+                .unwrap()
+            {
+                assert!(Instant::now() < deadline, "the endpoint did not close");
+            }
+            received
+        };
+        let directory = tempfile::tempdir().unwrap();
+        let path = directory.path().join("ready.sock");
+        // A client that sent nothing, as a VMM that fails as it starts, and one that sent its
+        // event, each closed before the listener accepted their connection: what they sent
+        // remains to read, and then the end of the connection.
+        for sent in [&b""[..], b"the event\n"] {
+            let (endpoint, mut listener) = listen_ready(&path).unwrap();
+            let mut client = UnixStream::connect(&endpoint).unwrap();
+            std::io::Write::write_all(&mut client, sent).unwrap();
+            drop(client);
+            assert_eq!(receive_all(&mut listener), sent);
+        }
+        // A client that closes once the listener accepted its connection.
+        let (endpoint, mut listener) = listen_ready(&path).unwrap();
+        let client = UnixStream::connect(&endpoint).unwrap();
+        let mut received = Vec::new();
+        assert!(
+            listener
+                .receive(pid, &mut received, 64, Duration::from_millis(50))
+                .unwrap()
+        );
+        drop(client);
+        assert_eq!(receive_all(&mut listener), b"");
+    }
 }

nvx-rs/src/host/platform/windows.rs

@@ -2137,7 +2137,7 @@ pub(crate) fn interrupted() -> bool {
 
 #[cfg(test)]
 mod tests {
-    use std::io::Read;
+    use std::io::{Read, Write};
 
     use super::*;
 
@@ -2165,6 +2165,43 @@ mod tests {
         end_process(false, 7, || panic!("terminated"), || Ok(true)).unwrap();
     }
 
+    #[test]
+    fn a_restore_ready_client_that_closed_before_its_connection_was_accepted_is_read_to_its_end() {
+        let pid = std::process::id();
+        let receive_all = |listener: &mut ReadyListener| {
+            let mut received = Vec::new();
+            let deadline = Instant::now() + Duration::from_secs(10);
+            while listener
+                .receive(pid, &mut received, 64, Duration::from_millis(50))
+                .unwrap()
+            {
+                assert!(Instant::now() < deadline, "the endpoint did not close");
+            }
+            received
+        };
+        // A client that sent nothing, as a VMM that fails as it starts, and one that sent its
+        // event, each closed before the listener accepted their connection: what they sent
+        // remains to read, and then the end of the pipe.
+        for sent in [&b""[..], b"the event\n"] {
+            let (endpoint, mut listener) = listen_ready(Path::new("unused")).unwrap();
+            let mut client = fs::OpenOptions::new().write(true).open(&endpoint).unwrap();
+            client.write_all(sent).unwrap();
+            drop(client);
+            assert_eq!(receive_all(&mut listener), sent);
+        }
+        // A client that closes once the listener accepted its connection.
+        let (endpoint, mut listener) = listen_ready(Path::new("unused")).unwrap();
+        let client = fs::OpenOptions::new().write(true).open(&endpoint).unwrap();
+        let mut received = Vec::new();
+        assert!(
+            listener
+                .receive(pid, &mut received, 64, Duration::from_millis(50))
+                .unwrap()
+        );
+        drop(client);
+        assert_eq!(receive_all(&mut listener), b"");
+    }
+
     #[test]
     fn windows_refuses_to_terminate_a_process_that_exited_with_access_denied() {
         let mut child = std::process::Command::new("ping")

nvx-rs/src/host/sandbox/distro.rs

@@ -337,6 +337,12 @@ impl DistroBinding {
         self.policy
     }
 
+    /// Returns the binding's version, which a [check](Self::check) requires to be
+    /// [`BINDING_VERSION`].
+    pub(crate) fn version(&self) -> u32 {
+        self.version
+    }
+
     /// Returns the binding of `layer`, with the seals of its image and manifest and `policy`.
     fn new(
         layer: &SandboxLayer,

nvx-rs/src/host/sandbox/scratch.rs

@@ -257,6 +257,12 @@ impl ScratchBinding {
         self.length
     }
 
+    /// Returns the binding's version, which a [check](Self::check) requires to be
+    /// [`SCRATCH_BINDING_VERSION`].
+    pub(crate) fn version(&self) -> u32 {
+        self.version
+    }
+
     /// Returns the binding of the image that `seal` seals.
     fn of(seal: &FileSeal) -> Self {
         Self {

nvx-rs/src/host/store/lifecycle/tests.rs

@@ -11,8 +11,8 @@ use std::time::Duration;
 
 use super::*;
 use crate::host::store::{
-    CAPABILITY_NAME, CLONE_ACTIVATION_NAME, CLONE_SCRATCH_NAME, CONSOLE_NAME, LOG_NAME,
-    OUTCOME_NAME, RESTORE_READY_NAME, SOCKET_NAME,
+    CAPABILITY_NAME, CLONE_ACTIVATION_NAME, CLONE_FAILURE_NAME, CLONE_PROGRESS_NAME,
+    CLONE_SCRATCH_NAME, CONSOLE_NAME, LOG_NAME, OUTCOME_NAME, RESTORE_READY_NAME, SOCKET_NAME,
 };
 use crate::host::test_support::released;
 
@@ -326,6 +326,8 @@ fn a_removal_removes_the_store_the_callers_files_and_the_directory() {
         CLONE_BINDING_NAME,
         CLONE_SCRATCH_NAME,
         CLONE_ACTIVATION_NAME,
+        CLONE_PROGRESS_NAME,
+        CLONE_FAILURE_NAME,
         RESTORE_READY_NAME,
         BOOT,
         ".runtime.json.12345.tmp",
@@ -417,6 +419,8 @@ fn write_clone_files(dir: &Path) {
         CLONE_BINDING_NAME,
         CLONE_SCRATCH_NAME,
         CLONE_ACTIVATION_NAME,
+        CLONE_PROGRESS_NAME,
+        CLONE_FAILURE_NAME,
         RESTORE_READY_NAME,
     ] {
         fs::write(dir.join(name), b"{}").unwrap();
@@ -536,6 +540,8 @@ fn linked_entries(more: &[&str]) -> Vec<String> {
     let mut names = vec![
         CLONE_ACTIVATION_NAME,
         CLONE_BINDING_NAME,
+        CLONE_FAILURE_NAME,
+        CLONE_PROGRESS_NAME,
         CLONE_SCRATCH_NAME,
         CONFIG,
         LOCK_NAME,

nvx-rs/src/host/store/mod.rs

@@ -19,6 +19,8 @@
 //!   clone-scratch.raw      the file of the scratch device of a clone whose upper is in RAM, a
 //!                          sparse file that the clone's plan creates
 //!   clone-activation.bin   the activation record of a clone, which its launch writes
+//!   clone-progress.log     the progress log of a clone's launch and restore sequence
+//!   clone-failure.json     the record of a clone's failure
 //!   restore.sock           the endpoint at which the launch of a clone awaits the restore-ready
 //!                          event of its VMM, on Linux
 //!   ...                    the caller's configuration record and other files, which it declares
@@ -316,19 +318,34 @@ pub const CLONE_ACTIVATION_NAME: &str = "clone-activation.bin";
 /// random. [`template::Store::release`](crate::host::template::Store::release) removes it once the
 /// clone's VM has exited, as a [removal](self#removal) of the sandbox does.
 pub const RESTORE_READY_NAME: &str = "restore.sock";
+/// Name of the progress log of a clone's launch, which the launch,
+/// [`template::ClonePlan::launch`](crate::host::template::ClonePlan::launch), creates, and to which
+/// it and the clone's restore sequence append a line for each step that the clone reaches, without
+/// syncing it, so that a reattach tells how far the clone got once the caller of its launch
+/// crashed, and which [`template::Store::release`](crate::host::template::Store::release) removes,
+/// as a [removal](self#removal) of the sandbox does. No other launch writes it.
+pub const CLONE_PROGRESS_NAME: &str = "clone-progress.log";
+/// Name of the record of a clone's failure, which a failure of the clone from its launch on
+/// writes before its teardown, and again once its teardown is known, as
+/// [`template::CloneFailure`](crate::host::template::CloneFailure) describes, and which
+/// [`template::Store::release`](crate::host::template::Store::release) removes, as a
+/// [removal](self#removal) of the sandbox does.
+pub const CLONE_FAILURE_NAME: &str = "clone-failure.json";
 /// The files that the clone of a sandbox keeps of its launch, which a release removes once the
 /// clone's VM has exited, as a [removal](self#removal) of the sandbox does, before the clone
 /// binding.
-const CLONE_FILE_NAMES: [&str; 3] = [
+const CLONE_FILE_NAMES: [&str; 5] = [
     CLONE_SCRATCH_NAME,
     CLONE_ACTIVATION_NAME,
     RESTORE_READY_NAME,
+    CLONE_PROGRESS_NAME,
+    CLONE_FAILURE_NAME,
 ];
 /// Every file that the store keeps in a sandbox's directory. Replacing a record leaves a
 /// temporary file named `.NAME.NUMBER.tmp` beside it only if the replacement was interrupted, and
 /// a VMM that is killed while it writes its outcome report leaves the file in which it stages it;
 /// see [the report staging file](self#the-report-staging-file).
-pub const FILE_NAMES: [&str; 12] = [
+pub const FILE_NAMES: [&str; 14] = [
     LOCK_NAME,
     JOURNAL_NAME,
     RUNTIME_NAME,
@@ -341,6 +358,8 @@ pub const FILE_NAMES: [&str; 12] = [
     CLONE_SCRATCH_NAME,
     CLONE_ACTIVATION_NAME,
     RESTORE_READY_NAME,
+    CLONE_PROGRESS_NAME,
+    CLONE_FAILURE_NAME,
 ];
 
 /// Interval at which a lock that another holder has is tried again: the sandbox's lock, by a
@@ -550,6 +569,16 @@ impl SandboxStore {
         self.path(RESTORE_READY_NAME)
     }
 
+    /// Returns the path of the progress log of a clone's launch, [`CLONE_PROGRESS_NAME`].
+    pub fn clone_progress_path(&self) -> PathBuf {
+        self.path(CLONE_PROGRESS_NAME)
+    }
+
+    /// Returns the path of the record of a clone's failure, [`CLONE_FAILURE_NAME`].
+    pub fn clone_failure_path(&self) -> PathBuf {
+        self.path(CLONE_FAILURE_NAME)
+    }
+
     /// Reads the runtime record, which names the sandbox's VM from the commit of its launch until
     /// its runtime is cleared.
     pub fn runtime(&self) -> Result<Option<RuntimeRecord>> {
@@ -807,8 +836,9 @@ impl LockedStore {
     /// Removes what the clone of the sandbox kept of its launch in the sandbox's directory, whose
     /// VM has exited, through `directory`, the handle of the directory that
     /// [`open_dir`](Self::open_dir) returned: the file of its scratch device,
-    /// [`CLONE_SCRATCH_NAME`], its activation record, [`CLONE_ACTIVATION_NAME`], and the socket of
-    /// its restore-ready event, [`RESTORE_READY_NAME`]. A release, as a [removal](self#removal)
+    /// [`CLONE_SCRATCH_NAME`], its activation record, [`CLONE_ACTIVATION_NAME`], the socket of its
+    /// restore-ready event, [`RESTORE_READY_NAME`], its progress log, [`CLONE_PROGRESS_NAME`], and
+    /// the record of its failure, [`CLONE_FAILURE_NAME`]. A release, as a [removal](self#removal)
     /// does, removes them before the clone binding, whose removal, once durable, makes theirs
     /// durable too, since all are entries of the sandbox's directory. A file whose deletion is
     /// pending, as a Windows file system without POSIX deletion leaves a removed file that another

nvx-rs/src/host/store/tests.rs

@@ -410,6 +410,8 @@ fn the_files_that_a_clone_kept_of_its_launch_go_and_nothing_else() {
     for path in [
         store.clone_scratch_path(),
         store.clone_activation_path(),
+        store.clone_progress_path(),
+        store.clone_failure_path(),
         store.restore_ready_path(),
         store.log_path(),
         store.binding_path(),