diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 66dfdd3eb..0311c2294 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -635,13 +635,27 @@ and no launch. the launch that the lease reserves, from metadata alone: the leased generation, its key, which must speak the current contracts, the boot and control contracts of the runtime that captured it included, and its snapshot; a network exactly -when the template has a NIC; a memory target in whole blocks of -`MEMORY_BLOCK_SIZE` from the template's RAM to its capacity; the clone's image, -which the check of its `DistroBinding` opens; its upper, either the caller's -scratch image, which the check of its `ScratchBinding` claims, and which nothing -formats, copies, or reads, or for an upper in RAM a fresh sparse file, -`clone-scratch.raw` in the sandbox's directory, which the release removes; and a -fresh capability. +when the template has a NIC, on the guest network of that NIC, whose address +OpenVMM restores from the snapshot, and under the policy of that NIC, which +OpenVMM checks as it restores it, since the key names the policy only by +OpenVMM's identity of it; a memory target in whole blocks of `MEMORY_BLOCK_SIZE` +from the template's RAM to its capacity; the clone's image, which the check of +its `DistroBinding` opens; its upper, either the caller's scratch image, which +the check of its `ScratchBinding` claims, and which nothing formats, copies, or +reads, or for an upper in RAM a fresh sparse file, `clone-scratch.raw` in the +sandbox's directory, which the release removes; and a fresh capability. + +It builds the clone's activation record, `ClonePlan::activation`, which the +clone's guest applies in place of the configuration that the template captured, +from the request's `WorkloadView`, overlay mode, and journal policy, and from +the checked inputs: a restore section with a fresh transaction ID of 16 random +bytes and the lease's `TemplateReference`; the image, an EROFS layer, with its +binding's identity and the UUID that the adapter's plan expects of it, since a +GPT image is refused; a tmpfs, or the scratch image's binding identity with the +journal policy; no export; the template NIC's policy identity, as its key +records it, unpublished listeners, and the guest's proxy URL; and the template's +processors and the memory target. A binding's identity is the SHA-256 digest of +a domain and of the binding's serde form, never of a file's content. The image and the scratch device must present the `BlockGeometry` of the template's, which a restore compares, and be, at the paths by which OpenVMM @@ -652,6 +666,34 @@ scratch image may be a file that the sandbox's store owns, such as transitions replace or remove them, and a link or a directory under one of the store's names is refused. +`ClonePlan::launch` then launches the clone as the launch that its lease +reserves, once it has checked that the sandbox's clone binding records the +lease, and returns a `Restoring`: it writes the clone's activation record in its +canonical encoding, `clone-activation.bin`, from which a caller resends it +unchanged; it listens for the VMM's restore-ready event, at the socket +`restore.sock` on Linux, and at a named pipe of a random name on Windows; and it +runs `supervisor::launch` with OpenVMM's restore of the clone's devices, +`--restore-snapshot` with the generation's snapshot in place, which no clone +copies or changes, so that clones of one generation restore it at once, +`--restore-memory` with the memory target, and `--restore-ready-path`, under the +template's backend, CPU profile, and processors, with a boot console that +discards its output, as cold boots have, and the control console, but no kernel, +initramfs, command line, or guest address, which the snapshot fixes. + +On Linux, where OpenVMM requires a restore's control socket in the snapshot's +parent, the clone's VMM listens in the generation's payload, at `.sock`, +named after the launch that the lease reserves, whose path must be shorter than +108 bytes with or without the payload's links resolved; on Windows, it listens +at a named pipe, as every launch does. The VMM inherits the claim of a scratch +image, on Windows through the image's guard, once the launch has closed the +claim's handle, which OpenVMM's open of the image would refuse. + +A payload holds the sockets of its clones only while they lease it: the VMM +removes its socket as it exits, a stop or a recovery removes the one that an +abrupt exit leaves, and the release makes the socket's absence durable before +the lease ends, and removes the record and `restore.sock`, as it removes +`clone-scratch.raw`. + `Store::tombstone` quarantines a generation, which takes no new lease while its leases keep working, and `Store::retire` deletes it once its leases have drained, crash-safely, removing the records of leases that no clone binding diff --git a/nvx-rs/src/host/platform/linux.rs b/nvx-rs/src/host/platform/linux.rs index c26a28516..1157a689f 100644 --- a/nvx-rs/src/host/platform/linux.rs +++ b/nvx-rs/src/host/platform/linux.rs @@ -2,7 +2,7 @@ use std::fs::{self, DirBuilder, File, OpenOptions, Permissions, TryLockError}; use std::io::{self, Read}; use std::os::fd::{AsFd, AsRawFd, RawFd}; use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}; -use std::os::unix::net::UnixStream; +use std::os::unix::net::{UnixListener, UnixStream}; use std::path::Path; use std::process::Command; use std::thread; @@ -639,6 +639,25 @@ pub(crate) fn opened_file_identity(file: &File) -> io::Result<(u64, u64)> { Ok((metadata.dev(), metadata.ino())) } +/// Returns the identity by which the stores tell a file or directory apart from every other, +/// following its links: its device and inode, as [`opened_file_id`] gives them. +pub(crate) fn file_id(path: &Path) -> io::Result<(u64, u128)> { + let metadata = fs::metadata(path)?; + Ok((metadata.dev(), u128::from(metadata.ino()))) +} + +/// Returns the identity by which the stores tell the file that `file` holds open apart from every +/// other: its device and inode, as its [`FileSeal`](crate::host::seal::FileSeal) holds them. +pub(crate) fn opened_file_id(file: &File) -> io::Result<(u64, u128)> { + let metadata = file.metadata()?; + Ok((metadata.dev(), u128::from(metadata.ino()))) +} + +/// Returns the number of names, its hard links, of the file that `file` holds open. +pub(crate) fn link_count(file: &File) -> io::Result { + Ok(file.metadata()?.nlink()) +} + /// An open directory, through which operations name its entries, so that they reach the entries /// of the directory that it opened, whatever its path names meanwhile. pub(crate) struct Directory(File); @@ -655,19 +674,102 @@ impl Directory { .map(Self) } - /// Returns the device and inode of its entry `name`, not following a link, as - /// [`opened_file_identity`] gives them, or `None` if it has no such entry. - pub(crate) fn entry_identity(&self, name: &str) -> io::Result> { + /// Returns the identity of its entry `name`, not following a link, as [`opened_file_id`] gives + /// it, or `None` if it has no such entry. + pub(crate) fn entry_id(&self, name: &str) -> io::Result> { use nix::fcntl::AtFlags; use nix::sys::stat::fstatat; match fstatat(&self.0, name, AtFlags::AT_SYMLINK_NOFOLLOW) { - Ok(status) => Ok(Some((status.st_dev, status.st_ino))), + Ok(status) => Ok(Some((status.st_dev, u128::from(status.st_ino)))), Err(Errno::ENOENT) => Ok(None), Err(errno) => Err(errno.into()), } } + /// Opens its entry `name` for reading, not following a link, and without waiting, as an open + /// of a FIFO without a writer would, or returns `None` if it has no such entry. The caller + /// checks what the entry is. + pub(crate) fn open_file(&self, name: &str) -> io::Result> { + use nix::fcntl::{OFlag, openat}; + use nix::sys::stat::Mode; + + match openat( + &self.0, + name, + OFlag::O_RDONLY | OFlag::O_NOFOLLOW | OFlag::O_NONBLOCK | OFlag::O_CLOEXEC, + Mode::empty(), + ) { + Ok(file) => Ok(Some(File::from(file))), + Err(Errno::ENOENT) => Ok(None), + Err(errno) => Err(errno.into()), + } + } + + /// Opens the directory `path` with the right to search it alone, without following a link at + /// its own name, though links on the way to it are followed, or returns `None` if it does not + /// exist. Fails if it is a link, or not a directory. + /// + /// The handle names the directory's entries, and needs no right to read the directory, which + /// an owner-only directory may withhold; it cannot [sync](Self::sync) it. + pub(crate) fn search(path: &Path) -> io::Result> { + use nix::fcntl::OFlag; + + match OpenOptions::new() + .read(true) + .custom_flags((OFlag::O_PATH | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW).bits()) + .open(path) + { + Ok(dir) => Ok(Some(Self(dir))), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) + if [Errno::ELOOP, Errno::ENOTDIR] + .map(|errno| Some(errno as i32)) + .contains(&error.raw_os_error()) => + { + Err(io::Error::other(format!( + "{} is a link, or not a directory", + path.display() + ))) + } + Err(error) => Err(error), + } + } + + /// Opens its subdirectory `name` with the right to search it alone, not following a link, as + /// [`search`](Self::search) opens a directory, or returns `None` if it has no such entry. + /// Fails if the entry is a link, or not a directory. + pub(crate) fn search_dir(&self, name: &str) -> io::Result> { + self.open_dir_with(name, nix::fcntl::OFlag::O_PATH) + } + + /// Opens its subdirectory `name` for reading, not following a link, or returns `None` if it + /// has no such entry. Fails if the entry is a link, or not a directory. + pub(crate) fn open_dir(&self, name: &str) -> io::Result> { + self.open_dir_with(name, nix::fcntl::OFlag::O_RDONLY) + } + + /// Opens its subdirectory `name` with the access `access`, not following a link, as + /// [`open_dir`](Self::open_dir) and [`search_dir`](Self::search_dir) do. + fn open_dir_with(&self, name: &str, access: nix::fcntl::OFlag) -> io::Result> { + use nix::fcntl::{OFlag, openat}; + use nix::sys::stat::Mode; + + match openat( + &self.0, + name, + access | OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC, + Mode::empty(), + ) { + Ok(dir) => Ok(Some(Self(File::from(dir)))), + Err(Errno::ENOENT) => Ok(None), + Err(Errno::ELOOP | Errno::ENOTDIR) => Err(io::Error::other(format!( + "{name} is a link, or not a directory" + ))), + Err(errno) => Err(errno.into()), + } + } + /// Removes its file `name`, if it has one. pub(crate) fn remove_file(&self, name: &str) -> io::Result<()> { use nix::unistd::{UnlinkatFlags, unlinkat}; @@ -753,6 +855,31 @@ pub(crate) fn control_endpoint(socket_path: &Path) -> io::Result { .ok_or_else(|| io::Error::other("control socket path is not valid UTF-8")) } +/// The listener at which the launch of a clone awaits the restore-ready event of its VMM: a Unix +/// socket, which the VMM connects to once, and which queues its connection until the host +/// accepts it. +#[derive(Debug)] +pub(crate) struct ReadyListener { + /// The listening socket, which the VMM finds as long as it is open. + _socket: UnixListener, +} + +/// Listens at the Unix socket `socket_path`, in place of a socket that an earlier launch left +/// there, as the restore-ready endpoint of a clone's launch, and returns the endpoint, as +/// OpenVMM's `--restore-ready-path` takes it, and its listener. +pub(crate) fn listen_ready(socket_path: &Path) -> io::Result<(String, ReadyListener)> { + let endpoint = socket_path + .to_str() + .map(str::to_owned) + .ok_or_else(|| io::Error::other("restore-ready socket path is not valid UTF-8"))?; + match fs::remove_file(socket_path) { + Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error), + _ => {} + } + let socket = UnixListener::bind(socket_path)?; + Ok((endpoint, ReadyListener { _socket: socket })) +} + /// Connects to a control endpoint served by the process `expected_pid`. /// /// The connection attempt never blocks: a listener whose backlog is full yields diff --git a/nvx-rs/src/host/platform/mod.rs b/nvx-rs/src/host/platform/mod.rs index 3243f86ca..c1886f7dd 100644 --- a/nvx-rs/src/host/platform/mod.rs +++ b/nvx-rs/src/host/platform/mod.rs @@ -171,4 +171,30 @@ mod tests { Err(error) => panic!("{error}"), } } + + /// The stores tell files apart by the identities that their seals hold, with the 128-bit file + /// IDs of `FILE_ID_INFO` on Windows, whether they examine a path, an open file, or an entry of + /// an open directory. + #[cfg(any(target_os = "linux", windows))] + #[test] + fn the_stores_tell_files_apart_by_the_identities_that_their_seals_hold() { + use crate::host::seal::FileSeal; + + let directory = tempfile::tempdir().unwrap(); + let opened = Directory::open(directory.path()).unwrap(); + let mut ids = Vec::new(); + for name in ["first", "second"] { + let path = directory.path().join(name); + std::fs::write(&path, b"").unwrap(); + let file = std::fs::File::open(&path).unwrap(); + let seal = FileSeal::of(&file).unwrap(); + let id = (seal.device(), seal.file_id()); + assert_eq!(opened_file_id(&file).unwrap(), id, "{name}"); + assert_eq!(file_id(&path).unwrap(), id, "{name}"); + assert_eq!(opened.entry_id(name).unwrap(), Some(id), "{name}"); + ids.push(id); + } + assert_ne!(ids[0], ids[1]); + assert_eq!(opened.entry_id("missing").unwrap(), None); + } } diff --git a/nvx-rs/src/host/platform/other.rs b/nvx-rs/src/host/platform/other.rs index 57af35f57..443b748db 100644 --- a/nvx-rs/src/host/platform/other.rs +++ b/nvx-rs/src/host/platform/other.rs @@ -98,6 +98,20 @@ pub(crate) fn opened_file_identity(_file: &File) -> io::Result<(u64, u64)> { Err(unsupported()) } +/// Fails: this host keeps no store. +pub(crate) fn file_id(_path: &Path) -> io::Result<(u64, u128)> { + Err(unsupported()) +} + +/// Fails: this host keeps no store. +pub(crate) fn opened_file_id(_file: &File) -> io::Result<(u64, u128)> { + Err(unsupported()) +} + +pub(crate) fn link_count(_file: &File) -> io::Result { + Err(unsupported()) +} + /// An open directory, which this host never holds: it keeps no sandbox's store. pub(crate) enum Directory {} @@ -108,7 +122,27 @@ impl Directory { } /// Never called: no directory is open. - pub(crate) fn entry_identity(&self, _name: &str) -> io::Result> { + pub(crate) fn entry_id(&self, _name: &str) -> io::Result> { + match *self {} + } + + /// Never called: no directory is open. + pub(crate) fn open_file(&self, _name: &str) -> io::Result> { + match *self {} + } + + /// Never called: no directory is open. + pub(crate) fn open_dir(&self, _name: &str) -> io::Result> { + match *self {} + } + + /// Fails: this host keeps no store. + pub(crate) fn search(_path: &Path) -> io::Result> { + Err(unsupported()) + } + + /// Never called: no directory is open. + pub(crate) fn search_dir(&self, _name: &str) -> io::Result> { match *self {} } @@ -210,6 +244,14 @@ pub(crate) fn control_endpoint(_socket_path: &Path) -> io::Result { Err(unsupported()) } +/// The listener of the restore-ready endpoint of a clone's launch, which this host never makes. +#[derive(Debug)] +pub(crate) struct ReadyListener; + +pub(crate) fn listen_ready(_socket_path: &Path) -> io::Result<(String, ReadyListener)> { + Err(unsupported()) +} + pub(crate) fn connect_endpoint( _endpoint: &str, _expected_pid: u32, diff --git a/nvx-rs/src/host/platform/windows.rs b/nvx-rs/src/host/platform/windows.rs index 287f4a566..b5dd0fa6d 100644 --- a/nvx-rs/src/host/platform/windows.rs +++ b/nvx-rs/src/host/platform/windows.rs @@ -33,10 +33,10 @@ use windows_sys::Win32::Security::{ use windows_sys::Win32::Storage::FileSystem::{ BY_HANDLE_FILE_INFORMATION, CreateDirectoryW, CreateFileW, DELETE, FILE_ALL_ACCESS, FILE_APPEND_DATA, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_DELETE_CHILD, - FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_OPEN_REPARSE_POINT, FILE_FLAG_OVERLAPPED, - FILE_WRITE_ATTRIBUTES, FILE_WRITE_DATA, FILE_WRITE_EA, GetFileInformationByHandle, MoveFileExW, - OPEN_EXISTING, READ_CONTROL, ReadFile, SECURITY_IDENTIFICATION, SECURITY_SQOS_PRESENT, - WRITE_DAC, WRITE_OWNER, WriteFile, + FILE_FLAG_BACKUP_SEMANTICS, FILE_FLAG_FIRST_PIPE_INSTANCE, FILE_FLAG_OPEN_REPARSE_POINT, + FILE_FLAG_OVERLAPPED, FILE_WRITE_ATTRIBUTES, FILE_WRITE_DATA, FILE_WRITE_EA, + GetFileInformationByHandle, MoveFileExW, OPEN_EXISTING, PIPE_ACCESS_INBOUND, READ_CONTROL, + ReadFile, SECURITY_IDENTIFICATION, SECURITY_SQOS_PRESENT, WRITE_DAC, WRITE_OWNER, WriteFile, }; #[cfg(feature = "cli")] use windows_sys::Win32::System::Console::{ @@ -49,7 +49,10 @@ use windows_sys::Win32::System::IO::{ use windows_sys::Win32::System::LibraryLoader::{ GetProcAddress, LOAD_LIBRARY_SEARCH_SYSTEM32, LoadLibraryExW, }; -use windows_sys::Win32::System::Pipes::{GetNamedPipeServerProcessId, WaitNamedPipeW}; +use windows_sys::Win32::System::Pipes::{ + CreateNamedPipeW, GetNamedPipeServerProcessId, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, + PIPE_TYPE_BYTE, PIPE_WAIT, WaitNamedPipeW, +}; use windows_sys::Win32::System::Registry::{ HKEY_LOCAL_MACHINE, RRF_RT_REG_DWORD, RRF_RT_REG_SZ, RegGetValueW, }; @@ -1007,6 +1010,35 @@ impl CurrentUser { /// that allows others is not restricted, since a handle that they opened keeps its access. pub(crate) fn create_store_dir(path: &Path) -> io::Result> { let user = CurrentUser::get()?; + let name = wide(path); + with_owner_only_attributes( + &user, + FILE_ALL_ACCESS, + OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE, + |attributes| { + // SAFETY: the name is NUL-terminated, and the attributes hold a valid descriptor. + if unsafe { CreateDirectoryW(name.as_ptr(), attributes) } == 0 { + let error = last_error(); + if error != ERROR_ALREADY_EXISTS { + return Err(io::Error::from_raw_os_error(error as i32)); + } + } + Ok(()) + }, + )??; + store_dir_problem(path, &user) +} + +/// Runs `create` with the attributes of a new object that `user` owns, whose protected access +/// control list allows `user` alone, `access`, with an entry that the objects created in it +/// inherit as `inheritance` says, and returns what `create` returns. The attributes let no child +/// process inherit the object's handle. +fn with_owner_only_attributes( + user: &CurrentUser, + access: u32, + inheritance: u32, + create: impl FnOnce(&SECURITY_ATTRIBUTES) -> T, +) -> io::Result { let sid = user.sid(); // SAFETY: the identifier is valid. let sid_len = unsafe { GetLengthSid(sid) } as usize; @@ -1025,13 +1057,7 @@ pub(crate) fn create_store_dir(path: &Path) -> io::Result> { // its use below. let built = unsafe { InitializeAcl(acl_ptr, acl_len, ACL_REVISION) != 0 - && AddAccessAllowedAceEx( - acl_ptr, - ACL_REVISION, - OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE, - FILE_ALL_ACCESS, - sid, - ) != 0 + && AddAccessAllowedAceEx(acl_ptr, ACL_REVISION, inheritance, access, sid) != 0 && InitializeSecurityDescriptor(descriptor_ptr, SECURITY_DESCRIPTOR_REVISION) != 0 && SetSecurityDescriptorOwner(descriptor_ptr, sid, 0) != 0 && SetSecurityDescriptorDacl(descriptor_ptr, 1, acl_ptr, 0) != 0 @@ -1046,15 +1072,7 @@ pub(crate) fn create_store_dir(path: &Path) -> io::Result> { lpSecurityDescriptor: descriptor_ptr, bInheritHandle: 0, }; - let name = wide(path); - // SAFETY: the name is NUL-terminated, and the attributes hold a valid descriptor. - if unsafe { CreateDirectoryW(name.as_ptr(), &attributes) } == 0 { - let error = last_error(); - if error != ERROR_ALREADY_EXISTS { - return Err(io::Error::from_raw_os_error(error as i32)); - } - } - store_dir_problem(path, &user) + Ok(create(&attributes)) } /// Returns why the directory `path` cannot hold a sandbox store, if it cannot: unless it is @@ -1348,7 +1366,8 @@ pub(crate) fn set_access(path: &Path, sddl: &str) -> io::Result<()> { result } -/// Returns the volume serial number and file index of a file or directory. +/// Returns the volume serial number and 64-bit file index of a file or directory, which a file +/// system such as ReFS does not keep unique. /// /// The object is opened without access rights, which succeeds even while OpenVMM holds it open. pub fn file_identity(path: &Path) -> io::Result<(u64, u64)> { @@ -1359,6 +1378,18 @@ pub fn file_identity(path: &Path) -> io::Result<(u64, u64)> { handle_identity(file.as_raw_handle() as HANDLE) } +/// Returns the identity by which the stores tell a file or directory apart from every other, +/// following its links, as [`opened_file_id`] gives it. +/// +/// The object is opened without access rights, which succeeds even while OpenVMM holds it open. +pub(crate) fn file_id(path: &Path) -> io::Result<(u64, u128)> { + let file = OpenOptions::new() + .access_mode(0) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(path)?; + opened_file_id(&file) +} + /// Returns whether the file at `path` is deleted: whether an open of it finds no file, or one /// whose deletion is pending, which a file system without POSIX deletion keeps under its name, /// refusing every open of it, until the last handle to it closes. A file that opens is not, nor is @@ -1407,10 +1438,9 @@ impl Directory { .map(Self) } - /// Returns the volume serial number and file index of its entry `name`, not following a link - /// or other reparse point, as [`opened_file_identity`] gives them, or `None` if it has no such - /// entry. - pub(crate) fn entry_identity(&self, name: &str) -> io::Result> { + /// Returns the identity of its entry `name`, not following a link or other reparse point, as + /// [`opened_file_id`] gives it, or `None` if it has no such entry. + pub(crate) fn entry_id(&self, name: &str) -> io::Result> { use windows_sys::Wdk::Storage::FileSystem::{ FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, }; @@ -1422,12 +1452,95 @@ impl Directory { FILE_READ_ATTRIBUTES | SYNCHRONIZE, FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, ) { - Ok(entry) => handle_identity(raw(&entry)).map(Some), + Ok(entry) => opened_file_id(&fs::File::from(entry)).map(Some), Err(STATUS_OBJECT_NAME_NOT_FOUND) => Ok(None), Err(status) => Err(status_error(status)), } } + /// Opens its entry `name` for reading, not following a link or other reparse point, or + /// returns `None` if it has no such entry, or one whose deletion is pending, which a file + /// system without POSIX deletion keeps under its name until its last handle closes, and which + /// is removed. The caller checks what the entry is. + pub(crate) fn open_file(&self, name: &str) -> io::Result> { + use windows_sys::Wdk::Storage::FileSystem::{ + FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, + }; + use windows_sys::Win32::Foundation::{STATUS_DELETE_PENDING, STATUS_OBJECT_NAME_NOT_FOUND}; + use windows_sys::Win32::Storage::FileSystem::FILE_GENERIC_READ; + + match self.open_entry( + OsStr::new(name), + FILE_GENERIC_READ, + FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, + ) { + Ok(entry) => Ok(Some(fs::File::from(entry))), + Err(STATUS_OBJECT_NAME_NOT_FOUND | STATUS_DELETE_PENDING) => Ok(None), + Err(status) => Err(status_error(status)), + } + } + + /// Opens the directory `path` without following a link or other reparse point at its own + /// name, though those on the way to it are followed, and without access rights, which the + /// opens of its entries do not need, or returns `None` if it does not exist. Fails if it is a + /// link, a junction or another reparse point, or not a directory. + pub(crate) fn search(path: &Path) -> io::Result> { + let opened = OpenOptions::new() + .access_mode(0) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path); + let dir = match opened { + Ok(dir) => dir, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let attributes = handle_information(dir.as_raw_handle() as HANDLE)?.dwFileAttributes; + if attributes & FILE_ATTRIBUTE_DIRECTORY == 0 + || attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 + { + return Err(io::Error::other(format!( + "{} is a link, or not a directory", + path.display() + ))); + } + Ok(Some(Self(dir))) + } + + /// Opens its subdirectory `name` as [`open_dir`](Self::open_dir) does: Windows needs no right + /// to list a directory to name its entries. + pub(crate) fn search_dir(&self, name: &str) -> io::Result> { + self.open_dir(name) + } + + /// Opens its subdirectory `name`, not following a link or other reparse point, or returns + /// `None` if it has no such entry, or one whose deletion is pending. Fails if the entry is a + /// link, a junction or another reparse point, or not a directory. + pub(crate) fn open_dir(&self, name: &str) -> io::Result> { + use windows_sys::Wdk::Storage::FileSystem::{ + FILE_DIRECTORY_FILE, FILE_OPEN_REPARSE_POINT, FILE_SYNCHRONOUS_IO_NONALERT, + }; + use windows_sys::Win32::Foundation::{ + STATUS_DELETE_PENDING, STATUS_NOT_A_DIRECTORY, STATUS_OBJECT_NAME_NOT_FOUND, + }; + use windows_sys::Win32::Storage::FileSystem::{FILE_READ_ATTRIBUTES, SYNCHRONIZE}; + + let refused = || io::Error::other(format!("{name} is a link, or not a directory")); + let entry = match self.open_entry( + OsStr::new(name), + FILE_READ_ATTRIBUTES | SYNCHRONIZE, + FILE_DIRECTORY_FILE | FILE_OPEN_REPARSE_POINT | FILE_SYNCHRONOUS_IO_NONALERT, + ) { + Ok(entry) => entry, + Err(STATUS_OBJECT_NAME_NOT_FOUND | STATUS_DELETE_PENDING) => return Ok(None), + Err(STATUS_NOT_A_DIRECTORY) => return Err(refused()), + Err(status) => return Err(status_error(status)), + }; + if handle_information(raw(&entry))?.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return Err(refused()); + } + Ok(Some(Self(fs::File::from(entry)))) + } + /// Removes its file `name`, if it has one. A file whose deletion is pending already, which a /// file system without POSIX deletion keeps under its name until its last handle closes, /// counts as removed. @@ -1571,12 +1684,29 @@ fn status_error(status: windows_sys::Win32::Foundation::NTSTATUS) -> io::Error { io::Error::from_raw_os_error(code as i32) } -/// Returns the volume serial number and file index of the file that `file` holds open. +/// Returns the volume serial number and 64-bit file index of the file that `file` holds open, +/// which a file system such as ReFS does not keep unique: [`opened_file_id`] tells files apart. pub(crate) fn opened_file_identity(file: &fs::File) -> io::Result<(u64, u64)> { handle_identity(file.as_raw_handle() as HANDLE) } -/// Returns the volume serial number and file index of the file that `handle` holds open. +/// Returns the identity by which the stores tell the file that `file` holds open apart from every +/// other, as its [`FileSeal`](crate::host::seal::FileSeal) holds it: the volume serial number and +/// the 128-bit file ID of `FILE_ID_INFO`, which a file system such as ReFS keeps unique where the +/// 64-bit file index is not. +pub(crate) fn opened_file_id(file: &fs::File) -> io::Result<(u64, u128)> { + let seal = crate::host::seal::FileSeal::of(file)?; + Ok((seal.device(), seal.file_id())) +} + +/// Returns the number of names, its hard links, of the file that `file` holds open. +pub(crate) fn link_count(file: &fs::File) -> io::Result { + Ok(u64::from( + handle_information(file.as_raw_handle() as HANDLE)?.nNumberOfLinks, + )) +} + +/// Returns the volume serial number and 64-bit file index of the file that `handle` holds open. fn handle_identity(handle: HANDLE) -> io::Result<(u64, u64)> { let information = handle_information(handle)?; Ok(( @@ -1647,6 +1777,48 @@ pub(crate) fn control_endpoint(_socket_path: &Path) -> io::Result { Ok(format!("//./pipe/openvmm-microvm-{suffix}")) } +/// The bytes that the inbound buffer of a restore-ready pipe holds, more than OpenVMM's event. +const READY_BUFFER_LEN: u32 = 4096; + +/// The listener at which the launch of a clone awaits the restore-ready event of its VMM: the +/// one instance of a named pipe, inbound, which only the current user may open, from this host +/// alone, and which takes the connection of the VMM before the host accepts it. +#[derive(Debug)] +pub(crate) struct ReadyListener { + /// The pipe's instance, which the VMM finds as long as it is open. + _pipe: OwnedHandle, +} + +/// Listens at a fresh named pipe, whose name it draws at random, as the restore-ready endpoint of +/// a clone's launch, and returns the endpoint, as OpenVMM's `--restore-ready-path` takes it, and +/// its listener. The first instance of the pipe is the listener's, so no other process holds the +/// name, and the pipe's protected access control list allows the current user alone. +pub(crate) fn listen_ready(_socket_path: &Path) -> io::Result<(String, ReadyListener)> { + let mut bytes = [0u8; 16]; + getrandom::fill(&mut bytes).map_err(io::Error::other)?; + let suffix: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); + let endpoint = format!("//./pipe/nvx-restore-ready-{suffix}"); + let name = wide(endpoint.replace('/', "\\")); + let pipe = + with_owner_only_attributes(&CurrentUser::get()?, FILE_ALL_ACCESS, 0, |attributes| { + // SAFETY: the name is NUL-terminated, and the attributes hold a valid descriptor. + let handle = unsafe { + CreateNamedPipeW( + name.as_ptr(), + PIPE_ACCESS_INBOUND | FILE_FLAG_FIRST_PIPE_INSTANCE, + PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, + 1, + 0, + READY_BUFFER_LEN, + 0, + attributes, + ) + }; + owned(handle).ok_or_else(io::Error::last_os_error) + })??; + Ok((endpoint, ReadyListener { _pipe: pipe })) +} + /// Connects to a named pipe served by the process `expected_pid`. pub(crate) fn connect_endpoint( endpoint: &str, diff --git a/nvx-rs/src/host/seal.rs b/nvx-rs/src/host/seal.rs index 91f5e75dc..02a0844ff 100644 --- a/nvx-rs/src/host/seal.rs +++ b/nvx-rs/src/host/seal.rs @@ -237,6 +237,14 @@ pub(crate) fn final_path(file: &File) -> io::Result { imp::final_path(file) } +/// Returns the final path of the open `file` as [`final_path`] does, but with its volume named by +/// its GUID, `\\?\Volume{GUID}\`, which names a volume that has no drive letter or mount point, +/// and so no path by a drive, too. +#[cfg(windows)] +pub(crate) fn final_volume_path(file: &File) -> io::Result { + imp::final_volume_path(file) +} + /// Opens `path` for its metadata alone, so that no read or mapping of its content can succeed /// through the file: with `O_PATH` on Linux, and on Windows with `FILE_READ_ATTRIBUTES` alone, /// sharing every access. diff --git a/nvx-rs/src/host/seal/windows.rs b/nvx-rs/src/host/seal/windows.rs index ea6236c20..27c5e00d2 100644 --- a/nvx-rs/src/host/seal/windows.rs +++ b/nvx-rs/src/host/seal/windows.rs @@ -12,8 +12,9 @@ use std::path::{Path, PathBuf}; use windows_sys::Win32::Foundation::{FILETIME, GetLastError, HANDLE}; use windows_sys::Win32::Storage::FileSystem::{ BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, FILE_FLAG_BACKUP_SEMANTICS, FILE_ID_INFO, - FILE_SHARE_READ, FILE_TYPE_DISK, FILE_TYPE_UNKNOWN, FileIdInfo, GetFileInformationByHandle, - GetFileInformationByHandleEx, GetFileType, GetFinalPathNameByHandleW, VOLUME_NAME_DOS, + FILE_SHARE_READ, FILE_TYPE_DISK, FILE_TYPE_UNKNOWN, FileIdInfo, GETFINALPATHNAMEBYHANDLE_FLAGS, + GetFileInformationByHandle, GetFileInformationByHandleEx, GetFileType, + GetFinalPathNameByHandleW, VOLUME_NAME_DOS, VOLUME_NAME_GUID, }; use super::{FileId, FileKind, FileSeal, SEAL_VERSION, SealScheme, SealTime}; @@ -110,18 +111,23 @@ pub(super) fn open_read(path: &Path) -> io::Result { /// See [`super::final_path`]. pub(super) fn final_path(file: &File) -> io::Result { + final_path_by(file, VOLUME_NAME_DOS) +} + +/// See [`super::final_volume_path`]. +pub(super) fn final_volume_path(file: &File) -> io::Result { + final_path_by(file, VOLUME_NAME_GUID) +} + +/// Returns the final path of the open `file`, which names its volume as `volume` asks. +fn final_path_by(file: &File, volume: GETFINALPATHNAMEBYHANDLE_FLAGS) -> io::Result { let handle = file.as_raw_handle() as HANDLE; let mut buffer = vec![0u16; 260]; loop { // SAFETY: the handle is open for as long as `file` is borrowed, and the buffer holds // `buffer.len()` units. let length = unsafe { - GetFinalPathNameByHandleW( - handle, - buffer.as_mut_ptr(), - buffer.len() as u32, - VOLUME_NAME_DOS, - ) + GetFinalPathNameByHandleW(handle, buffer.as_mut_ptr(), buffer.len() as u32, volume) } as usize; if length == 0 { return Err(io::Error::last_os_error()); @@ -216,4 +222,23 @@ mod tests { let file = File::open(deep.join("long")).unwrap(); assert!(final_path(&file).unwrap().as_os_str().len() > 260); } + + #[test] + fn final_volume_paths_name_the_file_by_its_volume() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("file"); + std::fs::write(&path, b"").unwrap(); + let found = final_volume_path(&File::open(&path).unwrap()).unwrap(); + assert!( + found.to_string_lossy().starts_with(r"\\?\Volume{"), + "{}", + found.display() + ); + // The path names the file, which its final path by its drive names too. + let reopened = File::open(&found).unwrap(); + assert_eq!( + final_path(&reopened).unwrap(), + std::fs::canonicalize(&path).unwrap() + ); + } } diff --git a/nvx-rs/src/host/store/lease.rs b/nvx-rs/src/host/store/lease.rs index 1ce7ac867..cb534e9fe 100644 --- a/nvx-rs/src/host/store/lease.rs +++ b/nvx-rs/src/host/store/lease.rs @@ -6,7 +6,9 @@ //! [`template::Store`](crate::host::template::Store), grants and ends leases, and reads its own //! records, with these, and [`template`](crate::host::template) re-exports what is public here. +use std::ffi::OsStr; use std::fmt; +use std::fs::File; use std::io; use std::num::NonZeroU64; use std::path::{Path, PathBuf}; @@ -15,7 +17,7 @@ use std::str::FromStr; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; -use super::regular::{open_regular, read_at_most, was_removed}; +use super::regular::{open_regular, open_regular_in, read_at_most, was_removed}; use super::{CLONE_BINDING_NAME, LaunchId, SandboxStore}; use crate::activation::{Identity, StoreGeneration, TemplateReference}; use crate::error::{Error, ErrorCode, ErrorKind, Result}; @@ -24,6 +26,15 @@ use crate::host::platform::{self, parse_hex_id, parse_hex32}; /// Version of the store's records: its identity record and its index. pub const STORE_FORMAT: u32 = 1; +/// Name of the directory of the generations' payloads. +pub const GENERATIONS_NAME: &str = "generations"; + +/// Name of the store's identity record. +pub const STORE_RECORD_NAME: &str = "store.json"; + +/// Longest identity record, in bytes. +pub(crate) const MAX_STORE_RECORD_LEN: u64 = 4 * 1024; + /// Bytes of a [`StoreId`] and of a [`StorageGeneration`](crate::host::template::StorageGeneration). pub(crate) const ID_LEN: usize = 16; @@ -247,16 +258,152 @@ impl fmt::Debug for Lease { } } +/// Returns the name of the control socket of the VMM of the clone whose launch is `launch`, in the +/// payload of the leased generation: `.sock`. +pub(crate) fn clone_socket_name(launch: &LaunchId) -> String { + format!("{launch}.sock") +} + +/// Returns whether `path` names the control socket of the VMM of the clone that holds `lease`, as +/// [`template::Store::clone_socket_path`](crate::host::template::Store::clone_socket_path) returns +/// it: `generations//.sock`, in a template store, which a lease names by its +/// identity alone. +pub(crate) fn is_clone_socket(lease: &Lease, path: &Path) -> bool { + let name = clone_socket_name(&lease.launch); + let generation = lease.generation.to_string(); + let payload = path.parent(); + path.file_name() == Some(OsStr::new(&name)) + && payload.and_then(Path::file_name) == Some(OsStr::new(&generation)) + && payload.and_then(Path::parent).and_then(Path::file_name) + == Some(OsStr::new(GENERATIONS_NAME)) +} + +/// Opens the payload of the generation `generation` of the template store `id` in `store`, which +/// holds the control sockets of the VMMs of the generation's clones on Linux: +/// `generations/`, for reading, so that the caller can sync it once it removed a +/// socket from it. +/// +/// It follows no link at the store's own name, at the directory of the generations, or at the +/// payload, and fails if one of them is a link, or not a directory, so that a link that takes the +/// place of one of them never leads a removal to the files of another directory. It reads the +/// store's identity through the handle of the store's directory, failing if it cannot read the +/// identity record, and returns `None` if that directory holds another store, or none, as it does +/// if the store, its directory of the generations, or the payload is gone: the payload of a store +/// that is gone holds no socket. It opens the store's directory and the directory of the +/// generations with the right to search them alone, which an owner-only directory may grant +/// without the right to read it. +pub(crate) fn open_payload( + store: &Path, + id: StoreId, + generation: StoreGeneration, +) -> Result> { + let payload = store.join(GENERATIONS_NAME).join(generation.to_string()); + let failed = || { + store_error(format!( + "cannot open the payload of the generation {}", + payload.display() + )) + }; + let Some(dir) = platform::Directory::search(store).map_err(failed())? else { + return Ok(None); + }; + if read_identity_in(&dir, store)? != Some(id) { + return Ok(None); + } + let Some(generations) = dir.search_dir(GENERATIONS_NAME).map_err(failed())? else { + return Ok(None); + }; + generations + .open_dir(&generation.to_string()) + .map_err(failed()) +} + +/// Reads the identity of the store in `dir` from its identity record, or returns `None` if the +/// directory holds none. +pub(crate) fn read_identity(dir: &Path) -> Result> { + let path = dir.join(STORE_RECORD_NAME); + read_identity_with(&path, || open_existing(&path)) +} + +/// Reads the identity of the store whose directory `directory` holds open, in `dir`, as +/// [`read_identity`] reads it at its path, or returns `None` if the directory holds no identity +/// record, or, on Windows, one whose deletion is pending: through the handle, so that it reads +/// the identity of the directory that the handle holds, whatever `dir` names meanwhile. +pub(crate) fn read_identity_in( + directory: &platform::Directory, + dir: &Path, +) -> Result> { + read_identity_with(&dir.join(STORE_RECORD_NAME), || { + open_regular_in(directory, STORE_RECORD_NAME) + }) +} + +/// Reads the identity of a store from its identity record at `path`, which `open` opens, or +/// returns `None` if `open` finds none. +fn read_identity_with( + path: &Path, + open: impl FnOnce() -> io::Result>, +) -> Result> { + let description = "identity record of the template store"; + let Some(record) = + read_record_with::(path, description, MAX_STORE_RECORD_LEN, open)? + else { + return Ok(None); + }; + record.store.parse().map(Some).map_err(|error: Error| { + Error::io(format!( + "the {description} {} is malformed: {}", + path.display(), + error.message() + )) + }) +} + +/// The identity record of a store, as JSON. +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct StoreRecord { + /// [`STORE_FORMAT`]. + pub(crate) format: u32, + /// The store's identity. + pub(crate) store: String, +} + /// Reads the clone binding of the sandbox in `dir`, or returns `None` if it holds none, or if the /// directory does not exist. pub(crate) fn read_binding(dir: &Path) -> Result> { read_lease(&dir.join(CLONE_BINDING_NAME), BINDING_DESCRIPTION) } +/// Reads the clone binding of the sandbox whose directory `directory` holds open, in `dir`, as +/// [`read_binding`] reads it at its path, or returns `None` if the directory holds none, or, on +/// Windows, one whose deletion is pending, which is removed: through the handle, so that it reads +/// the binding of the directory that the handle holds, whatever `dir` names meanwhile. +pub(crate) fn read_binding_in( + directory: &platform::Directory, + dir: &Path, +) -> Result> { + read_lease_with(&dir.join(CLONE_BINDING_NAME), BINDING_DESCRIPTION, || { + open_regular_in(directory, CLONE_BINDING_NAME) + }) +} + /// Reads the record of a lease at `path`, its `description`, or returns `None` if it does not /// exist. pub(crate) fn read_lease(path: &Path, description: &str) -> Result> { - let Some(record) = read_record::(path, description, MAX_LEASE_RECORD_LEN)? else { + read_lease_with(path, description, || open_existing(path)) +} + +/// Reads the record of a lease, its `description` at `path`, from the file that `open` opens, or +/// returns `None` if `open` finds none. +fn read_lease_with( + path: &Path, + description: &str, + open: impl FnOnce() -> io::Result>, +) -> Result> { + let Some(record) = + read_record_with::(path, description, MAX_LEASE_RECORD_LEN, open)? + else { return Ok(None); }; record.into_lease().map(Some).map_err(|problem| { @@ -363,13 +510,32 @@ pub(crate) fn store_error(context: String) -> impl FnOnce(io::Error) -> Error { /// Windows, a file's own access control list decides who may open it, whatever the store's /// directory allows. pub(crate) fn read_private(path: &Path, description: &str, limit: u64) -> Result>> { - let failed = || store_error(format!("cannot read the {description} {}", path.display())); - let file = match open_regular(path) { - Ok(file) => file, + read_private_with(path, description, limit, || open_existing(path)) +} + +/// Opens the regular file at `path`, as [`open_regular`] does, or returns `None` if it does not +/// exist, or was removed as this opened it, as [`was_removed`] tells. +fn open_existing(path: &Path) -> io::Result> { + match open_regular(path) { + Ok(file) => Ok(Some(file)), Err(error) if error.kind() == io::ErrorKind::NotFound || was_removed(path, &error) => { - return Ok(None); + Ok(None) } - Err(error) => return Err(failed()(error)), + Err(error) => Err(error), + } +} + +/// Reads the private regular file that `open` opens, the store's `description` at `path`, of at +/// most `limit` bytes, as [`read_private`] reads it, or returns `None` if `open` finds none. +fn read_private_with( + path: &Path, + description: &str, + limit: u64, + open: impl FnOnce() -> io::Result>, +) -> Result>> { + let failed = || store_error(format!("cannot read the {description} {}", path.display())); + let Some(file) = open().map_err(failed())? else { + return Ok(None); }; if let Some(problem) = platform::private_file_problem(&file).map_err(failed())? { return Err(Error::io(format!( @@ -389,7 +555,18 @@ pub(crate) fn read_record( description: &str, limit: u64, ) -> Result> { - let Some(text) = read_private(path, description, limit)? else { + read_record_with(path, description, limit, || open_existing(path)) +} + +/// Reads the store's record that `open` opens, its `description` at `path`, as [`read_record`] +/// reads it, or returns `None` if `open` finds none. +fn read_record_with( + path: &Path, + description: &str, + limit: u64, + open: impl FnOnce() -> io::Result>, +) -> Result> { + let Some(text) = read_private_with(path, description, limit, open)? else { return Ok(None); }; let malformed = || format!("the {description} {} is malformed", path.display()); diff --git a/nvx-rs/src/host/store/lifecycle.rs b/nvx-rs/src/host/store/lifecycle.rs index 0310337b3..0dedfb0c8 100644 --- a/nvx-rs/src/host/store/lifecycle.rs +++ b/nvx-rs/src/host/store/lifecycle.rs @@ -11,8 +11,8 @@ use std::time::Instant; use super::caller::{is_report_staging, is_store_temporary}; use super::{ - CLONE_BINDING_NAME, CallerFiles, FILE_NAMES, JOURNAL_NAME, LOCK_NAME, LockedStore, - RUNTIME_NAME, Races, SandboxStore, Taken, absolute, pause, transition_in_progress, + CLONE_BINDING_NAME, CLONE_FILE_NAMES, CallerFiles, FILE_NAMES, JOURNAL_NAME, LOCK_NAME, + LockedStore, RUNTIME_NAME, Races, SandboxStore, Taken, absolute, pause, transition_in_progress, }; use crate::error::{Error, ErrorCode, ErrorKind, Result}; use crate::host::paths::not_unicode; @@ -246,9 +246,8 @@ impl LockedStore { return; } let path = self.path(LOCK_NAME); - let held = platform::opened_file_identity(&self.lock); - if matches!((held, platform::file_identity(&path)), (Ok(held), Ok(current)) if held == current) - { + let held = platform::opened_file_id(&self.lock); + if matches!((held, platform::file_id(&path)), (Ok(held), Ok(current)) if held == current) { let _ = fs::remove_file(&path); } } @@ -269,32 +268,32 @@ impl LockedStore { /// [`supervisor::reattach`](crate::host::supervisor::reattach) does. It also refuses, with /// [`RemovalError::Io`], and removes nothing, once the directory that its path names holds /// another lock file than this lock's, as after a link in the path changed. It makes both - /// checks, and removes the sandbox's [clone binding](CLONE_BINDING_NAME), if it holds one, - /// through one handle of the directory that it opens, so that no change of a link in the path - /// meanwhile redirects the binding's removal to another sandbox, whose VM may run. It makes - /// the binding's absence durable before anything else goes, so that no crash after the - /// removal of the directory, or of one of its ancestors, brings back a binding whose lease a - /// retirement may have ended. It then removes the store's files but its lock, the caller's - /// files, its configuration record last, so that an interrupted removal leaves a provisioned - /// sandbox, which can be removed again, and the temporary files that the store owns. It - /// refuses the entries that are neither the store's nor the caller's, which it names in the - /// order in which the directory lists them, and keeps them and the directory; the lock file - /// goes as the lock is released, since the configuration record is gone. Otherwise it removes - /// its lock file while it still holds the lock, so that a transition that waited for the lock - /// finds the sandbox gone rather than halfway removed, and then the directory, as long as the - /// path still names it. + /// checks, and removes what a clone kept of its launch, as a release does, and the sandbox's + /// [clone binding](CLONE_BINDING_NAME), if it holds one, through one handle of the directory + /// that it opens, so that no change of a link in the path meanwhile redirects their removal + /// to another sandbox, whose VM may run. It makes their absence durable before anything else + /// goes, so that no crash after the removal of the directory, or of one of its ancestors, + /// brings back a binding whose lease a retirement may have ended. It then removes the store's + /// other files but its lock, the caller's files, its configuration record last, so that an + /// interrupted removal leaves a provisioned sandbox, which can be removed again, and the + /// temporary files that the store owns. It refuses the entries that are neither the store's + /// nor the caller's, which it names in the order in which the directory lists them, and keeps + /// them and the directory; the lock file goes as the lock is released, since the + /// configuration record is gone. Otherwise it removes its lock file while it still holds the + /// lock, so that a transition that waited for the lock finds the sandbox gone rather than + /// halfway removed, and then the directory, as long as the path still names it. /// - /// Lock files of other transitions, a clone binding whose deletion is pending, and on NFS other - /// removed files that handles still hold, can keep the emptied directory, and are not refused: - /// NFS renames a removed file that is still open to `.nfs*`; a Windows file system without - /// POSIX deletion keeps a removed lock file, or a removed binding that a reader holds, under - /// its own name until its last handle closes; and a transition that opens the lock file - /// between the removals of the lock file and of the directory creates a new one. Each of those - /// transitions finds no sandbox, and removes its lock file as it releases its lock, and the - /// handles of other removed files close, so the directory empties. The removal then holds the - /// directory's identity, releases the lock, and returns the [`Lingering`] removal, whose - /// [`wait`](Lingering::wait) removes the directory once it empties. It returns `None` once the - /// directory is gone, whoever removed it. + /// Lock files of other transitions, a clone binding or a file that the clone kept of its launch + /// whose deletion is pending, and on NFS other removed files that handles still hold, can keep + /// the emptied directory, and are not refused: NFS renames a removed file that is still open to + /// `.nfs*`; a Windows file system without POSIX deletion keeps a removed lock file, or a + /// removed binding or clone's file that a reader holds, under its own name until its last + /// handle closes; and a transition that opens the lock file between the removals of the lock + /// file and of the directory creates a new one. Each of those transitions finds no sandbox, and + /// removes its lock file as it releases its lock, and the handles of other removed files close, + /// so the directory empties. The removal then holds the directory's identity, releases the + /// lock, and returns the [`Lingering`] removal, whose [`wait`](Lingering::wait) removes the + /// directory once it empties. It returns `None` once the directory is gone, whoever removed it. pub fn remove(self) -> std::result::Result, RemovalError> { self.remove_contents()?; self.remove_directory() @@ -302,8 +301,9 @@ impl LockedStore { /// Empties the directory but for lingering files, its own lock file included, as /// [`remove`](Self::remove) does first: it refuses without the caller's configuration record, - /// removes the clone binding as [`unbind`](Self::unbind) does, then the store's files and the - /// caller's, and the temporary files that the store owns, and refuses any other entry. + /// removes what a clone kept of its launch and the clone binding as [`unbind`](Self::unbind) + /// does, then the store's other files and the caller's, and the temporary files that the store + /// owns, and refuses any other entry. fn remove_contents(&self) -> std::result::Result<(), RemovalError> { self.remove_contents_with(&|| {}, &platform::Directory::sync) } @@ -319,10 +319,14 @@ impl LockedStore { return Err(RemovalError::Undeclared); } self.unbind(checked, sync)?; - // The binding is removed once: one whose deletion is pending refuses a second removal. + // What a clone kept of its launch, and the binding, go once, through the directory's + // handle: one whose deletion is pending refuses a second removal, and its path may name + // another sandbox meanwhile. let owned: Vec = FILE_NAMES .into_iter() - .filter(|&name| name != LOCK_NAME && name != CLONE_BINDING_NAME) + .filter(|&name| { + name != LOCK_NAME && name != CLONE_BINDING_NAME && !CLONE_FILE_NAMES.contains(&name) + }) .chain(self.files.names()) .map(|name| self.path(name)) .collect(); @@ -335,9 +339,9 @@ impl LockedStore { })?; let mut foreign = Vec::new(); for name in listed { - // Lock files, a binding whose deletion is pending, and removed files that NFS keeps - // while handles hold them, go once their holders release them, which the removal - // waits for, if they keep the directory. + // Lock files, a binding or a clone's file whose deletion is pending, and removed files + // that NFS keeps while handles hold them, go once their holders release them, which + // the removal waits for, if they keep the directory. if lingers(self.dir(), &name) { continue; } @@ -354,17 +358,18 @@ impl LockedStore { } } - /// Removes the sandbox's clone binding, if it holds one, and makes its absence durable with - /// `sync`, before anything else of the sandbox goes, through one handle of the directory that - /// the path names. Through that handle, it checks first that the directory holds this lock's - /// file, and refuses otherwise, then that the store records no VM and no launch, and runs - /// `checked` once both checks pass, before it removes the binding. No change of a link in the - /// path meanwhile redirects these steps to another sandbox, whose VM may run, and whose - /// binding then stays. + /// Removes what the clone of the sandbox kept of its launch, as a release does, and then the + /// sandbox's clone binding, if it holds one, and makes their absence durable with `sync`, + /// before anything else of the sandbox goes, through one handle of the directory that the path + /// names. Through that handle, it checks first that the directory holds this lock's file, and + /// refuses otherwise, then that the store records no VM and no launch, and runs `checked` once + /// both checks pass, before it removes anything. No change of a link in the path meanwhile + /// redirects these steps to another sandbox, whose VM may run, and whose files then stay. /// /// The binding's absence, after this removal or after a release that a crash interrupted /// before its sync, is then durable while the directory, which keeps the sandbox provisioned, - /// remains. A binding whose deletion is pending already counts as removed. + /// remains. A binding, or a file of the clone, whose deletion is pending already counts as + /// removed. fn unbind( &self, checked: &dyn Fn(), @@ -380,9 +385,9 @@ impl LockedStore { move |error: io::Error| RemovalError::Io { path, error } }; let directory = platform::Directory::open(dir).map_err(failed)?; - let held = platform::opened_file_identity(&self.lock).map_err(entry_failed(LOCK_NAME))?; + let held = platform::opened_file_id(&self.lock).map_err(entry_failed(LOCK_NAME))?; if directory - .entry_identity(LOCK_NAME) + .entry_id(LOCK_NAME) .map_err(entry_failed(LOCK_NAME))? != Some(held) { @@ -392,7 +397,7 @@ impl LockedStore { } for name in [JOURNAL_NAME, RUNTIME_NAME] { if directory - .entry_identity(name) + .entry_id(name) .map_err(entry_failed(name))? .is_some() { @@ -400,6 +405,11 @@ impl LockedStore { } } checked(); + // What a clone kept of its launch goes before its binding, as a release removes it, so + // that the binding's durable absence makes theirs durable too. + for name in CLONE_FILE_NAMES { + directory.remove_file(name).map_err(entry_failed(name))?; + } directory .remove_file(CLONE_BINDING_NAME) .map_err(entry_failed(CLONE_BINDING_NAME))?; @@ -489,8 +499,9 @@ impl Lingering { /// appears in it, as a provision that took the lock after the removal released it writes it, /// which it looks up by its path, so that a file system that ignores case finds it whatever /// the case of the name that the provision gave it, with [`RemovalError::Foreign`] for other - /// entries than lock files, a clone binding whose deletion is pending, and the removed files - /// that NFS keeps, sorted, and with [`RemovalError::TimedOut`] once the deadline passes. + /// entries than lock files, a clone binding or a clone's file whose deletion is pending, and + /// the removed files that NFS keeps, sorted, and with [`RemovalError::TimedOut`] once the + /// deadline passes. pub fn wait(&mut self, deadline: Option) -> std::result::Result<(), RemovalError> { loop { let error = match remove_held_directory(&self.dir, self.identity) { @@ -651,12 +662,13 @@ fn remove_entry(path: PathBuf) -> std::result::Result<(), RemovalError> { /// a handle still keeps, which goes once its holder releases or closes it: NFS renames a removed /// file that is still open to `.nfs*`, and Windows file systems without POSIX deletion keep a /// removed lock file under its own name, as they keep the lock files that waiting transitions -/// create until they release them, and a removed clone binding until its readers close it, whose -/// deletion is then pending. +/// create until they release them, and a removed clone binding, or a removed file that the clone +/// kept of its launch, until its readers close it, whose deletion is then pending. fn lingers(dir: &Path, name: &str) -> bool { name == LOCK_NAME || name.starts_with(".nfs") - || (name == CLONE_BINDING_NAME && platform::is_deleted(&dir.join(name))) + || ((name == CLONE_BINDING_NAME || CLONE_FILE_NAMES.contains(&name)) + && platform::is_deleted(&dir.join(name))) } /// Returns whether `error` is how the removal of a directory reports one that is not empty. diff --git a/nvx-rs/src/host/store/lifecycle/tests.rs b/nvx-rs/src/host/store/lifecycle/tests.rs index cdafeaaf7..ee248e2f6 100644 --- a/nvx-rs/src/host/store/lifecycle/tests.rs +++ b/nvx-rs/src/host/store/lifecycle/tests.rs @@ -11,7 +11,8 @@ use std::time::Duration; use super::*; use crate::host::store::{ - CAPABILITY_NAME, CLONE_SCRATCH_NAME, CONSOLE_NAME, LOG_NAME, OUTCOME_NAME, SOCKET_NAME, + CAPABILITY_NAME, CLONE_ACTIVATION_NAME, CLONE_SCRATCH_NAME, CONSOLE_NAME, LOG_NAME, + OUTCOME_NAME, RESTORE_READY_NAME, SOCKET_NAME, }; use crate::host::test_support::released; @@ -324,6 +325,8 @@ fn a_removal_removes_the_store_the_callers_files_and_the_directory() { SOCKET_NAME, CLONE_BINDING_NAME, CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, BOOT, ".runtime.json.12345.tmp", ".sandbox.lock.7.tmp", @@ -408,10 +411,23 @@ fn a_removal_is_refused_while_a_vm_or_a_launch_is_recorded() { } } +/// Writes the files that a clone keeps of its launch in the sandbox `dir`, its binding included. +fn write_clone_files(dir: &Path) { + for name in [ + CLONE_BINDING_NAME, + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, + ] { + fs::write(dir.join(name), b"{}").unwrap(); + } +} + #[test] -fn a_removal_makes_the_absence_of_the_clone_binding_durable_before_it_removes_anything_else() { +fn a_removal_makes_the_absence_of_the_clones_files_durable_before_it_removes_anything_else() { let (_directory, dir) = sandbox(); - for name in [CLONE_BINDING_NAME, LOG_NAME, BOOT] { + write_clone_files(&dir); + for name in [LOG_NAME, BOOT] { fs::write(dir.join(name), b"{}").unwrap(); } let locked = SandboxStore::lock_existing(&dir, &files(), None).unwrap(); @@ -419,18 +435,18 @@ fn a_removal_makes_the_absence_of_the_clone_binding_durable_before_it_removes_an locked .remove_contents_with(&|| {}, &|directory| { // The directory that the sync makes durable holds the sandbox's lock file. - let lock = directory.entry_identity(LOCK_NAME).unwrap(); + let lock = directory.entry_id(LOCK_NAME).unwrap(); synced.borrow_mut().push((lock, entries_of(&dir))); Ok(()) }) .unwrap(); // A crash after the sync, even once the directory or one of its ancestors is gone, cannot - // bring the binding back, and one before the configuration record goes leaves a provisioned - // sandbox, whose removal can be retried. + // bring the binding or the clone's files back, and one before the configuration record goes + // leaves a provisioned sandbox, whose removal can be retried. assert_eq!( synced.into_inner(), [( - Some(platform::file_identity(&dir.join(LOCK_NAME)).unwrap()), + Some(platform::file_id(&dir.join(LOCK_NAME)).unwrap()), sorted(&[BOOT, CONFIG, LOCK_NAME, LOG_NAME]) )] ); @@ -440,11 +456,10 @@ fn a_removal_makes_the_absence_of_the_clone_binding_durable_before_it_removes_an } #[test] -fn a_removal_that_cannot_make_the_absence_of_the_clone_binding_durable_keeps_the_sandbox() { +fn a_removal_that_cannot_make_the_absence_of_the_clones_files_durable_keeps_the_sandbox() { let (_directory, dir) = sandbox(); - for name in [CLONE_BINDING_NAME, LOG_NAME] { - fs::write(dir.join(name), b"{}").unwrap(); - } + write_clone_files(&dir); + fs::write(dir.join(LOG_NAME), b"{}").unwrap(); let locked = SandboxStore::lock_existing(&dir, &files(), None).unwrap(); let error = locked .remove_contents_with(&|| {}, &|_| Err(io::Error::other("the sync failed"))) @@ -455,7 +470,8 @@ fn a_removal_that_cannot_make_the_absence_of_the_clone_binding_durable_keeps_the ); assert_eq!(Error::from(error).kind(), ErrorKind::Io); drop(locked); - // Only the binding went, so the sandbox is still provisioned, and its removal can be retried. + // Only the clone's files went, so the sandbox is still provisioned, and its removal can be + // retried. assert_eq!(entries_of(&dir), sorted(&[CONFIG, LOCK_NAME, LOG_NAME])); assert!(remove(&dir).unwrap().is_none()); assert!(!dir.exists()); @@ -501,24 +517,40 @@ fn relink(target: &Path, link: &Path) { } /// Provisions the sandboxes `one/sandbox` and `two/sandbox` in `root`, each holding a clone -/// binding, links `root/link` to `one`, and returns `one`, `two`, and the link. +/// binding and the files that a clone keeps of its launch, links `root/link` to `one`, and returns +/// `one`, `two`, and the link. fn linked_sandboxes(root: &Path) -> (PathBuf, PathBuf, PathBuf) { let (one, two, link) = (root.join("one"), root.join("two"), root.join("link")); for parent in [&one, &two] { fs::create_dir(parent).unwrap(); let sandbox = parent.join("sandbox"); provision(&sandbox); - fs::write(sandbox.join(CLONE_BINDING_NAME), b"{}").unwrap(); + write_clone_files(&sandbox); } link_dir(&one, &link); (one, two, link) } +/// The entries of a provisioned sandbox that [`linked_sandboxes`] made, with `more`, sorted. +fn linked_entries(more: &[&str]) -> Vec { + let mut names = vec![ + CLONE_ACTIVATION_NAME, + CLONE_BINDING_NAME, + CLONE_SCRATCH_NAME, + CONFIG, + LOCK_NAME, + RESTORE_READY_NAME, + ]; + names.extend_from_slice(more); + sorted(&names) +} + #[test] fn a_removal_unbinds_the_sandbox_whose_lock_it_holds_whatever_its_path_names_meanwhile() { let directory = tempfile::tempdir().unwrap(); let (one, two, link) = linked_sandboxes(directory.path()); - // The other sandbox records a launch, so its VM may run, and its binding must stay. + // The other sandbox records a launch, so its VM may run, and its binding and the files of its + // clone's launch must stay. fs::write(two.join("sandbox").join(JOURNAL_NAME), b"{}").unwrap(); let locked = SandboxStore::lock_existing(link.join("sandbox"), &files(), None).unwrap(); // The link names the other sandbox's parent from the checks to the binding's removal, and @@ -532,7 +564,7 @@ fn a_removal_unbinds_the_sandbox_whose_lock_it_holds_whatever_its_path_names_mea assert_eq!(entries_of(&one.join("sandbox")), [LOCK_NAME]); assert_eq!( entries_of(&two.join("sandbox")), - sorted(&[CLONE_BINDING_NAME, CONFIG, JOURNAL_NAME, LOCK_NAME]) + linked_entries(&[JOURNAL_NAME]) ); assert!(locked.remove_directory().unwrap().is_none()); assert!(!one.join("sandbox").exists()); @@ -554,10 +586,7 @@ fn a_removal_through_a_path_that_names_another_sandbox_meanwhile_removes_nothing ); assert_eq!(Error::from(error).kind(), ErrorKind::Io); for parent in [&one, &two] { - assert_eq!( - entries_of(&parent.join("sandbox")), - sorted(&[CLONE_BINDING_NAME, CONFIG, LOCK_NAME]) - ); + assert_eq!(entries_of(&parent.join("sandbox")), linked_entries(&[])); } } @@ -1139,6 +1168,59 @@ mod windows { assert!(!dir.exists()); } + #[test] + fn a_removal_takes_the_clone_files_whose_deletion_is_pending_for_removed() { + let (_directory, dir) = sandbox(); + fs::write(dir.join(CLONE_BINDING_NAME), b"{}").unwrap(); + for name in CLONE_FILE_NAMES { + fs::write(dir.join(name), b"").unwrap(); + } + // The removal of what a clone kept of its launch leaves the deletion of each file that a + // reader holds pending, which the removal's own removal of them then finds. + let pending: Vec = CLONE_FILE_NAMES + .into_iter() + .map(|name| delete_pending(&dir.join(name))) + .collect(); + let mut lingering = remove(&dir) + .unwrap() + .expect("the files whose deletion is pending keep the directory"); + // Everything else went, the binding included, and the clone's files linger, as a removed + // lock file does, rather than being refused as entries that are not the store's. + assert_eq!(entries_of(&dir), sorted(&CLONE_FILE_NAMES)); + let error = lingering.wait(Some(Instant::now())).unwrap_err(); + assert!(matches!(error, RemovalError::TimedOut(_)), "{error}"); + drop(pending); + lingering.wait(within(10)).unwrap(); + assert!(!dir.exists()); + } + + #[test] + fn a_retried_removal_of_the_clone_files_takes_those_whose_deletion_is_pending_for_removed() { + let (_directory, dir) = sandbox(); + for name in CLONE_FILE_NAMES { + fs::write(dir.join(name), b"").unwrap(); + } + // A release that failed once it removed the clone's files, which readers hold, retries + // their removal, which finds each one's deletion pending. + let pending: Vec = CLONE_FILE_NAMES + .into_iter() + .map(|name| delete_pending(&dir.join(name))) + .collect(); + let locked = SandboxStore::lock_existing(&dir, &files(), None).unwrap(); + locked + .remove_clone_files(&locked.open_dir().unwrap()) + .unwrap(); + drop(locked); + drop(pending); + let names = entries_of(&dir); + assert!( + names + .iter() + .all(|name| !CLONE_FILE_NAMES.contains(&name.as_str())), + "{names:?}" + ); + } + #[test] fn locks_refuse_a_lock_file_that_refuses_access_for_good() { // A read-only lock file, and a directory at its path, refuse an open for writing as a diff --git a/nvx-rs/src/host/store/mod.rs b/nvx-rs/src/host/store/mod.rs index 0bef346a1..d6a66e3c8 100644 --- a/nvx-rs/src/host/store/mod.rs +++ b/nvx-rs/src/host/store/mod.rs @@ -8,7 +8,8 @@ //! launch.json the journal of a launch in progress, or of one that was interrupted //! runtime.json the runtime record of its running VM //! control.capability the launch capability of its VM -//! control.sock the control endpoint of its VM, on Linux +//! control.sock the control endpoint of its VM, on Linux, but for a clone, whose VMM +//! listens beside the snapshot that it restores //! openvmm.log the VMM's output, claimed by the launch that created it //! console.log the guest console of the latest launch, or of every launch, if the //! caller captures it @@ -17,6 +18,9 @@ //! clone.json the binding of a clone to the generation of a template that it leases //! clone-scratch.raw the file of the scratch device of a clone whose upper is in RAM, a //! sparse file that the clone's plan creates +//! clone-activation.bin the activation record of a clone, which its launch writes +//! restore.sock the endpoint at which the launch of a clone awaits the restore-ready +//! event of its VMM, on Linux //! ... the caller's configuration record and other files, which it declares //! ``` //! @@ -141,7 +145,8 @@ //! the VMM through the control endpoint that the journal names, holds it, confirms that the //! endpoint still names it once held, and terminates it. //! On Windows the endpoint is a pipe whose name each launch draws at random, which names the -//! VMM by itself. On Linux it is the same socket path for every launch of the sandbox, so the +//! VMM by itself. On Linux it is the same socket path for every launch of the sandbox, but the +//! one launch of a clone, whose VMM listens at a socket that its lease reserves, so the //! VMM must also hold the claimed log as its standard output. Until the VMM serves its //! endpoint, recovery is indeterminate and keeps the journal; once the VMM exits, the released //! claim proves it. @@ -197,12 +202,13 @@ //! # Removal //! //! [`LockedStore::remove`] removes a sandbox that no VMM can run in: it refuses while the store -//! records a VM or a launch, which the caller reattaches to, or recovers, first. It removes the -//! sandbox's [clone binding](CLONE_BINDING_NAME) first, and makes its absence durable before -//! anything else goes. It checks for those records, and removes the binding, through one handle of -//! the directory, once it has checked through that handle that the directory holds its lock, so -//! that no change of a link in the path meanwhile makes it remove another sandbox's binding. It -//! then removes the store's files and the caller's, the configuration record last, and the +//! records a VM or a launch, which the caller reattaches to, or recovers, first. It removes what a +//! clone kept of its launch, as a release does, and the sandbox's +//! [clone binding](CLONE_BINDING_NAME) first, and makes their absence durable before anything +//! else goes. It checks for those records, and removes those files, through one handle of the +//! directory, once it has checked through that handle that the directory holds its lock, so that +//! no change of a link in the path meanwhile makes it remove another sandbox's files. It then +//! removes the store's other files and the caller's, the configuration record last, and the //! temporary files that the store owns: those that an interrupted replacement of one of its files //! leaves, `.NAME.NUMBER.tmp`, and the VMM's [report staging file](#the-report-staging-file). It //! then refuses entries that are neither the store's nor the caller's, naming them, and keeps the @@ -245,7 +251,7 @@ use self::files::{io_error, remove_if_present}; pub use self::identity::{ ArtifactIdentity, IdentityCheck, IdentityPart, MAX_IDENTITY_LEN, RuntimeIdentity, }; -use self::lease::Lease; +use self::lease::{Lease, clone_socket_name, is_clone_socket, open_payload}; pub use self::lifecycle::{Lingering, RemovalError}; pub use self::record::{ LaunchId, LaunchJournal, LaunchState, LogClaim, MAX_ENDPOINT_LEN, RuntimeRecord, @@ -271,7 +277,10 @@ pub const JOURNAL_NAME: &str = "launch.json"; pub const RUNTIME_NAME: &str = "runtime.json"; /// Name of the file that holds the launch capability. pub const CAPABILITY_NAME: &str = "control.capability"; -/// Name of the Linux control socket. +/// Name of the Linux control socket, at which the VMM of every launch of the sandbox listens, but +/// that of a clone, which listens beside the snapshot that it restores, at the socket that its +/// lease reserves, as [`template::Store::acquire`](crate::host::template::Store::acquire) +/// describes. pub const SOCKET_NAME: &str = "control.sock"; /// Name of the VMM's log. pub const LOG_NAME: &str = "openvmm.log"; @@ -293,11 +302,33 @@ pub const CLONE_BINDING_NAME: &str = "clone.json"; /// [`template::Store::release`](crate::host::template::Store::release) removes once the clone's VM /// has exited, as a [removal](self#removal) of the sandbox does. pub const CLONE_SCRATCH_NAME: &str = "clone-scratch.raw"; +/// Name of the activation record of a clone, in its canonical encoding, which the clone's launch, +/// [`template::ClonePlan::launch`](crate::host::template::ClonePlan::launch), writes before it +/// spawns the clone's VMM, so that the restore activates the clone with that record and its +/// transaction whatever happens to the launch's caller, and which +/// [`template::Store::release`](crate::host::template::Store::release) removes once the clone's VM +/// has exited, as a [removal](self#removal) of the sandbox does. +pub const CLONE_ACTIVATION_NAME: &str = "clone-activation.bin"; +/// Name of the Linux socket at which the launch of a clone, +/// [`template::ClonePlan::launch`](crate::host::template::ClonePlan::launch), listens for the +/// restore-ready event of its VMM, which OpenVMM sends once it has prepared the restore, before +/// the restored VM runs; on Windows, the endpoint is a named pipe, whose name each launch draws at +/// random. [`template::Store::release`](crate::host::template::Store::release) removes it once the +/// clone's VM has exited, as a [removal](self#removal) of the sandbox does. +pub const RESTORE_READY_NAME: &str = "restore.sock"; +/// The files that the clone of a sandbox keeps of its launch, which a release removes once the +/// clone's VM has exited, as a [removal](self#removal) of the sandbox does, before the clone +/// binding. +const CLONE_FILE_NAMES: [&str; 3] = [ + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, +]; /// Every file that the store keeps in a sandbox's directory. Replacing a record leaves a /// temporary file named `.NAME.NUMBER.tmp` beside it only if the replacement was interrupted, and /// a VMM that is killed while it writes its outcome report leaves the file in which it stages it; /// see [the report staging file](self#the-report-staging-file). -pub const FILE_NAMES: [&str; 10] = [ +pub const FILE_NAMES: [&str; 12] = [ LOCK_NAME, JOURNAL_NAME, RUNTIME_NAME, @@ -308,6 +339,8 @@ pub const FILE_NAMES: [&str; 10] = [ OUTCOME_NAME, CLONE_BINDING_NAME, CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, ]; /// Interval at which a lock that another holder has is tried again: the sandbox's lock, by a @@ -506,6 +539,17 @@ impl SandboxStore { self.path(CLONE_SCRATCH_NAME) } + /// Returns the path of the activation record of a clone, [`CLONE_ACTIVATION_NAME`]. + pub fn clone_activation_path(&self) -> PathBuf { + self.path(CLONE_ACTIVATION_NAME) + } + + /// Returns the path of the Linux socket at which the launch of a clone awaits the + /// restore-ready event of its VMM, [`RESTORE_READY_NAME`]. + pub fn restore_ready_path(&self) -> PathBuf { + self.path(RESTORE_READY_NAME) + } + /// Reads the runtime record, which names the sandbox's VM from the commit of its launch until /// its runtime is cleared. pub fn runtime(&self) -> Result> { @@ -595,8 +639,8 @@ impl SandboxStore { } }, } - let locked = platform::opened_file_identity(&file).map_err(failed)?; - match platform::file_identity(&path) { + let locked = platform::opened_file_id(&file).map_err(failed)?; + match platform::file_id(&path) { Ok(current) if current == locked => Ok(Taken::Locked(file)), Ok(_) => Ok(Taken::Replaced), Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Taken::Replaced), @@ -648,7 +692,24 @@ pub struct LockedStore { lock_removed: bool, /// The launch of the sandbox's clone that the lease acquired under this lock reserves, until /// [`begin_launch`](Self::begin_launch) begins it, or a release ends the lease. - reserved: Option, + reserved: Option, +} + +/// The launch of a sandbox's clone that a lease reserves, as +/// [`template::Store::acquire`](crate::host::template::Store::acquire) acquires the lease, and +/// where the clone's VMM listens. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Reservation { + /// The identity of the launch, [`Lease::launch`]. + pub(crate) launch: LaunchId, + /// The path of the Linux socket at which the clone's VMM listens, its control endpoint: beside + /// the snapshot that it restores, in the payload of the leased generation, since OpenVMM + /// requires a restore's control socket in the snapshot's parent, rather than the sandbox's + /// own control socket. Windows endpoints are named pipes, which [`begin_launch`] draws as it + /// does for every launch. + /// + /// [`begin_launch`]: LockedStore::begin_launch + pub(crate) socket: PathBuf, } impl Deref for LockedStore { @@ -672,17 +733,28 @@ impl LockedStore { } } - /// Lets [`begin_launch`](Self::begin_launch) begin `launch`, once, as the launch of the - /// sandbox's clone that the lease acquired under this lock reserves, or, given `None`, no - /// launch of a clone, once a release ends the lease. - pub(crate) fn reserve(&mut self, launch: Option) { - self.reserved = launch; + /// Lets [`begin_launch`](Self::begin_launch) begin the launch that `reservation` names, once, + /// as the launch of the sandbox's clone that the lease acquired under this lock reserves, + /// whose VMM listens at the reservation's socket, or, given `None`, no launch of a clone, once + /// a release ends the lease. + pub(crate) fn reserve(&mut self, reservation: Option) { + self.reserved = reservation; } /// Returns the launch of the sandbox's clone that the lease acquired under this lock reserves, /// until [`begin_launch`](Self::begin_launch) begins it, or a release ends the lease. pub(crate) fn reserved(&self) -> Option<&LaunchId> { - self.reserved.as_ref() + self.reserved + .as_ref() + .map(|reservation| &reservation.launch) + } + + /// Returns the path of the Linux socket at which the VMM of the launch that + /// [`reserved`](Self::reserved) returns listens. + pub(crate) fn reserved_socket(&self) -> Option<&Path> { + self.reserved + .as_ref() + .map(|reservation| reservation.socket.as_path()) } /// Returns the paths at which the store reaches the entries of the sandbox's directory that @@ -705,6 +777,53 @@ impl LockedStore { Ok(owned) } + /// Opens the sandbox's directory through its path, as one handle through which a release of + /// the lease of its clone names the directory's entries, and checks through that handle that + /// the directory holds this lock's file, as a [removal](self#removal) checks it: no change of + /// the path meanwhile, of a link in it or of one of its directories, then makes the release + /// reach the entries of another sandbox, whose lock it does not hold. + /// + /// Fails with [`ErrorKind::Other`] if the directory that the path names does not hold this + /// lock's file. + pub(crate) fn open_dir(&self) -> Result { + let dir = self.dir(); + let directory = platform::Directory::open(dir).map_err(io_error(format!( + "cannot open the directory {}", + dir.display() + )))?; + let lock = self.path(LOCK_NAME); + let examine = || io_error(format!("cannot examine {}", lock.display())); + let held = platform::opened_file_id(&self.lock).map_err(examine())?; + if directory.entry_id(LOCK_NAME).map_err(examine())? != Some(held) { + return Err(Error::other(format!( + "the directory that {} names does not hold the lock of its sandbox: the path names \ + another directory than when the lock was taken", + dir.display() + ))); + } + Ok(directory) + } + + /// Removes what the clone of the sandbox kept of its launch in the sandbox's directory, whose + /// VM has exited, through `directory`, the handle of the directory that + /// [`open_dir`](Self::open_dir) returned: the file of its scratch device, + /// [`CLONE_SCRATCH_NAME`], its activation record, [`CLONE_ACTIVATION_NAME`], and the socket of + /// its restore-ready event, [`RESTORE_READY_NAME`]. A release, as a [removal](self#removal) + /// does, removes them before the clone binding, whose removal, once durable, makes theirs + /// durable too, since all are entries of the sandbox's directory. A file whose deletion is + /// pending, as a Windows file system without POSIX deletion leaves a removed file that another + /// handle holds, is removed already, so that a release that failed once it removed the file can + /// be retried. + pub(crate) fn remove_clone_files(&self, directory: &platform::Directory) -> Result<()> { + for name in CLONE_FILE_NAMES { + directory.remove_file(name).map_err(io_error(format!( + "cannot remove {}", + self.path(name).display() + )))?; + } + Ok(()) + } + /// Returns the caller's files that the lock knows: those that a [lifecycle /// lock](self#lifecycle-locks) was taken with, and none for [`SandboxStore::lock`]. pub fn files(&self) -> &CallerFiles { @@ -721,6 +840,12 @@ impl LockedStore { /// of a caller that keeps the console of every launch, as [`CallerFiles::keeping_console`] /// declares. Nothing is spawned yet: the caller spawns the VMM with [`Launch::take_log`]. /// + /// The control endpoint is a named pipe of a random name on Windows. On Linux, it is the + /// sandbox's control socket, [`socket_path`](SandboxStore::socket_path), but for the launch + /// that the lease of the sandbox's clone reserves: its VMM restores the leased generation's + /// snapshot, and listens beside it, at the socket that the lease reserves in the generation's + /// payload, as [`template::Store::acquire`](crate::host::template::Store::acquire) describes. + /// /// Fails if the sandbox's VM runs, or a launch was interrupted and is not recovered yet, and /// with [`ErrorKind::Unsupported`] for an all-zero capability or an invalid runtime identity. /// A sandbox whose clone holds a lease of a template's generation, which its @@ -754,7 +879,7 @@ impl LockedStore { ))); } if let Some(lease) = Lease::of(self)? - && (lease.launch() != launch || self.reserved.as_ref() != Some(launch)) + && (lease.launch() != launch || self.reserved() != Some(launch)) { let reservation = if lease.launch() == launch { format!("whose launch {launch} begins once, under the lock that acquired the lease") @@ -781,7 +906,13 @@ impl LockedStore { remove_if_present(&self.console_path())?; } self.remove_report_staging()?; - let endpoint = platform::control_endpoint(&socket) + // The VMM of a clone listens at the socket that its lease reserves, beside the snapshot + // that it restores. + let listening = match &self.reserved { + Some(reservation) if reservation.launch == *launch => reservation.socket.as_path(), + _ => &socket, + }; + let endpoint = platform::control_endpoint(listening) .map_err(|error| Error::io("cannot choose a control endpoint").with_source(error))?; let (log, claim) = self.create_log()?; files::write_private(&self.capability_path(), capability)?; @@ -854,6 +985,7 @@ impl LockedStore { for name in [CAPABILITY_NAME, SOCKET_NAME] { remove_if_present(&self.path(name))?; } + self.remove_clone_socket(journal.endpoint())?; self.remove_report_staging()?; // The journal goes last, so that an interrupted removal leaves it for a later // recovery to finish. @@ -962,14 +1094,74 @@ impl LockedStore { /// Removes the runtime record, the launch capability, and the control socket of a VM that has /// exited, and any outcome report that its VMM staged without publishing it, the runtime - /// record last. The log, console log, and outcome report outlive the run. + /// record last. The control socket of a clone's VMM, beside the snapshot that it restores, goes + /// too, and the clearing fails, keeping the runtime record, if a link takes the place of the + /// template store's directory, of its directory of the generations, or of the payload of the + /// socket's generation. The log, console log, and outcome report outlive the run. pub fn clear_runtime(&mut self) -> Result<()> { + // Only a Linux endpoint is a file, which may be a clone's beside its snapshot. A record that + // cannot be read names none, and a release removes a clone's socket all the same. + let record = if cfg!(unix) { + self.runtime().ok().flatten() + } else { + None + }; + self.clear_runtime_of(record.as_ref()) + } + + /// Clears the runtime record of a VM that has exited, as [`clear_runtime`](Self::clear_runtime) + /// does, given `record`, the runtime record that the caller read under this lock, if any, whose + /// control endpoint is the VM's. + pub(crate) fn clear_runtime_of(&mut self, record: Option<&RuntimeRecord>) -> Result<()> { for name in [CAPABILITY_NAME, SOCKET_NAME] { remove_if_present(&self.path(name))?; } + if let Some(record) = record { + self.remove_clone_socket(record.endpoint())?; + } self.remove_report_staging()?; remove_if_present(&self.path(RUNTIME_NAME)) } + + /// Removes the control socket at `endpoint`, the control endpoint that a launch of the + /// sandbox recorded, if it is the socket of the sandbox's clone beside the snapshot that its + /// VMM restores, which the lease that the sandbox's clone binding records reserved, as + /// [`template::Store::clone_socket_path`](crate::host::template::Store::clone_socket_path) + /// names it. Any other endpoint stays: the sandbox's own control socket, which the caller + /// removes, and a named pipe. Neither reads the binding. + /// + /// The socket goes through a handle of the generation's payload, which a handle of the + /// template store's directory opens, as a release opens it: following no link at the store's + /// directory, at its directory of the generations, or at the payload, and once it checked + /// through that handle that the directory holds the lease's store, so that neither a link that + /// takes the place of one of them nor a directory that takes the place of the store ever leads + /// the removal to the files of another directory. Such a link fails the removal; a payload + /// that is gone holds no socket, and neither does a directory that holds another store, or + /// none, in the place of the lease's, whose files stay. + /// + /// The removal is not made durable here: + /// [`template::Store::release`](crate::host::template::Store::release) makes it durable before + /// the lease ends, and a retirement removes whatever a crash brings back with the generation's + /// payload. + fn remove_clone_socket(&self, endpoint: &str) -> Result<()> { + let socket = Path::new(endpoint); + if !cfg!(unix) || socket == self.socket_path() { + return Ok(()); + } + let Some(lease) = Lease::of(self)?.filter(|lease| is_clone_socket(lease, socket)) else { + return Ok(()); + }; + // The socket is `/generations//.sock`, as the check found. + let Some(store) = socket.ancestors().nth(3) else { + return Ok(()); + }; + match open_payload(store, lease.store(), lease.generation())? { + Some(payload) => payload + .remove_file(&clone_socket_name(lease.launch())) + .map_err(store_error(format!("cannot remove {}", socket.display()))), + None => Ok(()), + } + } } /// Ends a launch by terminating its verified VMM `process`. diff --git a/nvx-rs/src/host/store/regular.rs b/nvx-rs/src/host/store/regular.rs index 771c18ab0..e85f2845b 100644 --- a/nvx-rs/src/host/store/regular.rs +++ b/nvx-rs/src/host/store/regular.rs @@ -50,19 +50,42 @@ pub(crate) fn open_regular(path: &Path) -> io::Result { const OPEN_REPARSE_POINT: u32 = 0x0020_0000; options.custom_flags(OPEN_REPARSE_POINT); } - let file = match options.open(path) { - Ok(file) => file, - // `O_NOFOLLOW` refuses a link, and an open that does not wait refuses a socket. - #[cfg(unix)] - Err(error) - if [nix::errno::Errno::ELOOP, nix::errno::Errno::ENXIO] - .into_iter() - .any(|errno| error.raw_os_error() == Some(errno as i32)) => - { - return Err(not_regular()); - } - Err(error) => return Err(error), - }; + let file = options.open(path).map_err(refused_open)?; + regular(file) +} + +/// Opens the file `name` of `directory` for reading, as [`open_regular`] opens the file at a +/// path, if it is a regular file, or returns `None` if the directory has no such entry, or, on +/// Windows, one whose deletion is pending, which is removed. The open names the entry through the +/// handle, so that it reaches the entry of the directory that the handle holds, whatever the +/// directory's path names meanwhile. +pub(crate) fn open_regular_in( + directory: &platform::Directory, + name: &str, +) -> io::Result> { + match directory.open_file(name) { + Ok(Some(file)) => regular(file).map(Some), + Ok(None) => Ok(None), + Err(error) => Err(refused_open(error)), + } +} + +/// Returns `error`, with which an open failed, as the error of a file that is not a regular file if +/// the open refused one: on Unix, a link, which `O_NOFOLLOW` refuses, or a socket, which an open +/// that does not wait refuses. +fn refused_open(error: io::Error) -> io::Error { + #[cfg(unix)] + if [nix::errno::Errno::ELOOP, nix::errno::Errno::ENXIO] + .into_iter() + .any(|errno| error.raw_os_error() == Some(errno as i32)) + { + return not_regular(); + } + error +} + +/// Returns `file`, which an open that follows no link opened, if it is a regular file. +fn regular(file: File) -> io::Result { let metadata = file.metadata()?; if !metadata.file_type().is_file() || filesystem::is_reparse_point(&metadata) { return Err(not_regular()); diff --git a/nvx-rs/src/host/store/tests.rs b/nvx-rs/src/host/store/tests.rs index 44300b42d..ef11598a0 100644 --- a/nvx-rs/src/host/store/tests.rs +++ b/nvx-rs/src/host/store/tests.rs @@ -323,8 +323,8 @@ fn a_lock_file_replaced_while_waiting_locks_nothing() { move || { let locked = store.lock(None).unwrap(); // The waiter holds the lock of the file that the path names now. - let current = platform::file_identity(&locked.path(LOCK_NAME)).unwrap(); - let held = platform::opened_file_identity(&locked.lock).unwrap(); + let current = platform::file_id(&locked.path(LOCK_NAME)).unwrap(); + let held = platform::opened_file_id(&locked.lock).unwrap(); assert_eq!(current, held); } }); @@ -401,6 +401,36 @@ fn a_lock_file_that_is_a_link_fails_at_once() { assert_eq!(fs::read(&target).unwrap(), CAPABILITY); } +// Other hosts open no handle of a sandbox's directory, through which a clone's files go. +#[cfg(any(target_os = "linux", windows))] +#[test] +fn the_files_that_a_clone_kept_of_its_launch_go_and_nothing_else() { + let (_directory, store) = store(); + let locked = store.lock(None).unwrap(); + for path in [ + store.clone_scratch_path(), + store.clone_activation_path(), + store.restore_ready_path(), + store.log_path(), + store.binding_path(), + ] { + fs::write(path, b"left").unwrap(); + } + let directory = locked.open_dir().unwrap(); + locked.remove_clone_files(&directory).unwrap(); + let mut left: Vec = fs::read_dir(store.dir()) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + // The binding goes later, once the release or the removal has removed the clone's files. + let mut expected = [CLONE_BINDING_NAME, LOCK_NAME, LOG_NAME]; + expected.sort_unstable(); + assert_eq!(left, expected); + // Nothing to remove is no failure. + locked.remove_clone_files(&directory).unwrap(); +} + #[cfg(unix)] #[test] fn records_and_capabilities_that_are_not_regular_files_fail_at_once() { diff --git a/nvx-rs/src/host/supervisor/mod.rs b/nvx-rs/src/host/supervisor/mod.rs index 1b2bc240e..49299a3e2 100644 --- a/nvx-rs/src/host/supervisor/mod.rs +++ b/nvx-rs/src/host/supervisor/mod.rs @@ -115,12 +115,22 @@ //! upper, and memory target against the generation, and claims the clone's scratch image, if //! it has one, as //! [`ScratchBinding::check`](crate::host::sandbox::scratch::ScratchBinding::check) does. It -//! then runs [`launch`] as the reserved launch, which [`LockedStore::begin_launch`] requires of -//! a sandbox that holds a lease, and begins once, under that lock alone, whatever its outcome. +//! then launches the clone with +//! [`ClonePlan::launch`](crate::host::template::ClonePlan::launch), which checks that the +//! binding records the lease, writes the clone's files in the sandbox, and runs [`launch`] as +//! the reserved launch, which [`LockedStore::begin_launch`] requires of a sandbox that holds a +//! lease, and begins once, under that lock alone, whatever its outcome. The clone's VMM +//! restores the generation's snapshot in place, and on Linux listens beside it, at the control +//! socket that the lease reserves in the generation's payload, named after the launch, which +//! [`Starting::endpoint`] and the runtime record name, as every launch's endpoint, rather than +//! at the sandbox's own socket. //! 3. Once the clone's VM has exited, as a stop, a termination, or a reattach verifies, //! [`Store::release`](crate::host::template::Store::release) ends the lease, under the //! sandbox's lock, taken through the lease's path, -//! [`Lease::sandbox`](crate::host::template::Lease::sandbox), as the acquisition took it. +//! [`Lease::sandbox`](crate::host::template::Lease::sandbox), as the acquisition took it. The +//! VMM removes its control socket as it exits, the stop or the reattach removes the one that an +//! abrupt exit leaves, and the release makes the socket's absence durable before the lease +//! ends. //! //! ## The lock order //! @@ -543,7 +553,7 @@ impl Vmm { .runtime()? .is_some_and(|record| record == self.record) { - locked.clear_runtime()?; + locked.clear_runtime_of(Some(&self.record))?; } Ok(()) } @@ -761,7 +771,7 @@ fn reattach_with( let process = match open(record.process()) { Ok(Some(process)) => process, Ok(None) => { - locked.clear_runtime()?; + locked.clear_runtime_of(Some(&record))?; return Ok(Reattached::Exited { record: Some(record), recovery, diff --git a/nvx-rs/src/host/template/clone.rs b/nvx-rs/src/host/template/clone.rs index adc8ddda4..ac78de75c 100644 --- a/nvx-rs/src/host/template/clone.rs +++ b/nvx-rs/src/host/template/clone.rs @@ -1,11 +1,17 @@ //! The plan of a clone, [`ClonePlan`]: the generation of a template that a clone leased, with the //! clone's own image, upper, and memory target, which the plan checks against the generation //! before the clone's launch, so that a request that the template cannot serve fails before any -//! VMM starts. +//! VMM starts; and the clone's launch, [`ClonePlan::launch`], OpenVMM's restore of the +//! generation's snapshot with what the plan checked. #[cfg(all(test, any(target_os = "linux", windows)))] mod tests; +mod launch; +mod record; + +pub use self::launch::Restoring; + use std::fmt; use std::fs::{self, File}; use std::io; @@ -16,6 +22,7 @@ use super::key::TemplateKey; use super::manifest::BlockGeometry; use super::plan::MEMORY_BLOCK_SIZE; use super::store::{Acquired, Generation, Lease, Store}; +use crate::activation::{ActivationRecord, Journal, OverlayMode, WorkloadView}; use crate::contract; use crate::error::{Error, ErrorKind, Result}; use crate::host::filesystem; @@ -26,7 +33,8 @@ use crate::host::sandbox::distro::{self, CheckedDistro, DistroBinding, DistroFor use crate::host::sandbox::plan::bound_image; use crate::host::sandbox::scratch::{self, CheckedScratch, ScratchBinding}; use crate::host::sandbox::{ - LayerRole, SandboxLayer, SandboxPlan, block_geometry, reject_container_path, reject_disk_path, + LayerRole, SandboxLayer, SandboxPlan, SandboxUpper, block_geometry, reject_container_path, + reject_disk_path, }; use crate::host::store::LockedStore; use crate::host::store::files::{create_new, remove_if_present}; @@ -42,14 +50,20 @@ const SPARSE_WHAT: &str = "scratch device of the clone"; /// What the clone of a sandbox restores of the generation of a template that it leased: the /// adapter's plan of the clone's image and upper, the binding of the image, the binding of the /// scratch image of an upper on one, and the memory target, which [`ClonePlan::new`] checks -/// against the generation. +/// against the generation, and the settings of the clone's activation that the template does not +/// fix, which the clone's [activation record](ClonePlan::activation) names: the workload's view +/// of its VM, the overlay's mode, and the journal policy of a scratch image. #[derive(Debug, Clone)] pub struct CloneRequest { /// The adapter's plan, which [`SandboxPlan::adapter`] makes, of the clone's image, its - /// `distro`, over its upper: in RAM, a tmpfs, as + /// `distro`, an EROFS layer, over its upper: in RAM, a tmpfs, as /// [`SandboxUpper::RAM`](crate::host::sandbox::SandboxUpper::RAM) plans it, or on the /// caller's scratch image. Its network is the one of the template's NIC, which every restore - /// of the template attaches, or none for a template without a NIC. + /// of the template attaches, or none for a template without a NIC: on the NIC's guest + /// network, which [`ClonePlan::new`] checks, and under the NIC's policy, which OpenVMM checks + /// as it restores the NIC, since the template's key names the policy only by OpenVMM's + /// identity of it. OpenVMM refuses to restore a clone under another policy, which ends its + /// launch once it began. pub adapter: SandboxPlan, /// The binding of the image. pub binding: DistroBinding, @@ -62,6 +76,15 @@ pub struct CloneRequest { /// The memory target, in bytes: the RAM of the clone's VM once restored, to which the restore /// expands the template's RAM in whole blocks. pub memory: u64, + /// The view that the clone's workload has of its VM: its hostname, the machine ID that it + /// sees, and the identity in which every execution runs, unless the executions name their + /// own. + pub workload: WorkloadView, + /// The mode of the clone's overlay, which selects its options. + pub overlay: OverlayMode, + /// Whether the filesystem of the scratch image of an upper on one may have a journal. An + /// upper in RAM has none, so the record of a clone in RAM leaves the policy out. + pub journal: Journal, } /// The upper of a clone's plan, with the file of the clone's `scratch` block device, which every @@ -83,9 +106,9 @@ pub enum CloneUpper { } /// The plan of a clone: the generation of a template that the clone leased, with the clone's own -/// image, upper, and memory target, checked against the generation, and a fresh capability, which -/// [`new`](Self::new) makes under the lock of the sandbox that acquired the lease, before the -/// clone's launch. +/// image, upper, and memory target, checked against the generation, the clone's activation +/// record, and a fresh capability, which [`new`](Self::new) makes under the lock of the sandbox +/// that acquired the lease, before the clone's launch. /// /// A restore of a template attaches the template's two block devices, its `distro` and its /// `scratch`, with files of the clone's, which OpenVMM opens itself, by their paths, as @@ -118,20 +141,39 @@ pub enum CloneUpper { /// for the clone, and present the geometry of the template's scratch, its length included, /// which [`CheckedScratch::geometry`] reads through the checked file. At its path, it must be a /// regular file, not a link, nor on Windows any other reparse point, that is the file that the -/// check found, and that no one else may change. The plan holds the checked image, and with it -/// the claim. Nothing reads, formats, or copies the image. +/// check found, and that no one else may change. Nor may it be a file of the snapshot of any of +/// the store's generations, by any path, through a hard link, say: every clone of a generation +/// restores its snapshot in place, while its guest writes its scratch image. So, once claimed, +/// it must be outside the store's directory, wherever the links on its path lead: the store's +/// generations hold their snapshots there, and its captures write theirs there, which their +/// publications make generations' by renaming directories within it, so a file in the store +/// could be, or become as the plan examines it, a file of a snapshot, while nothing that the +/// host does moves a file into the store's directory. And it must have a single name: another, +/// a hard link, could make it a file of a snapshot, of this store or of another, that its +/// location does not show. These checks open nothing in the store, however many generations it +/// holds. They resolve the aliases that the file system makes, links, junctions, directories +/// held open, and other names, but not a mount or a network share that exposes the store's +/// directories elsewhere: a store's directories must not be exposed that way. The plan holds +/// the checked image, and with it the claim. Nothing reads, formats, or copies the image. /// - The memory target must be whole blocks of [`MEMORY_BLOCK_SIZE`], and of the template's own /// memory blocks, from the template's RAM to its RAM capacity. -/// - The capability is fresh: the plan draws it for the clone's launch. +/// - The activation record, which the clone's guest applies in place of the configuration that +/// the template captured, names the restore, with a fresh transaction ID, the clone's image, +/// upper, and settings, the template's network, and the resources that the clone must have, +/// as [`activation`](Self::activation) describes. Its image must be an EROFS layer, and its +/// settings must keep the limits of a record, as [`Configuration::check`] checks them. +/// - The capability is fresh: the plan draws it for the clone's launch, [`launch`](Self::launch). /// /// The plan's checks read the metadata of the clone's files, never their content. /// /// ```no_run +/// use std::path::Path; /// use std::time::{Duration, Instant}; /// +/// use nvx::activation::{Journal, MachineIdMode, OverlayMode, WorkloadView}; /// use nvx::host::sandbox::distro::{DISTRO_POLICY_REVISION, DistroBinding}; /// use nvx::host::sandbox::{SandboxDistro, SandboxPlan, SandboxUpper}; -/// use nvx::host::store::CallerFiles; +/// use nvx::host::store::{CallerFiles, RuntimeIdentity}; /// use nvx::host::template::{ /// CloneRequest, ClonePlan, Generation, HostTime, MEMORY_BLOCK_SIZE, Store, /// }; @@ -141,6 +183,7 @@ pub enum CloneUpper { /// # generation: &Generation, /// # image: SandboxDistro, /// # binding: DistroBinding, +/// # runtime: &RuntimeIdentity, /// # ) -> nvx::Result<()> { /// let files = CallerFiles::new("config.json")?; /// let mut acquired = store.acquire( @@ -156,15 +199,27 @@ pub enum CloneUpper { /// policy_revision: DISTRO_POLICY_REVISION, /// scratch: None, /// memory: 8 * MEMORY_BLOCK_SIZE, +/// workload: WorkloadView { +/// hostname: b"clone".to_vec(), +/// machine_id: MachineIdMode::Image, +/// identity: None, +/// }, +/// overlay: OverlayMode::UntrustedLowers, +/// journal: Journal::Allowed, /// }; /// let deadline = Instant::now() + Duration::from_secs(5); /// let plan = ClonePlan::new(store, &mut acquired, &request, deadline)?; -/// // ... the clone's launch, under `acquired.locked`, as the launch that the lease reserves, of -/// // `plan.snapshot()` with `plan.image().path()` and `plan.scratch_path()` ... -/// # let _ = plan; +/// // OpenVMM restores the generation's snapshot under the runtime that captured the template. +/// let restoring = plan.launch(&mut acquired, Path::new("openvmm"), runtime)?; +/// // ... the restore, whose VMM sends its restore-ready event to `restoring.ready_endpoint()`, +/// // serves its control console at `restoring.starting().endpoint()`, and whose guest applies +/// // `restoring.activation()` ... +/// # let _ = restoring; /// # Ok(()) /// # } /// ``` +/// +/// [`Configuration::check`]: crate::activation::Configuration::check pub struct ClonePlan { /// The lease of the generation. lease: Lease, @@ -180,8 +235,13 @@ pub struct ClonePlan { upper: CloneUpper, /// The memory target, in bytes. memory: u64, + /// The clone's activation record. + activation: ActivationRecord, /// The capability of the clone's launch. capability: [u8; CAPABILITY_LEN], + /// The clone's devices, which its restore attaches: the adapter's plan of its image and + /// network, over the file of its `scratch` block device. + devices: SandboxPlan, } /// How a plan reaches the clone's files: as the checks of their bindings reach them, and as it @@ -213,7 +273,7 @@ impl ClonePlan { /// /// The plan is made under the lock that acquired the lease, which `acquired` holds, before the /// launch that the lease reserves, [`Lease::launch`], which the caller then runs under the - /// same lock: the sandbox records no VM and no launch, so no VM of an earlier clone holds a + /// same lock, with [`launch`](Self::launch): the sandbox records no VM and no launch, so no VM of an earlier clone holds a /// file that the plan replaces. It takes `acquired` exclusively while it plans, since it may /// create a file in the sandbox's directory, as the transitions of a sandbox's store take its /// lock, and borrows nothing of it once it returns. It reads the generation's key, @@ -226,11 +286,13 @@ impl ClonePlan { /// - with [`ErrorKind::Unsupported`] for a request that no clone takes: a plan of the sandbox /// profile, with workload settings and its shares; a plan whose image is not the format and /// identity that the binding names, as for a binding whose identity is not a UUID in its - /// canonical text; an image or a scratch image whose path is relative, which OpenVMM would - /// resolve against the directory in which it runs, holds a comma or a semicolon, which - /// OpenVMM's block device options would split, or whose name ends in `.vhd`, `.vhdx`, - /// `.vmgs`, or `.iso`, which OpenVMM would not attach as a raw image; and an upper on a - /// scratch image without the image's binding, or one in RAM with a binding; + /// canonical text; an image that is a GPT disk, of which this build restores no clone, since + /// a clone's guest assembles its root only from an EROFS layer; an image or a scratch image + /// whose path is relative, which OpenVMM would resolve against the directory in which it + /// runs, holds a comma or a semicolon, which OpenVMM's block device options would split, or + /// whose name ends in `.vhd`, `.vhdx`, `.vmgs`, or `.iso`, which OpenVMM would not attach as + /// a raw image; and an upper on a scratch image without the image's binding, or one in RAM + /// with a binding; /// - with [`ErrorKind::Other`] for a lease of another store, and a lock that is not the one /// that acquired the lease, before the launch that it reserves began; /// - with [`ErrorKind::NotFound`] if the store no longer indexes the leased generation, with @@ -239,9 +301,18 @@ impl ClonePlan { /// - with [`ErrorKind::Unsupported`] for a request that the template cannot serve: a template /// whose key names other contracts than this build speaks, the boot and control contracts /// of the runtime that captured it included; a plan with a network for a template without a - /// NIC, or without one for a template with a NIC; and a memory target that is not whole - /// blocks of [`MEMORY_BLOCK_SIZE`] and of the template's memory blocks, from the template's - /// RAM to its RAM capacity; + /// NIC, without one for a template with a NIC, or with another guest network than the + /// template's NIC, whose address OpenVMM restores from the snapshot; and a memory target + /// that is not whole blocks of [`MEMORY_BLOCK_SIZE`] and of the template's memory blocks, + /// from the template's RAM to its RAM capacity; + /// - with [`ErrorKind::Io`] for a template whose command line names the DNS server of its + /// guest's resolver in another form than OpenVMM writes: more than once, without a NIC, or + /// other than as one IP address in its canonical spelling; + /// - with [`ErrorKind::Unsupported`] for settings that no activation record holds, as + /// [`Configuration::check`] finds them, such as a hostname that is empty, longer than + /// [`MAX_HOSTNAME_BYTES`](crate::activation::MAX_HOSTNAME_BYTES), or holds a NUL, and a + /// workload identity whose UID or GID is 0, and with [`ErrorKind::Other`] if the + /// transaction ID of the clone's restore cannot be drawn; /// - with [`ErrorKind::Unavailable`] for a link or a directory under a name that the sandbox's /// store owns, and with [`ErrorKind::Io`] if the store's files cannot be listed or examined; /// - as [`DistroBinding::check`] and [`CheckedDistro::geometry`] fail, with @@ -254,13 +325,17 @@ impl ClonePlan { /// comma or a semicolon, or whose filesystem presents another geometry than the template's /// scratch, and with [`ErrorKind::Io`] if the sparse file cannot be created, sized, or /// checked; - /// - for an upper on a scratch image, as [`ScratchBinding::check`] and - /// [`CheckedScratch::geometry`] fail, with [`ErrorKind::Unsupported`] for an image of - /// another geometry than the template's scratch, its length included, with - /// [`ErrorKind::Unavailable`] for an image that is not, at its path, the regular file that - /// the check found, of the current user or an administrator, and that no one else may - /// change, and with [`ErrorKind::Unsupported`] for an image that is a file of the sandbox's - /// store. + /// - for an upper on a scratch image, with [`ErrorKind::Io`] if the image's location cannot be + /// compared with the template store's directory, or its names cannot be counted, as + /// [`ScratchBinding::check`] and [`CheckedScratch::geometry`] fail, with + /// [`ErrorKind::Unsupported`] for an image of another geometry than the template's scratch, + /// its length included, with [`ErrorKind::Unavailable`] for an image that is not, at its + /// path, the regular file that the check found, of the current user or an administrator, + /// and that no one else may change, and with [`ErrorKind::Unsupported`] for an image that + /// is a file of the sandbox's store, or that is in the template store's directory, as the + /// files of its snapshots are, or has another name. + /// + /// [`Configuration::check`]: crate::activation::Configuration::check pub fn new( store: &Store, acquired: &mut Acquired, @@ -310,17 +385,24 @@ impl ClonePlan { let key = store.read_key(&generation)?; let snapshot = store.snapshot_dir(&generation)?; check_template(request, &key, &generation)?; + let activation = record::build(request, lease, &key)?; let owned = StoreFiles::of(locked)?; let image = check_image(request, &key, &owned, access)?; let capability = generate_capability()?; let upper = match (request.adapter.upper().scratch_path(), &request.scratch) { (Some(path), Some(binding)) => CloneUpper::Scratch(check_scratch( - path, binding, &key, &owned, deadline, access, + path, + binding, + &key, + (&owned, store), + deadline, + access, )?), _ => CloneUpper::Tmpfs { path: create_scratch(locked, &key, access)?, }, }; + let devices = devices_of(request, &upper); Ok(Self { lease: lease.clone(), generation, @@ -329,7 +411,9 @@ impl ClonePlan { image, upper, memory: request.memory, + activation, capability, + devices, }) } @@ -381,6 +465,29 @@ impl ClonePlan { self.memory } + /// Returns the clone's activation record, which its guest applies in place of the + /// configuration that the template captured, and whose restore transaction ID, as + /// [`ReadyActivation::of`](crate::activation::ReadyActivation::of) returns it, the guest's + /// Ready reports. + /// + /// The record holds the restore section, with the restore's transaction ID, 16 random bytes + /// that the plan drew, and the template that the [lease](Lease::template) names; the image, + /// an EROFS layer, with the identity of its binding and the UUID that the guest expects of its + /// filesystem, if the adapter's plan names one; the upper, a tmpfs, or the scratch image, with + /// the identity of its binding and the request's journal policy; the request's overlay mode + /// and workload view; no export; the network of the template's NIC, if it has one, with the + /// identity of the policy that OpenVMM enforces there, as the template's key records it, the + /// DNS server with which the template's guest configured its resolver at boot, if the + /// template's [command line](TemplateKey::command_line) names one, unpublished listeners, and + /// the URL of the policy's proxy as the guest sees it, if any; and the template's processors, + /// all online, and the memory target, as the resources that the clone must have. + /// + /// The identity of a binding is the SHA-256 digest of a domain of its kind and of its serde + /// form, a few hundred bytes of metadata: the plan reads no content to compute it. + pub fn activation(&self) -> &ActivationRecord { + &self.activation + } + /// Returns the capability of the clone's launch, which the plan drew, and which the VMM /// receives, as [`LockedStore::begin_launch`] stores it. pub fn capability(&self) -> &[u8; CAPABILITY_LEN] { @@ -399,10 +506,23 @@ impl fmt::Debug for ClonePlan { .field("image", &self.image) .field("upper", &self.upper) .field("memory", &self.memory) + .field("activation", &self.activation) .finish_non_exhaustive() } } +/// Returns the devices of the clone of `request`, whose upper is `upper`: the adapter's plan of its +/// image and network, over the file of its `scratch` block device, which every restore attaches, +/// the sparse file of an upper in RAM included. +fn devices_of(request: &CloneRequest, upper: &CloneUpper) -> SandboxPlan { + let adapter = &request.adapter; + let scratch = match upper { + CloneUpper::Tmpfs { path } => SandboxUpper::scratch_at(paths::normalized(path)), + CloneUpper::Scratch(_) => adapter.upper().clone(), + }; + SandboxPlan::adapter(adapter.distro().clone(), scratch).with_network(adapter.network().cloned()) +} + /// Refuses a request that no clone takes, as [`ClonePlan::new`] describes, without reading or /// opening anything. fn check_request(request: &CloneRequest) -> Result<()> { @@ -423,6 +543,9 @@ fn check_request(request: &CloneRequest) -> Result<()> { binding.identity() ))); } + if matches!(binding.format(), DistroFormat::Gpt) { + return Err(record::gpt_refused()); + } check_disk_path(distro.path(), "image")?; match (adapter.upper().scratch_path(), &request.scratch) { (Some(path), Some(_)) => check_disk_path(path, "scratch image"), @@ -517,6 +640,18 @@ fn check_template( ) })); } + // OpenVMM restores the guest's address on the NIC from the snapshot, and a clone's launch + // drops the plan's, so the plan must name the template's. + if let Some(network) = request.adapter.network().filter(|_| attached) + && let Some(template) = key.guest_network() + && network.guest_network() != template + { + return Err(Error::unsupported(format!( + "the adapter's plan of a clone gives its guest the network {}, but the template's \ + generation {name} restores its NIC on {template}, from its snapshot", + network.guest_network() + ))); + } let memory = request.memory; let block = key.memory_block_size(); if memory < key.memory() @@ -574,14 +709,15 @@ fn check_image( /// Checks and claims the scratch image at `path` against `binding`, through `access`, waiting /// until `deadline` for a claim that another launch holds to be released, and then checks its -/// geometry, against the template's scratch that `key` records, the image at its path, and that -/// it is none of `owned`, the files of the sandbox's store, as [`ClonePlan`] describes, and +/// geometry, against the template's scratch that `key` records, the image at its path, that it +/// is none of `owned`, the files of the sandbox's store, that it is outside the directory of +/// `store`, the template store, and that it has no other name, as [`ClonePlan`] describes, and /// returns the image that the binding's check opened and claimed. fn check_scratch( path: &Path, binding: &ScratchBinding, key: &TemplateKey, - owned: &StoreFiles, + (owned, store): (&StoreFiles, &Store), deadline: Instant, access: Access<'_>, ) -> Result { @@ -599,9 +735,111 @@ fn check_scratch( .map_err(|error| refused(format!("cannot be identified: {error}")))?; check_at_path(path, identity, refused)?; owned.check(identity, "scratch image", path)?; + check_outside_store(&checked, store, path)?; + check_single_name(&checked, path)?; Ok(checked) } +/// Refuses the clone's scratch image at `path`, which `checked` holds open and claims, if it is in +/// the directory of `store`, the template store, where it is now, every link on the way to it +/// resolved: the store's generations hold their snapshots there, which every clone of a +/// generation restores in place, and its captures write theirs there, which their publications +/// make generations' by renaming directories within it, so a file in the store could be, or +/// become as the plan examines it, a file of such a snapshot. Nothing that the host does moves a +/// file into the store's directory, so one outside it stays outside it. The location resolves the +/// aliases that the file system makes, links and junctions on the way and directories held open, +/// but not a mount or a network share that exposes the store's directories elsewhere, which +/// [`ClonePlan`] leaves to the system's administration. +fn check_outside_store(checked: &CheckedScratch, store: &Store, path: &Path) -> Result<()> { + let failed = |error: io::Error| { + Error::io(format!( + "cannot compare the location of the scratch image {} with the template store in {}", + path.display(), + store.dir().display() + )) + .with_source(error) + }; + let location = location_of(checked.file()).map_err(failed)?; + if is_within(&location, store.dir()).map_err(failed)? { + return Err(Error::unsupported(format!( + "the scratch image {} is {}, in the directory of the template store {}, where its \ + captures write the snapshots that every clone of a generation restores in place: a \ + clone's scratch image must be outside it", + path.display(), + location.display(), + store.dir().display() + ))); + } + Ok(()) +} + +/// Returns whether the directory `dir` is one of those above `location`, by their identities, past +/// those that went from their paths meanwhile, as a publication renames a capture's transaction, +/// which `dir`, the store's own directory, outlives. +fn is_within(location: &Path, dir: &Path) -> io::Result { + let dir = platform::file_id(dir)?; + for ancestor in location.ancestors().skip(1) { + match platform::file_id(ancestor) { + Ok(identity) if identity == dir => return Ok(true), + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + io::ErrorKind::NotFound | io::ErrorKind::NotADirectory + ) => {} + Err(error) => return Err(error), + } + } + Ok(false) +} + +/// Returns where the open `file` is now, every link on the way to it resolved: on Linux, the +/// target of its link in `/proc/self/fd`, which follows renames of its directories, and on +/// Windows its final path, which names its volume by a drive letter or a mount point, or else by +/// the volume's GUID. +fn location_of(file: &File) -> io::Result { + #[cfg(target_os = "linux")] + { + use std::os::fd::AsRawFd; + + fs::read_link(Path::new("/proc/self/fd").join(file.as_raw_fd().to_string())) + } + #[cfg(windows)] + { + // A volume without a drive letter or a mount point has no path by a drive. + crate::host::seal::final_path(file) + .or_else(|error| crate::host::seal::final_volume_path(file).map_err(|_| error)) + } + #[cfg(not(any(target_os = "linux", windows)))] + { + let _ = file; + Err(io::Error::from(io::ErrorKind::Unsupported)) + } +} + +/// Refuses the clone's scratch image at `path`, which `checked` holds open, unless it has a single +/// name: another name, a hard link, could make it a file of a snapshot that the clone's guest then +/// writes, of one of the store's generations or captures, or of another store, which the image's +/// location does not show. +fn check_single_name(checked: &CheckedScratch, path: &Path) -> Result<()> { + let links = platform::link_count(checked.file()).map_err(|error| { + Error::io(format!( + "cannot count the names of the scratch image {}", + path.display() + )) + .with_source(error) + })?; + if links > 1 { + return Err(Error::unsupported(format!( + "the scratch image {} has {links} names, but a clone's scratch image must have a \ + single name: another, a hard link, could make it a file of a snapshot, which every \ + clone of its generation restores in place", + path.display() + ))); + } + Ok(()) +} + /// The regular files that the store of a sandbox owns, with their identities, which no image or /// scratch image of a clone may be. struct StoreFiles(Vec<(PathBuf, (u64, u128))>); diff --git a/nvx-rs/src/host/template/clone/launch.rs b/nvx-rs/src/host/template/clone/launch.rs new file mode 100644 index 000000000..bb6b9d399 --- /dev/null +++ b/nvx-rs/src/host/template/clone/launch.rs @@ -0,0 +1,566 @@ +//! The launch of a clone, [`ClonePlan::launch`]: OpenVMM's restore of the snapshot of the leased +//! generation, as the launch that the clone's lease reserves, with the clone's devices, its +//! activation record, and the endpoint at which the launch awaits the VMM's restore-ready event. + +use std::ffi::{OsStr, OsString}; +use std::fmt; +use std::path::{Path, PathBuf}; + +use super::{ClonePlan, CloneUpper, MIB, SPARSE_WHAT, check_geometry}; +use crate::activation::ActivationRecord; +use crate::error::{Error, Result}; +use crate::host::artifacts; +use crate::host::filesystem; +use crate::host::platform::{self, ReadyListener}; +use crate::host::sandbox::distro::CheckedDistro; +use crate::host::sandbox::{SandboxPlan, block_geometry}; +use crate::host::store::files::write_private; +use crate::host::store::regular::open_regular; +use crate::host::store::{LockedStore, Recovery, RuntimeIdentity}; +use crate::host::supervisor::{self, Inherited, Starting, VmmCommand}; +use crate::host::template::key::TemplateKey; +use crate::host::template::manifest::BlockGeometry; +use crate::host::template::store::{Acquired, Lease, SOCKET_PATH_LIMIT}; +use crate::protocol::CAPABILITY_LEN; + +/// A point of a clone's launch at which a test interrupts it, before the step that follows the +/// point. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Point { + /// The sandbox holds the clone's activation record, the launch listens for the restore-ready + /// event, and the launch has not begun. + Prepared, + /// The launch's journal names it, and its VMM is not spawned yet. + Begun, +} + +/// The seams of a clone's launch, through which tests observe and interrupt it. +#[derive(Clone, Copy)] +pub(super) struct Hooks<'a> { + /// Returns the command that starts the VMM, given the one that the launch made. + pub(super) command: &'a dyn for<'h> Fn(VmmCommand<'h>) -> VmmCommand<'h>, + /// Fails the launch at a point, as a test asks. + pub(super) interrupt: &'a dyn Fn(Point) -> Result<()>, +} + +impl Hooks<'static> { + /// The seams of every launch outside tests. + pub(super) const LAUNCH: Self = Self { + command: &unchanged, + interrupt: &uninterrupted, + }; +} + +/// Returns `command`, as launches outside tests start their VMMs. +fn unchanged(command: VmmCommand<'_>) -> VmmCommand<'_> { + command +} + +/// Lets a launch pass every point, as launches outside tests do. +fn uninterrupted(_: Point) -> Result<()> { + Ok(()) +} + +impl ClonePlan { + /// Launches the clone that the plan describes, under the lock that acquired its lease, which + /// `acquired` holds, as the launch that the lease reserves: the OpenVMM executable `openvmm`, + /// which a relative path names from the current directory, restores the leased generation's + /// snapshot with the clone's devices, under the runtime identity `runtime`, which must be the + /// template's, [`TemplateKey::runtime`], and receives the plan's capability. Returns the + /// launch, whose VMM runs, as a [`Restoring`]. + /// + /// The launch first checks, without writing anything, that `acquired` holds the plan's lease, + /// under the lock that acquired it, before the launch that it reserves began, and that the + /// sandbox's clone binding records the lease: the acquisition made the binding durable, and no + /// VMM of the clone starts without it. For an upper in RAM, it checks that the file of the + /// scratch device that the plan created is still there: a later plan under the same + /// acquisition replaces it, and removes it if that plan fails. It then: + /// + /// 1. Writes the clone's activation record, in its canonical encoding, as the file + /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME) in the sandbox's + /// directory, in place of one that an earlier launch left, before the VMM starts: the + /// record, with the transaction ID of its restore section, is the one that the clone + /// activates with, which the caller, or a caller that takes the sandbox over after a + /// crash, reads there again and resends unchanged. + /// 2. Listens at the endpoint of the VMM's restore-ready event, which OpenVMM connects to once + /// it has prepared the restore, before the restored VM runs, and which must exist before + /// the VMM starts: on Linux, the socket + /// [`RESTORE_READY_NAME`](crate::host::store::RESTORE_READY_NAME) in the sandbox's + /// directory, in place of one that an earlier launch left; on Windows, a named pipe whose + /// name the launch draws at random. + /// 3. Runs [`supervisor::launch`] as the launch that the lease reserves, which begins it with + /// the store's launch journal, starts the VMM, and records its identity. + /// + /// The VMM restores the generation's snapshot in place, in the generation's payload, which it + /// only reads, and which the launch neither copies nor changes, so any number of clones + /// restore one generation at once. OpenVMM requires the control socket of a restore's VMM on + /// Linux in the snapshot's parent, so the VMM listens beside the snapshot, at the socket that + /// the lease reserved, `.sock` in the payload, after the launch's identity, rather + /// than at the sandbox's own control socket, as + /// [`Store::acquire`](crate::host::template::Store::acquire) describes; on Windows, at a named + /// pipe, as every launch does. [`Starting::endpoint`] returns it. + /// + /// The VMM's arguments are those of the clone's devices: the microVM machine, the clone's + /// image as the read-only `distro` block device and the file of its `scratch` block device, + /// which OpenVMM opens at their paths, as [Block devices by + /// path](crate::host::template::Store#block-devices-by-path) describes, and the policy of the + /// template's NIC, if it has one, without the guest's address, which OpenVMM takes from the + /// snapshot. Then come `--single-process`, the template's backend, CPU profile, and + /// processors, `--restore-snapshot` with the directory of the generation's snapshot, + /// [`ClonePlan::snapshot`], `--restore-memory` with the memory target, `--restore-ready-path` + /// with the endpoint of the restore-ready event, a boot console that discards its output, as + /// cold boots have, the control console at the launch's endpoint, which authenticates with + /// the capability that the VMM reads from its standard input, and the outcome report, + /// [`SandboxStore::outcome_path`](crate::host::store::SandboxStore::outcome_path). They name + /// no kernel, initramfs, kernel command line, or RAM, which the snapshot fixes. The VMM runs + /// in the sandbox's directory. + /// + /// The claim of a scratch image passes to the VMM, which holds it until it exits. On Linux, + /// the VMM inherits the claim's descriptor, which no argument names. On Windows, the claim's + /// handle, open for writing, would make OpenVMM's open of the image fail, since that open + /// shares no writing: the launch closes it before the VMM starts, and the VMM inherits the + /// image's guard instead, which keeps every other claim out, and the image's path naming the + /// image, while the VMM runs. The [`Restoring`] holds the plan's image, open for reading. + /// + /// A crash of the caller at any step leaves the clone's files in the sandbox, which a release + /// of the lease, [`Store::release`](crate::host::template::Store::release), removes, and a + /// launch that the journal names, which a reattach ends, as + /// [Crashes](crate::host::supervisor#crashes) describes, removing the control socket that the + /// VMM, if it started, left beside the snapshot. + /// + /// Like every launch of the sandbox's store, the launch names the sandbox's files by their + /// paths, under its lock: the activation record that it writes, the restore-ready socket at + /// which it listens on Linux, and the journal, capability, and log with which it begins; and + /// the VMM, which runs in the sandbox's directory, opens the files that its arguments name + /// there. So the directories above the sandbox's must be ones that only the current user, or + /// an administrator, may change, as the [store](crate::host::store) requires of them, and as + /// [Block devices by path](crate::host::template::Store#block-devices-by-path) requires of + /// those above the clone's block devices. + /// + /// Fails, and starts no VMM, checking in this order: + /// + /// - with [`ErrorKind::Other`](crate::ErrorKind::Other) if `acquired` holds another lease + /// than the plan's, under another lock than the one that acquired it, or after the launch + /// that it reserves began, and if the sandbox's clone binding does not record the plan's + /// lease, and as [`Lease::of`] fails, before anything is written; + /// - with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a runtime identity + /// other than the template's; + /// - for an upper in RAM, with [`ErrorKind::Unavailable`](crate::ErrorKind::Unavailable) if + /// the file of its scratch device, which the plan created, cannot be opened, as when a later + /// plan of the clone replaced it and failed, and with + /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) if it no longer presents the + /// template's scratch geometry; + /// - on Linux, with [`ErrorKind::Other`](crate::ErrorKind::Other) if the control socket that + /// the lease reserves is not beside the plan's snapshot, as for a plan and a lease that + /// name the template store by different directories; with + /// [`ErrorKind::Io`](crate::ErrorKind::Io) if the generation's payload cannot be resolved + /// or opened, or is not a directory of the current user with mode 0700, as OpenVMM requires + /// of the directory of a control socket; and with + /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) if the path of the control + /// socket, with or without the payload's links resolved, or of the restore-ready socket, is + /// not shorter than 108 bytes, as a Unix socket's address requires, or is not valid UTF-8, + /// as OpenVMM and the host take it, or if the path of the control socket holds a comma or an + /// equals sign, which OpenVMM's option `listen=` cannot take; + /// - with [`ErrorKind::Unavailable`](crate::ErrorKind::Unavailable) for an `openvmm` whose + /// path cannot be resolved; + /// - with [`ErrorKind::Io`](crate::ErrorKind::Io) if the activation record cannot be written, + /// or the endpoint of the restore-ready event cannot be opened; + /// - and as [`supervisor::launch`] fails, which ends the launch. + /// + /// A launch that fails leaves the clone's files that it wrote for the release. + pub fn launch<'s>( + self, + acquired: &'s mut Acquired, + openvmm: &Path, + runtime: &RuntimeIdentity, + ) -> Result> { + self.launch_with(acquired, openvmm, runtime, Hooks::LAUNCH) + } + + /// Launches the clone as [`launch`](Self::launch) does, through `hooks`. + pub(super) fn launch_with<'s>( + self, + acquired: &'s mut Acquired, + openvmm: &Path, + runtime: &RuntimeIdentity, + hooks: Hooks<'_>, + ) -> Result> { + let Acquired { lease, locked } = acquired; + self.check_launch(lease, locked, runtime)?; + let openvmm = artifacts::absolute(openvmm)?; + let Self { + lease: _, + generation: _, + key, + snapshot, + image, + upper, + memory, + activation, + capability, + devices, + } = self; + let devices = device_arguments(&devices)?; + write_private(&locked.clone_activation_path(), &activation.encode())?; + let (ready, listener) = + platform::listen_ready(&locked.restore_ready_path()).map_err(|error| { + Error::io(format!( + "cannot listen for the restore-ready event of the clone in {}", + locked.dir().display() + )) + .with_source(error) + })?; + (hooks.interrupt)(Point::Prepared)?; + // On Windows, this closes the claim's handle, and keeps the image's guard. + let claim = match upper { + CloneUpper::Scratch(checked) => Some(checked.into_file()), + CloneUpper::Tmpfs { .. } => None, + }; + // No spawn passes a descriptor of a standard stream, on which the claim is if the caller + // had closed that stream: it moves above them, with its open file description. + #[cfg(target_os = "linux")] + let claim = match claim { + Some(mut file) => { + crate::host::sys::move_above_standard_streams(&mut file).map_err(|error| { + Error::io("cannot pass the claim of the clone's scratch image to its VMM") + .with_source(error) + })?; + Some(file) + } + None => None, + }; + let restore = Restore { + devices, + key: &key, + snapshot, + memory, + ready: &ready, + report: locked.outcome_path(), + }; + let dir = locked.dir().to_owned(); + let inherit: Vec> = claim.iter().map(Inherited::new).collect(); + let starting = + supervisor::launch(locked, lease.launch(), runtime, &capability, |endpoint| { + (hooks.interrupt)(Point::Begun)?; + let mut command = VmmCommand::new(&openvmm, restore.arguments(endpoint), &dir); + command.inherit = inherit; + Ok((hooks.command)(command)) + }); + // The VMM holds the claim, if it started. + drop(claim); + Ok(Restoring { + starting: starting?, + ready, + listener, + image, + activation, + capability, + }) + } + + /// Refuses a launch of the plan through `lease` and `locked` that is not the one that the + /// plan's lease reserves, or whose binding the sandbox does not record, one under `runtime` + /// that the template does not take, and one of a plan in RAM whose scratch device's file is + /// gone, as [`launch`](Self::launch) describes, without writing anything. + fn check_launch( + &self, + lease: &Lease, + locked: &LockedStore, + runtime: &RuntimeIdentity, + ) -> Result<()> { + let planned = &self.lease; + if lease != planned + || locked.dir() != planned.sandbox() + || locked.reserved() != Some(planned.launch()) + { + return Err(Error::other(format!( + "a clone is launched under the lock that acquired the lease of its plan, {}, \ + before the launch that the lease reserves begins, but the lock of the sandbox in \ + {} holds the lease {}, and reserves no launch {} of it", + planned.id(), + locked.dir().display(), + lease.id(), + planned.launch() + ))); + } + if Lease::of(locked)?.as_ref() != Some(planned) { + return Err(Error::other(format!( + "the sandbox in {} records no clone binding of the lease {}, which its \ + acquisition makes durable before any VMM of the clone starts", + locked.dir().display(), + planned.id() + ))); + } + if runtime != self.key.runtime() { + return Err(Error::unsupported(format!( + "the template's generation {} restores under the runtime that captured it, {:?}, \ + not under {runtime:?}", + self.generation.generation(), + self.key.runtime() + ))); + } + if let CloneUpper::Tmpfs { path } = &self.upper { + check_sparse(path, self.key.scratch())?; + } + if cfg!(target_os = "linux") + && let Some(socket) = locked.reserved_socket() + { + self.check_sockets(socket, &locked.restore_ready_path())?; + } + Ok(()) + } + + /// Refuses, on Linux, a control socket of the clone's VMM at `socket`, beside the plan's + /// snapshot, and a socket of the restore-ready event at `ready`, that OpenVMM or the host + /// could not use, as [`launch`](Self::launch) describes, without writing anything. + /// + /// OpenVMM requires the control socket of a restore in the snapshot's parent, a directory of + /// the current user with mode 0700, which the payload of a generation is, and listens at the + /// socket's path with the parent's links resolved, while the host connects at the path that it + /// recorded: each must be shorter than 108 bytes, as a Unix socket's address requires, and + /// valid UTF-8, as OpenVMM and the host take them, as must the path of the restore-ready + /// socket. OpenVMM takes the control socket's path in its option `listen=`, among + /// options that commas separate, and whose values follow an equals sign, so the path must hold + /// neither. + fn check_sockets(&self, socket: &Path, ready: &Path) -> Result<()> { + let (Some(payload), Some(name)) = (socket.parent(), socket.file_name()) else { + return Err(Error::other(format!( + "the control socket {} of the clone's VMM names no directory", + socket.display() + ))); + }; + let resolve = |path: &Path| { + filesystem::canonicalize(path).map_err(|error| { + Error::io(format!( + "cannot resolve the payload of the generation {}", + path.display() + )) + .with_source(error) + }) + }; + let canonical = resolve(payload)?; + let beside = match self.snapshot.parent() { + Some(parent) if parent == payload => true, + Some(parent) => resolve(parent)? == canonical, + None => false, + }; + if !beside { + return Err(Error::other(format!( + "the control socket {} of the clone's VMM, which its lease reserves, is not beside \ + the snapshot {} of its plan, where OpenVMM requires it", + socket.display(), + self.snapshot.display() + ))); + } + let problem = platform::check_store_dir(payload).map_err(|error| { + Error::io(format!( + "cannot open the payload of the generation {}", + payload.display() + )) + .with_source(error) + })?; + if let Some(problem) = problem { + return Err(Error::io(format!( + "the payload of the generation {} {problem}", + payload.display() + ))); + } + let resolved = canonical.join(name); + for path in [socket, &resolved, ready] { + if path.as_os_str().len() >= SOCKET_PATH_LIMIT { + return Err(Error::unsupported(format!( + "the path of the socket {} of a clone's launch must be shorter than \ + {SOCKET_PATH_LIMIT} bytes, as a Unix socket's address requires", + path.display() + ))); + } + } + for path in [socket, &resolved, ready] { + if path.to_str().is_none() { + return Err(Error::unsupported(format!( + "the path of the socket {} of a clone's launch must be valid UTF-8, as \ + OpenVMM takes it", + path.display() + ))); + } + } + if socket.to_string_lossy().contains([',', '=']) { + return Err(Error::unsupported(format!( + "the path of the control socket {} of a clone's VMM must hold no ',' or '=', \ + which end the path in OpenVMM's option `listen=`", + socket.display() + ))); + } + Ok(()) + } +} + +/// Refuses the file of the `scratch` block device of a clone whose upper is in RAM, at `path`, +/// which the clone's plan created, unless it is still a regular file that presents the template's +/// scratch geometry, `scratch`: OpenVMM opens it by its path, and a later plan under the same +/// acquisition replaces it, or removes it if that plan fails. +fn check_sparse(path: &Path, scratch: BlockGeometry) -> Result<()> { + let shown = path.display(); + let file = open_regular(path).map_err(|error| { + Error::unavailable(format!( + "the {SPARSE_WHAT} {shown}, which the clone's plan created, cannot be opened, as when \ + a later plan of the clone replaced it and failed" + )) + .with_source(error) + })?; + check_geometry( + block_geometry(&file, path, SPARSE_WHAT)?, + scratch, + &format!("the {SPARSE_WHAT} {shown}"), + "scratch", + ) +} + +/// Returns OpenVMM's arguments of the devices of a clone's restore, `devices`: those that +/// [`SandboxPlan::openvmm_arguments`] renders, without `--net` and the guest's address, which +/// OpenVMM takes from the snapshot, and refuses beside it. The options of the network's policy +/// stay: OpenVMM compares the policy that they name with the one of the snapshot's NIC, and +/// refuses to restore under another, so the plan must keep its template NIC's policy, as +/// [`CloneRequest::adapter`](super::CloneRequest::adapter) says, and a plan of another one fails +/// closed, ending the launch. +fn device_arguments(devices: &SandboxPlan) -> Result> { + let mut arguments = devices.openvmm_arguments()?; + if let Some(at) = arguments.iter().position(|argument| argument == "--net") { + arguments.drain(at..(at + 2).min(arguments.len())); + } + Ok(arguments) +} + +/// What OpenVMM's arguments of a clone's restore name besides its control endpoint. +struct Restore<'a> { + /// The arguments of the clone's devices, as [`device_arguments`] renders them. + devices: Vec, + /// The generation's key. + key: &'a TemplateKey, + /// The directory of the snapshot that the VMM restores. + snapshot: PathBuf, + /// The memory target, in bytes. + memory: u64, + /// The endpoint of the restore-ready event. + ready: &'a str, + /// The path of the VMM's outcome report. + report: PathBuf, +} + +impl Restore<'_> { + /// Returns OpenVMM's arguments of the restore, whose control console listens on `endpoint`, + /// as [`ClonePlan::launch`] describes them. + fn arguments(&self, endpoint: &str) -> Vec { + let key = self.key; + let profile = key.cpu_profile(); + let processors = key.processors().to_string(); + let memory = format!("{}M", self.memory / MIB); + let listen = format!("listen={endpoint}"); + let mut arguments: Vec = self.devices.iter().map(OsString::from).collect(); + for argument in [ + OsStr::new("--single-process"), + OsStr::new("--hypervisor"), + OsStr::new(key.hypervisor().as_str()), + OsStr::new("--cpu-profile"), + OsStr::new(&profile), + OsStr::new("--processors"), + OsStr::new(&processors), + OsStr::new("--restore-snapshot"), + self.snapshot.as_os_str(), + OsStr::new("--restore-memory"), + OsStr::new(&memory), + OsStr::new("--restore-ready-path"), + OsStr::new(self.ready), + OsStr::new("--virtio-console"), + OsStr::new("none"), + OsStr::new("--microvm-control-console"), + OsStr::new(&listen), + OsStr::new("--microvm-control-auth-stdin"), + OsStr::new("--microvm-report"), + self.report.as_os_str(), + ] { + arguments.push(argument.to_owned()); + } + arguments + } +} + +/// The launch of a clone whose VMM restores the leased generation's snapshot, as +/// [`ClonePlan::launch`] returns it: the [`Starting`] launch of the VMM, whose journal records +/// the VMM's identity, the endpoint at which the launch awaits the VMM's restore-ready event, the +/// clone's activation record and capability, and its image, open for reading. +/// +/// It borrows the sandbox's locked store exclusively, as [`Starting`] does. Dropping it abandons +/// the launch, as [`abandon`](Self::abandon) does, ignoring failures, and stops listening for the +/// restore-ready event; a crash of the caller leaves the launch's journal instead, for +/// [`LockedStore::recover`]. +pub struct Restoring<'s> { + /// The launch, whose journal records the VMM. + starting: Starting<'s>, + /// The endpoint of the restore-ready event, as OpenVMM's `--restore-ready-path` takes it. + ready: String, + /// The listener of the restore-ready event. + listener: ReadyListener, + /// The clone's image, open for reading, which the plan checked. + image: CheckedDistro, + /// The clone's activation record. + activation: ActivationRecord, + /// The capability of the launch. + capability: [u8; CAPABILITY_LEN], +} + +impl<'s> Restoring<'s> { + /// Returns the launch, whose VMM runs, and whose journal records its identity. + pub fn starting(&self) -> &Starting<'s> { + &self.starting + } + + /// Returns the endpoint at which the launch awaits the restore-ready event of the VMM, as + /// OpenVMM's `--restore-ready-path` takes it: the path of a Unix socket on Linux, and the name + /// of a named pipe, `//./pipe/NAME`, on Windows. + pub fn ready_endpoint(&self) -> &str { + &self.ready + } + + /// Returns the clone's activation record, as the sandbox holds it. + pub fn activation(&self) -> &ActivationRecord { + &self.activation + } + + /// Returns the capability of the launch, with which the control console of the VMM + /// authenticates the caller. + pub fn capability(&self) -> &[u8; CAPABILITY_LEN] { + &self.capability + } + + /// Returns the clone's image, open for reading, which the plan checked. + pub fn image(&self) -> &CheckedDistro { + &self.image + } + + /// Ends the launch, as [`Starting::abandon`] does, which terminates its VMM and ends the + /// launch once nothing of it runs, and stops listening for the restore-ready event. + pub fn abandon(self) -> Result { + self.starting.abandon() + } + + /// Returns the launch, which a test commits as a clone's restore does once its VMM runs. + #[cfg(test)] + pub(super) fn into_starting(self) -> Starting<'s> { + self.starting + } +} + +impl fmt::Debug for Restoring<'_> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + // The capability authenticates control sessions, so it is never shown. + formatter + .debug_struct("Restoring") + .field("starting", &self.starting) + .field("ready", &self.ready) + .field("listener", &self.listener) + .field("image", &self.image) + .field("activation", &self.activation) + .finish_non_exhaustive() + } +} diff --git a/nvx-rs/src/host/template/clone/record.rs b/nvx-rs/src/host/template/clone/record.rs new file mode 100644 index 000000000..f644a6fc2 --- /dev/null +++ b/nvx-rs/src/host/template/clone/record.rs @@ -0,0 +1,159 @@ +//! The activation record of a clone, which its [plan](super::ClonePlan) builds before the clone's +//! launch from the request and the checked generation: the record of the settings that the +//! clone's guest applies in place of the configuration that its template captured, as +//! [`nvx::activation`](crate::activation) describes it. +//! +//! A record names the bindings of the clone's image and scratch image by their identities: the +//! SHA-256 digest of a domain of each kind of binding and of the binding's serde form, which its +//! callers record. The digest covers a few hundred bytes of the binding's metadata, never the +//! content of a file. + +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use super::CloneRequest; +use crate::activation::{ + ActivationRecord, Configuration, ExpectedResources, Identity, Image, Network, Publication, + RestoreSection, TransactionId, Upper, +}; +use crate::error::{Error, Result}; +use crate::host::sandbox::distro::{DistroBinding, DistroFormat}; +use crate::host::sandbox::scratch::ScratchBinding; +use crate::host::template::key::TemplateKey; +use crate::host::template::store::Lease; + +/// What the identity of an image's binding digests before the binding's serde form: a domain +/// that no other digest of this crate shares, and the NUL that ends it. +const DISTRO_DOMAIN: &[u8] = b"nvx distro binding\0"; + +/// What the identity of a scratch image's binding digests before the binding's serde form, as +/// [`DISTRO_DOMAIN`] does for an image's. +const SCRATCH_DOMAIN: &[u8] = b"nvx scratch binding\0"; + +/// Returns the activation record of the clone that `request` asks for, of the template whose key +/// is `key`, which `lease` leases, with a fresh transaction ID. +/// +/// The record holds: +/// +/// - the restore section: a transaction ID of 16 random bytes, and the template that `lease` +/// names, its generation and the identity of its key; +/// - the image: an EROFS layer, the identity of its binding, and the UUID of its filesystem, +/// which the adapter's plan names, as the binding does; +/// - the upper: a tmpfs, or the scratch, with the identity of its binding and the request's +/// journal policy; +/// - the request's overlay mode and workload view; +/// - no export, and so no mappings; +/// - the network of the template's NIC, if it has one: the identity of the policy that OpenVMM +/// enforces there, the digest that OpenVMM computed and the template's key records; the DNS +/// server with which the template's guest configured its resolver at boot, which the template's +/// command line names, [`TemplateKey::dns_server`], if it names one; unpublished listeners, +/// since a template's policy allows neither ingress nor host-loopback access both ways; and the +/// URL of the policy's proxy as the guest sees it, if it has one; +/// - the expected resources: the template's processors, all of which a template's VM keeps +/// online, and the request's memory target. +/// +/// Fails with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a request whose image +/// is not an EROFS layer, and for a configuration that no record holds, as +/// [`Configuration::check`] finds it, such as a hostname that is empty or longer than +/// [`MAX_HOSTNAME_BYTES`](crate::activation::MAX_HOSTNAME_BYTES), or a workload identity whose +/// UID or GID is 0; with [`ErrorKind::Io`](crate::ErrorKind::Io) for a key whose command line +/// names its DNS server in a form that OpenVMM does not write, as [`TemplateKey::dns_server`] +/// finds it; and with [`ErrorKind::Other`](crate::ErrorKind::Other) if the transaction ID cannot +/// be drawn. +pub(super) fn build( + request: &CloneRequest, + lease: &Lease, + key: &TemplateKey, +) -> Result { + let image = match request.binding.format() { + DistroFormat::Erofs => Image::Erofs { + binding: distro_identity(&request.binding)?, + uuid: request.adapter.distro().image().identity, + }, + DistroFormat::Gpt => return Err(gpt_refused()), + }; + let upper = match &request.scratch { + None => Upper::Tmpfs, + Some(binding) => Upper::Scratch { + binding: scratch_identity(binding)?, + journal: request.journal, + }, + }; + let dns = key.dns_server()?; + let network = key.network_policy_digest().map(|policy| Network { + policy: Identity::from_bytes(policy), + dns, + publication: Publication::Unpublished, + proxy: request + .adapter + .network() + .and_then(|network| network.proxy_url()), + }); + let configuration = Configuration { + image, + upper, + overlay: request.overlay, + export: None, + mappings: 0, + network, + workload: request.workload.clone(), + resources: ExpectedResources { + cpus: key.processors(), + memory: request.memory, + }, + }; + let restore = RestoreSection { + transaction: transaction()?, + template: lease.template(), + }; + ActivationRecord::new(Some(restore), configuration).map_err(|error| { + Error::unsupported(format!( + "no clone activates with the settings of the request: {error}" + )) + }) +} + +/// Returns the error of a request whose image is a GPT disk, which no clone restores yet. +pub(super) fn gpt_refused() -> Error { + Error::unsupported( + "the image of a clone must be an EROFS layer: this build restores no clone of a GPT \ + image", + ) +} + +/// Returns the identity of `binding`, the binding of a clone's image, as the [module](self) +/// describes it. +pub(super) fn distro_identity(binding: &DistroBinding) -> Result { + identity(DISTRO_DOMAIN, binding, "image") +} + +/// Returns the identity of `binding`, the binding of a clone's scratch image, as the +/// [module](self) describes it. +pub(super) fn scratch_identity(binding: &ScratchBinding) -> Result { + identity(SCRATCH_DOMAIN, binding, "scratch image") +} + +/// Returns the SHA-256 digest of `domain` and of the serde form of `binding`, the binding of a +/// clone's file, which messages call `what`. +fn identity(domain: &[u8], binding: &impl Serialize, what: &str) -> Result { + let form = serde_json::to_vec(binding).map_err(|error| { + Error::other(format!( + "cannot write the binding of the clone's {what} in its serde form" + )) + .with_source(error) + })?; + let digest = Sha256::new() + .chain_update(domain) + .chain_update(form) + .finalize(); + Ok(Identity::from_bytes(digest.into())) +} + +/// Returns a fresh transaction ID: 16 random bytes. +fn transaction() -> Result { + let mut bytes = [0; TransactionId::LEN]; + getrandom::fill(&mut bytes).map_err(|error| { + Error::other("cannot draw the transaction ID of a clone's restore").with_source(error) + })?; + Ok(TransactionId::from_bytes(bytes)) +} diff --git a/nvx-rs/src/host/template/clone/tests.rs b/nvx-rs/src/host/template/clone/tests.rs index 7808cb36a..063f7cc2b 100644 --- a/nvx-rs/src/host/template/clone/tests.rs +++ b/nvx-rs/src/host/template/clone/tests.rs @@ -1,20 +1,30 @@ //! Tests of the plans of clones: the checks of their images, uppers, and memory targets against //! the generation that they lease, which read no content, the claims of their scratch images, and -//! the sparse scratch device of an upper in RAM. +//! the sparse scratch device of an upper in RAM; and, in [`launch`], of their launches. + +mod launch; use std::io::Read; +use std::net::{IpAddr, Ipv4Addr}; use std::time::Duration; +use sha2::{Digest, Sha256}; + use super::*; use crate::ErrorCode; +use crate::activation::{ + Configuration, ExpectedResources, Identity, Image, MachineIdMode, Network, Publication, Upper, + WorkloadIdentity, +}; use crate::host::filesystem; -use crate::host::network::{Access as NetworkAccess, NetworkPolicy}; +use crate::host::network::{Access as NetworkAccess, NetworkPolicy, NetworkProxy, ProxyScheme}; use crate::host::paths; use crate::host::sandbox::distro::{DISTRO_POLICY_REVISION, PolicyVerdict}; use crate::host::sandbox::scratch::CLAIM_RELEASE_WAIT; use crate::host::sandbox::{SandboxDistro, SandboxNetwork, SandboxSettings, SandboxUpper}; use crate::host::seal::{FileSeal, open_metadata_only}; use crate::host::store::{CLONE_SCRATCH_NAME, CallerFiles, LOG_NAME, SandboxStore}; +use crate::host::template::store::{CAPTURES_NAME, SNAPSHOT_NAME, TRANSACTION_NAME}; use crate::host::template::test_support::{MIB, ManifestSpec, class, network_class}; use crate::host::template::{CaptureClass, HostTime}; use crate::host::test_support::released; @@ -80,6 +90,25 @@ fn stage(path: &Path, length: u64) { options.open(path).unwrap().set_len(length).unwrap(); } +/// Makes `link` a junction to the directory `target`, quoting both paths, since `cmd` splits its +/// arguments at commas and semicolons. +#[cfg(windows)] +fn junction(link: &Path, target: &Path) { + use std::os::windows::process::CommandExt; + use std::process::{Command, Stdio}; + + let status = Command::new("cmd") + .raw_arg(format!( + "/C mklink /J \"{}\" \"{}\"", + link.display(), + target.display() + )) + .stdout(Stdio::null()) + .status() + .unwrap(); + assert!(status.success(), "cannot link {}", link.display()); +} + /// Returns the physical block size that OpenVMM presents for the files of the directory `dir`: /// the I/O block size of its filesystem on Linux, and 4096 bytes on Windows. fn physical_block_size(dir: &Path) -> u32 { @@ -152,7 +181,9 @@ fn layer(path: &Path) -> SandboxLayer { } /// Returns the request of a clone of the image at `image`, which `binding` binds, over an upper in -/// RAM, with the template's RAM as its memory target. +/// RAM, with the template's RAM as its memory target, and the settings of [`workload`], for +/// untrusted lowers, and with a journal policy, [`Journal::Forbidden`], that its record leaves +/// out. fn tmpfs_request(image: &Path, binding: &DistroBinding) -> CloneRequest { CloneRequest { adapter: SandboxPlan::adapter( @@ -163,6 +194,21 @@ fn tmpfs_request(image: &Path, binding: &DistroBinding) -> CloneRequest { policy_revision: DISTRO_POLICY_REVISION, scratch: None, memory: MEMORY, + workload: workload(), + overlay: OverlayMode::UntrustedLowers, + journal: Journal::Forbidden, + } +} + +/// Returns the view that these tests' clones give their workloads of their VMs. +fn workload() -> WorkloadView { + WorkloadView { + hostname: b"clone".to_vec(), + machine_id: MachineIdMode::Generated, + identity: Some(WorkloadIdentity { + uid: 1000, + gid: 1000, + }), } } @@ -243,9 +289,18 @@ impl Fixture { /// Publishes a template as [`new`](Self::new) does, of the class and manifest that `change` /// changes, given the physical block size of the files of the directory. fn with(change: impl FnOnce(&mut CaptureClass, &mut ManifestSpec, u32)) -> Self { + Self::in_store(|directory| directory.join("templates"), change) + } + + /// Publishes a template as [`with`](Self::with) does, in a store at the path that `store` + /// returns, given the temporary directory, in which it creates the store's parent directory. + fn in_store( + store: impl FnOnce(&Path) -> PathBuf, + change: impl FnOnce(&mut CaptureClass, &mut ManifestSpec, u32), + ) -> Self { let directory = tempfile::tempdir().unwrap(); let physical = physical_block_size(directory.path()); - let store = Store::create(directory.path().join("templates")).unwrap(); + let store = Store::create(store(directory.path())).unwrap(); let (mut class, mut spec) = (class(), ManifestSpec::template()); set_geometries( &mut class, @@ -289,17 +344,24 @@ impl Fixture { (path, binding) } - /// Provisions the sandbox `name`, and leases the generation to it, a second after its - /// capture. - fn acquire(&self, name: &str) -> Acquired { + /// Provisions the sandbox `name`, and returns its directory, with its links resolved, as a + /// lease names it. + fn provision(&self, name: &str) -> PathBuf { let dir = self.path("sandboxes", name); let locked = SandboxStore::lock_for_provision(&dir, &files(), None).unwrap(); fs::write(dir.join(CONFIG), b"{}").unwrap(); drop(locked); + filesystem::canonicalize(&dir).unwrap() + } + + /// Provisions the sandbox `name`, and leases the generation to it, a second after its + /// capture. + fn acquire(&self, name: &str) -> Acquired { + let sandbox = self.provision(name); self.store .acquire( self.generation.generation(), - filesystem::canonicalize(&dir).unwrap(), + sandbox, &files(), &after(&self.generation, 1), None, @@ -693,6 +755,188 @@ fn links_and_directories_under_the_names_of_the_sandboxs_store_are_refused() { assert!(fs::symlink_metadata(plan.scratch_path()).unwrap().is_file()); } +#[test] +fn a_scratch_image_that_is_a_file_of_a_generation_s_snapshot_is_refused() { + let fixture = Fixture::new(); + let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + // The RAM of the leased generation's snapshot, and that of another generation's, which the + // store's index need not list, of the template's scratch length: the clone's guest would write + // a snapshot that every clone of its generation restores in place. + let snapshot = fixture.store.snapshot_dir(&fixture.generation).unwrap(); + let leased = snapshot.join("memory.bin"); + stage(&leased, SCRATCH_LEN); + let next = fixture.generation.generation().get().get() + 1; + let other = crate::activation::StoreGeneration::new(std::num::NonZeroU64::new(next).unwrap()); + let other = fixture + .store + .payload_dir(other) + .join(snapshot.file_name().unwrap()); + fs::create_dir_all(&other).unwrap(); + let other = other.join("memory.bin"); + stage(&other, SCRATCH_LEN); + // Through a hard link among the scratch images, another name of the file, and by its own + // path, in the store's directory. + let cases = [ + (&leased, fixture.path("scratches", "leased.raw")), + (&other, fixture.path("scratches", "other.raw")), + (&other, other.clone()), + ]; + for (memory, scratch) in cases { + let linked = scratch != *memory; + if linked { + fs::hard_link(memory, &scratch).unwrap(); + } + let binding = ScratchBinding::bind(&scratch).unwrap(); + let request = scratch_request(&image, &image_binding, &scratch, &binding); + let expected = if linked { + format!( + "the scratch image {} has 2 names, but a clone's scratch image must have a single \ + name: another, a hard link, could make it a file of a snapshot, which every \ + clone of its generation restores in place", + scratch.display() + ) + } else { + format!( + "the scratch image {} is {}, in the directory of the template store {}, where its \ + captures write the snapshots that every clone of a generation restores in place: \ + a clone's scratch image must be outside it", + scratch.display(), + location_of(&File::open(&scratch).unwrap()) + .unwrap() + .display(), + fixture.store.dir().display() + ) + }; + assert_eq!( + failure(fixture.plan(&mut acquired, &request, NO_CONTENT)), + (ErrorKind::Unsupported, expected) + ); + assert_untouched(&acquired); + } +} + +#[test] +fn a_scratch_image_in_the_template_store_is_refused() { + let fixture = Fixture::new(); + let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + // A file in the store's directory, and one of a snapshot that a capture of the store writes, + // which its publication makes a generation's by renaming the capture's transaction, of the + // template's scratch length: by their own paths, and through links on the way to them, which + // the image's location resolves. + let dir = fixture.store.dir().to_owned(); + let loose = dir.join("scratch.raw"); + let transaction = dir.join(CAPTURES_NAME).join("slot").join(TRANSACTION_NAME); + let memory = transaction.join(SNAPSHOT_NAME).join("memory.bin"); + fs::create_dir_all(memory.parent().unwrap()).unwrap(); + for file in [&loose, &memory] { + stage(file, SCRATCH_LEN); + } + let through = |link: &Path| link.join(SNAPSHOT_NAME).join("memory.bin"); + let mut cases = vec![(&loose, loose.clone()), (&memory, memory.clone())]; + let link = fixture.path("links", "transaction"); + fs::create_dir_all(link.parent().unwrap()).unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink(&transaction, &link).unwrap(); + #[cfg(windows)] + junction(&link, &transaction); + cases.push((&memory, through(&link))); + // The transaction held open, as a descriptor keeps a directory whatever renames it. + #[cfg(target_os = "linux")] + let held = File::open(&transaction).unwrap(); + #[cfg(target_os = "linux")] + cases.push(( + &memory, + through( + &Path::new("/proc/self/fd").join(std::os::fd::AsRawFd::as_raw_fd(&held).to_string()), + ), + )); + for (file, scratch) in cases { + let binding = ScratchBinding::bind(&scratch).unwrap(); + let request = scratch_request(&image, &image_binding, &scratch, &binding); + let location = location_of(&File::open(file).unwrap()).unwrap(); + assert_eq!( + failure(fixture.plan(&mut acquired, &request, NO_CONTENT)), + ( + ErrorKind::Unsupported, + format!( + "the scratch image {} is {}, in the directory of the template store {}, \ + where its captures write the snapshots that every clone of a generation \ + restores in place: a clone's scratch image must be outside it", + scratch.display(), + location.display(), + dir.display() + ) + ) + ); + assert_untouched(&acquired); + } +} + +/// On Windows, a file of a volume without a drive letter or a mount point is located by the +/// volume's GUID, above which the store's directory is found all the same. +#[cfg(windows)] +#[test] +fn locations_that_name_their_volume_by_its_guid_are_compared_with_the_store() { + let fixture = Fixture::new(); + let dir = fixture.store.dir().to_owned(); + let inside = dir + .join(CAPTURES_NAME) + .join("slot") + .join(TRANSACTION_NAME) + .join("memory.bin"); + fs::create_dir_all(inside.parent().unwrap()).unwrap(); + stage(&inside, SCRATCH_LEN); + let (outside, _) = fixture.scratch("scratch.ext4", SCRATCH_LEN); + for (file, within) in [(&inside, true), (&outside, false)] { + let location = crate::host::seal::final_volume_path(&File::open(file).unwrap()).unwrap(); + assert!( + location.to_string_lossy().starts_with(r"\\?\Volume{"), + "{}", + location.display() + ); + assert_eq!( + is_within(&location, &dir).unwrap(), + within, + "{}", + location.display() + ); + } +} + +#[test] +fn a_scratch_image_with_another_name_is_refused() { + let fixture = Fixture::new(); + let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + // Another name of the image, outside the template store, as a file of the snapshot of another + // store would be, or one of a snapshot that a capture of this store writes, which neither the + // store's generations nor the image's location show. + let (scratch, _) = fixture.scratch("scratch.ext4", SCRATCH_LEN); + let other = fixture.path("stores", "memory.bin"); + fs::create_dir_all(other.parent().unwrap()).unwrap(); + fs::hard_link(&scratch, &other).unwrap(); + let binding = ScratchBinding::bind(&scratch).unwrap(); + let request = scratch_request(&image, &image_binding, &scratch, &binding); + assert_eq!( + failure(fixture.plan(&mut acquired, &request, NO_CONTENT)), + ( + ErrorKind::Unsupported, + format!( + "the scratch image {} has 2 names, but a clone's scratch image must have a single \ + name: another, a hard link, could make it a file of a snapshot, which every clone \ + of its generation restores in place", + scratch.display() + ) + ) + ); + assert_untouched(&acquired); + // Once its other name went, the image is the clone's alone. + fs::remove_file(&other).unwrap(); + drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); +} + #[test] fn names_that_the_file_system_takes_for_those_of_the_stores_files_are_the_stores() { let fixture = Fixture::new(); @@ -1347,6 +1591,26 @@ fn networks_that_the_template_does_not_have_are_refused() { ) ); assert_untouched(&acquired); + // OpenVMM restores the guest's address on the NIC from the snapshot, so a plan of another + // address, or of another prefix, is refused. + for other in ["10.0.0.3/24", "10.0.0.2/16"] { + let moved = with_network( + request.clone(), + Some(SandboxNetwork::new(other, NetworkPolicy::new(NetworkAccess::Allow)).unwrap()), + ); + assert_eq!( + failure(fixture.plan(&mut acquired, &moved, NO_CONTENT)), + ( + ErrorKind::Unsupported, + format!( + "the adapter's plan of a clone gives its guest the network {other}, but the \ + template's generation {generation} restores its NIC on 10.0.0.2/24, from its \ + snapshot" + ) + ) + ); + assert_untouched(&acquired); + } let attached = with_network(request, Some(network(NetworkAccess::Allow))); drop(fixture.plan(&mut acquired, &attached, NO_CONTENT).unwrap()); } @@ -1467,3 +1731,257 @@ fn files_that_their_paths_do_not_name_as_checked_are_refused() { ); assert_untouched(&acquired); } + +/// Returns the SHA-256 digest of `domain` and of the serde form of `binding`, as the identity of a +/// binding is documented. +fn digest_of(domain: &[u8], binding: &impl serde::Serialize) -> Identity { + let digest = Sha256::new() + .chain_update(domain) + .chain_update(serde_json::to_vec(binding).unwrap()) + .finalize(); + Identity::from_bytes(digest.into()) +} + +#[test] +fn the_activation_record_of_a_clone_in_ram_names_its_restore_image_and_settings() { + let fixture = Fixture::new(); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let request = tmpfs_request(&image, &binding); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let record = plan.activation(); + let restore = record.restore().unwrap(); + assert_eq!(restore.template, acquired.lease.template()); + assert_eq!( + record.configuration(), + &Configuration { + image: Image::Erofs { + binding: digest_of(b"nvx distro binding\0", &binding), + uuid: Some(Uuid::parse(UUID).unwrap()), + }, + // A clone in RAM has no journal policy, whatever the request's. + upper: Upper::Tmpfs, + overlay: OverlayMode::UntrustedLowers, + export: None, + mappings: 0, + network: None, + workload: workload(), + resources: ExpectedResources { + cpus: plan.key().processors(), + memory: MEMORY, + }, + } + ); + // The plan shows the record, which holds nothing secret. + let shown_plan = format!("{plan:?}"); + assert!( + shown_plan.contains(&format!("activation: {record:?}")), + "{shown_plan}" + ); + // Each plan draws its own transaction. + let transaction = restore.transaction; + drop(plan); + let again = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let restored = again.activation().restore().unwrap(); + assert_ne!(restored.transaction, transaction); + assert_eq!(restored.template, acquired.lease.template()); + // A larger memory target is the resources that the clone must have. + drop(again); + let larger = CloneRequest { + memory: 2 * MEMORY, + ..request + }; + let plan = fixture.plan(&mut acquired, &larger, NO_CONTENT).unwrap(); + assert_eq!( + plan.activation().configuration().resources.memory, + 2 * MEMORY + ); +} + +#[test] +fn the_activation_record_of_a_clone_on_a_scratch_image_names_its_binding_and_journal() { + let fixture = Fixture::new(); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); + let mut acquired = fixture.acquire("sandbox"); + for journal in [Journal::Forbidden, Journal::Allowed] { + let request = CloneRequest { + overlay: OverlayMode::TrustedLowers, + journal, + ..scratch_request(&image, &binding, &scratch, &scratch_binding) + }; + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let configuration = plan.activation().configuration(); + assert_eq!( + configuration.upper, + Upper::Scratch { + binding: digest_of(b"nvx scratch binding\0", &scratch_binding), + journal, + } + ); + assert_eq!(configuration.overlay, OverlayMode::TrustedLowers); + assert_eq!( + *configuration.image.binding(), + digest_of(b"nvx distro binding\0", &binding) + ); + } + // The identities of two bindings differ, as their files do. + let (other, other_binding) = fixture.scratch("other.ext4", SCRATCH_LEN); + let request = scratch_request(&image, &binding, &other, &other_binding); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + assert_ne!( + plan.activation().configuration().upper, + Upper::Scratch { + binding: digest_of(b"nvx scratch binding\0", &scratch_binding), + journal: Journal::Forbidden, + } + ); +} + +#[test] +fn the_activation_record_of_a_clone_of_a_template_with_a_nic_names_its_policy_and_dns_server() { + let fixture = Fixture::with(|class, spec, physical| { + *class = network_class(); + *spec = ManifestSpec::with_network(); + let (distro, scratch) = ( + geometry(IMAGE_LEN, physical), + geometry(SCRATCH_LEN, physical), + ); + set_geometries(class, spec, distro, scratch); + }); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + // The policy's identity is the digest that OpenVMM computed, as the template's key records + // it, [7; 32] in this manifest, and the network names no proxy without one. + let attached = with_network( + tmpfs_request(&image, &binding), + Some(network(NetworkAccess::Allow)), + ); + let plan = fixture.plan(&mut acquired, &attached, NO_CONTENT).unwrap(); + let expected = Network { + policy: Identity::from_bytes([7; 32]), + dns: None, + publication: Publication::Unpublished, + proxy: None, + }; + assert_eq!( + plan.activation().configuration().network.as_ref(), + Some(&expected) + ); + drop(plan); + // A proxy's URL is the one that the guest sees, at its gateway. + let proxy = NetworkProxy::new(3128).with_scheme(ProxyScheme::Http); + let proxied = with_network( + tmpfs_request(&image, &binding), + Some(SandboxNetwork::new("10.0.0.2/24", NetworkPolicy::proxy_only(proxy)).unwrap()), + ); + let plan = fixture.plan(&mut acquired, &proxied, NO_CONTENT).unwrap(); + assert_eq!( + plan.activation().configuration().network, + Some(Network { + proxy: Some("http://10.0.0.1:3128".to_owned()), + ..expected.clone() + }) + ); + drop((plan, acquired, fixture)); + // The DNS server with which the template's guest configured its resolver at boot, as its + // command line asks, is the clone's. + let fixture = Fixture::with(|class, spec, physical| { + *class = network_class(); + *spec = ManifestSpec::with_network(); + spec.contract() + .command_line + .push_str(" virtnet_dns=10.0.0.1"); + let (distro, scratch) = ( + geometry(IMAGE_LEN, physical), + geometry(SCRATCH_LEN, physical), + ); + set_geometries(class, spec, distro, scratch); + }); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let attached = with_network( + tmpfs_request(&image, &binding), + Some(network(NetworkAccess::Allow)), + ); + let plan = fixture.plan(&mut acquired, &attached, NO_CONTENT).unwrap(); + assert_eq!( + plan.activation().configuration().network, + Some(Network { + dns: Some(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + ..expected + }) + ); +} + +#[test] +fn settings_that_no_activation_record_holds_are_refused_before_anything_is_created() { + let fixture = Fixture::new(); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let request = tmpfs_request(&image, &binding); + let with_workload = |workload: WorkloadView| CloneRequest { + workload, + ..request.clone() + }; + for (workload, problem) in [ + ( + WorkloadView { + hostname: Vec::new(), + ..workload() + }, + "the hostname is not 1 to 64 bytes without a NUL", + ), + ( + WorkloadView { + hostname: vec![b'h'; 65], + ..workload() + }, + "the hostname is not 1 to 64 bytes without a NUL", + ), + ( + WorkloadView { + hostname: b"clone\0".to_vec(), + ..workload() + }, + "the hostname is not 1 to 64 bytes without a NUL", + ), + ( + WorkloadView { + identity: Some(WorkloadIdentity { uid: 0, gid: 1000 }), + ..workload() + }, + "the workload identity's UID or GID is 0 or 4294967295", + ), + ] { + assert_eq!( + failure(fixture.plan(&mut acquired, &with_workload(workload), NO_CONTENT)), + ( + ErrorKind::Unsupported, + format!("no clone activates with the settings of the request: {problem}") + ) + ); + assert_untouched(&acquired); + } + // A GPT image, whose binding names it, is refused before anything is opened. + let path = fixture.path("images", "distro.gpt"); + stage(&path, IMAGE_LEN); + let gpt = SandboxLayer::gpt(LayerRole::Distro, path.to_str().unwrap(), UUID); + let gpt_binding = DistroBinding::import(&gpt, PolicyVerdict::ACCEPTED).unwrap(); + let gpt_request = CloneRequest { + adapter: SandboxPlan::adapter( + SandboxDistro::new(path.to_str().unwrap(), bound_image(&gpt_binding).unwrap()), + SandboxUpper::RAM, + ), + binding: gpt_binding, + ..request.clone() + }; + let (kind, message) = failure(fixture.plan(&mut acquired, &gpt_request, NO_CONTENT)); + assert_eq!(kind, ErrorKind::Unsupported, "{message}"); + assert!( + message.starts_with("the image of a clone must be an EROFS layer"), + "{message}" + ); + assert_untouched(&acquired); + drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); +} diff --git a/nvx-rs/src/host/template/clone/tests/launch.rs b/nvx-rs/src/host/template/clone/tests/launch.rs new file mode 100644 index 000000000..dc2495364 --- /dev/null +++ b/nvx-rs/src/host/template/clone/tests/launch.rs @@ -0,0 +1,1420 @@ +//! Tests of the launches of clones: OpenVMM's restore of the leased snapshot in place with the +//! clone's devices, activation record, and restore-ready endpoint, for which a +//! [`restore_vmm_role`] stands in, listening beside the snapshot on Linux as OpenVMM does; the +//! checks that keep a VMM from starting before the clone's binding is durable, or with a control +//! socket that OpenVMM or the host could not use; the removal of that socket; and the crashes of +//! a launch's caller, which a [`launch_role`] runs. + +use std::cell::RefCell; +use std::env; +use std::ffi::OsString; +use std::io::{Read, Write}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use super::*; +use crate::activation::StoreGeneration; +use crate::host::artifacts; +use crate::host::process::{Process, ProcessRef}; +use crate::host::store::{ + CLONE_ACTIVATION_NAME, CLONE_BINDING_NAME, CLONE_SCRATCH_NAME, JOURNAL_NAME, LaunchState, + LockedStore, OUTCOME_NAME, RESTORE_READY_NAME, Recovery, RuntimeIdentity, +}; +use crate::host::supervisor::{self, Reattached, StopOutcome, VmmCommand}; +use crate::host::template::clone::launch::{Hooks, Point}; +use crate::host::template::store::SNAPSHOT_ARTIFACTS; +use crate::host::template::test_support::artifact; + +/// The environment variable that names the template store of a [`launch_role`]. +const ROLE_STORE: &str = "NVX_CLONE_LAUNCH_TEST_STORE"; +/// The environment variable that names the store generation that a [`launch_role`] leases. +const ROLE_GENERATION: &str = "NVX_CLONE_LAUNCH_TEST_GENERATION"; +/// The environment variable that names the directory of the sandbox of a [`launch_role`]. +const ROLE_SANDBOX: &str = "NVX_CLONE_LAUNCH_TEST_SANDBOX"; +/// The environment variable that names the image of the clone of a [`launch_role`]. +const ROLE_IMAGE: &str = "NVX_CLONE_LAUNCH_TEST_IMAGE"; +/// The environment variable that names where a [`launch_role`] crashes: at the point `prepared` +/// or `begun` of its launch, or once its launch returned, `launched`. +const ROLE_CRASH: &str = "NVX_CLONE_LAUNCH_TEST_CRASH"; +/// The environment variable that names the working directory of the VMM of a [`launch_role`]. +const ROLE_VMM: &str = "NVX_CLONE_LAUNCH_TEST_VMM"; +/// The start of the line in which [`launch_role`] prints the reference of the VMM that it +/// spawned, as JSON. +const REFERENCE: &str = "nvx-clone-launch-test-vmm "; +/// The start of the line that [`launch_role`] prints as it crashes. +const CRASHED: &str = "nvx-clone-launch-test-crashed "; +/// File whose presence in its working directory marks a [`restore_vmm_role`]. Without it, the +/// role returns at once, as it does when `cargo test -- --ignored` runs it. +const VMM_MARKER: &str = "vmm-role"; +/// File whose appearance in its working directory ends a [`restore_vmm_role`]. +const EXIT_MARKER: &str = "exit"; +/// File in which [`fake_vmm`] writes the arguments that a launch gave its VMM, as JSON, for the +/// [`restore_vmm_role`] that runs instead. +const ARGUMENTS: &str = "arguments.json"; +/// File in which a [`restore_vmm_role`] writes, once it has tried to send its restore-ready +/// event, the capability that it read, in hexadecimal, and whether it sent the event. +const STARTED: &str = "started"; +/// The restore-ready event that OpenVMM sends. +const READY_EVENT: &[u8] = b"OPENVMM_RESTORE_READY_V1\n"; +/// The OpenVMM executable that the tests' launches name, which [`fake_vmm`] replaces. +const OPENVMM: &str = "openvmm"; +/// Longest wait for a VMM that is expected to start or exit, for a sandbox's lock, or for a +/// reattach to become determinate. +const TIMEOUT: Duration = Duration::from_secs(60); +/// Longest life of a [`restore_vmm_role`] that nothing ends, so that a failed test leaves no +/// process. +const VMM_LIFETIME: Duration = Duration::from_secs(180); + +/// Returns `bytes` in lowercase hexadecimal. +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Returns the arguments that run only the role `name` of this test executable, on libtest's main +/// thread, so that libtest prints the role's name before the role's own output. +fn role(name: &str) -> Vec { + [ + "--exact", + &format!("host::template::clone::tests::launch::{name}"), + "--ignored", + "--nocapture", + "--test-threads=1", + ] + .map(OsString::from) + .to_vec() +} + +/// Gives the snapshot of the generation of `fixture` its RAM and saved state beside its manifest, +/// as OpenVMM publishes them, and returns the snapshot's directory, and the content of each of +/// its files, by name. +fn complete_snapshot(fixture: &Fixture) -> (PathBuf, Vec<(&'static str, Vec)>) { + let snapshot = fixture.store.snapshot_dir(&fixture.generation).unwrap(); + let mut contents = Vec::new(); + for name in SNAPSHOT_ARTIFACTS { + let path = snapshot.join(name); + if !path.exists() { + fs::write(&path, format!("the {name} of the template")).unwrap(); + } + contents.push((name, fs::read(&path).unwrap())); + } + (snapshot, contents) +} + +/// Returns the seals of the files of the snapshot in `snapshot`, by name, which change with their +/// content, their length, their modification time, and on Linux their change time, which a +/// restore checks. +fn seals(snapshot: &Path) -> Vec<(&'static str, FileSeal)> { + SNAPSHOT_ARTIFACTS + .map(|name| { + let file = File::open(snapshot.join(name)).unwrap(); + (name, FileSeal::of(&file).unwrap()) + }) + .to_vec() +} + +/// Returns the runtime identity that captured the template of `fixture`, under which its clones +/// launch. +fn runtime_of(fixture: &Fixture) -> RuntimeIdentity { + fixture + .store + .read_key(&fixture.generation) + .unwrap() + .runtime() + .clone() +} + +/// Returns whether a socket, or another file, is at `path`. +fn exists(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok() +} + +/// Asserts that the payload of the generation of `fixture` holds no control socket of a clone's +/// VMM, and that the files of the generation's snapshot in `snapshot` have the contents +/// `contents` and the seals `sealed` still. +fn assert_untouched( + fixture: &Fixture, + snapshot: &Path, + contents: &[(&str, Vec)], + sealed: &[(&str, FileSeal)], +) { + let payload = fixture.store.payload_dir(fixture.generation.generation()); + for entry in fs::read_dir(payload).unwrap() { + let name = entry.unwrap().file_name(); + assert!(!name.to_string_lossy().ends_with(".sock"), "{name:?}"); + } + for (name, content) in contents { + assert_eq!(&fs::read(snapshot.join(name)).unwrap(), content, "{name}"); + } + assert_eq!(seals(snapshot), sealed); +} + +/// Asserts that no launch of the clone of `acquired` began or left a file: no journal, activation +/// record, restore-ready socket, or control socket of its VMM, and that the lease still reserves +/// its launch. +fn assert_unlaunched(fixture: &Fixture, acquired: &Acquired) { + let locked = &acquired.locked; + assert!(locked.journal().unwrap().is_none()); + for path in [ + locked.clone_activation_path(), + locked.restore_ready_path(), + fixture.store.clone_socket_path(&acquired.lease), + ] { + assert!(!exists(&path), "{}", path.display()); + } + assert_eq!(locked.reserved(), Some(acquired.lease.launch())); +} + +/// Asserts that the sandbox in `sandbox` holds none of the files that the launch of a clone +/// leaves for its release. +fn assert_released(sandbox: &Path) { + for name in [ + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, + ] { + assert!(!exists(&sandbox.join(name)), "{name}"); + } +} + +/// The seam of the command of a launch that must start no VMM. +fn never_spawned(_: VmmCommand<'_>) -> VmmCommand<'_> { + panic!("a launch that is refused made the command of a VMM") +} + +/// The command that a launch made for its VMM, before [`fake_vmm`] replaced it, and what the +/// sandbox held as the VMM was about to start. +#[derive(Debug)] +struct Made { + /// The VMM's executable. + program: PathBuf, + /// The VMM's arguments. + arguments: Vec, + /// The VMM's working directory. + working_dir: PathBuf, + /// How many descriptors or handles the VMM inherits. + inherited: usize, + /// Whether the sandbox held its clone binding. + bound: bool, + /// Whether the sandbox held the launch's journal. + begun: bool, +} + +/// Returns the seam of the command of a launch whose VMM is a [`restore_vmm_role`] in `vmm`: it +/// keeps the command that the launch made in `made`, writes its arguments in [`ARGUMENTS`] in +/// `vmm`, and runs the role there instead, with the descriptors or handles that the command +/// passes. +fn fake_vmm<'a>( + vmm: &'a Path, + made: &'a RefCell>, +) -> impl for<'h> Fn(VmmCommand<'h>) -> VmmCommand<'h> + 'a { + move |command| { + let arguments: Vec = command + .arguments + .iter() + .map(|argument| argument.to_str().unwrap().to_owned()) + .collect(); + fs::write(vmm.join(ARGUMENTS), serde_json::to_vec(&arguments).unwrap()).unwrap(); + made.replace(Some(Made { + program: command.program.clone(), + arguments, + working_dir: command.working_dir.clone(), + inherited: command.inherit.len(), + bound: command.working_dir.join(CLONE_BINDING_NAME).is_file(), + begun: command.working_dir.join(JOURNAL_NAME).is_file(), + })); + VmmCommand { + program: env::current_exe().unwrap(), + arguments: role("restore_vmm_role"), + working_dir: vmm.to_owned(), + breakaway_from_job: command.breakaway_from_job, + inherit: command.inherit, + } + } +} + +/// The working directory of a [`restore_vmm_role`], which it marks. Dropping it ends the VMM, and +/// waits for it to exit, so that a failed test leaves no process behind. +struct VmmDir { + /// The reference of the VMM, once the test knows it. + reference: Option, + /// The directory, which is removed after the VMM exits. + dir: tempfile::TempDir, +} + +impl VmmDir { + /// Creates the directory, and marks it. + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join(VMM_MARKER), b"").unwrap(); + Self { + reference: None, + dir, + } + } + + /// Returns the path of the directory. + fn path(&self) -> &Path { + self.dir.path() + } + + /// Waits until the VMM has tried to send its restore-ready event, and returns what it wrote + /// in [`STARTED`]. + fn started(&self) -> String { + let path = self.path().join(STARTED); + let deadline = Instant::now() + TIMEOUT; + loop { + match fs::read_to_string(&path) { + Ok(started) => return started, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + assert!(Instant::now() < deadline, "the VMM did not start"); + thread::sleep(Duration::from_millis(10)); + } + Err(error) => panic!("cannot read {}: {error}", path.display()), + } + } + } + + /// Asks the VMM to exit. + fn end(&self) { + fs::write(self.path().join(EXIT_MARKER), b"").unwrap(); + } + + /// Asks the VMM, which runs, to exit, and waits until it has. It opens the VMM's process + /// before it asks, since a process that has exited no longer opens. + fn end_and_wait(&self) { + let reference = self.reference.as_ref().expect("the test knows the VMM"); + let process = Process::open(reference) + .unwrap() + .expect("the VMM runs until the test ends it"); + self.end(); + assert!(process.wait(Instant::now() + TIMEOUT).unwrap()); + } +} + +impl Drop for VmmDir { + fn drop(&mut self) { + let _ = fs::write(self.path().join(EXIT_MARKER), b""); + if let Some(reference) = &self.reference + && let Ok(Some(process)) = Process::open(reference) + && !process.wait(Instant::now() + TIMEOUT).unwrap_or(false) + { + let _ = process.kill(); + let _ = process.wait(Instant::now() + TIMEOUT); + } + } +} + +/// The VMM of a clone, in a process of its own, in a directory that [`VMM_MARKER`] marks: reads +/// its capability as OpenVMM does, finds the endpoints of its control console and of the +/// restore-ready event, and the snapshot that it restores, among the arguments that its launch +/// made, which [`ARGUMENTS`] holds; holds the snapshot as OpenVMM does, which on Linux listens at +/// the control endpoint beside the snapshot, and on Windows opens the snapshot's directory and +/// files sharing nothing but reading; sends the restore-ready event as OpenVMM does; writes +/// [`STARTED`]; then runs until a file named [`EXIT_MARKER`] appears, or for [`VMM_LIFETIME`] at +/// most, and removes its control socket as it exits, as OpenVMM does at a normal exit. +/// +/// Where OpenVMM would refuse its arguments, this VMM writes why in [`STARTED`] after its +/// capability and whether it sent its event. Where OpenVMM would abort a restore whose event it +/// cannot send, as when the launch's caller crashed, this VMM runs on, so that the test ends it as +/// it ends any VMM. +#[test] +#[ignore = "a clone's VMM that the launch tests run in a process of its own"] +fn restore_vmm_role() { + let directory = env::current_dir().unwrap(); + if !directory.join(VMM_MARKER).exists() { + return; + } + let mut capability = [0; CAPABILITY_LEN]; + std::io::stdin().read_exact(&mut capability).unwrap(); + let arguments: Vec = + serde_json::from_slice(&fs::read(directory.join(ARGUMENTS)).unwrap()).unwrap(); + let (held, problem) = match hold_snapshot(&arguments) { + Ok(held) => (Some(held), String::new()), + Err(problem) => (None, format!(" {problem}")), + }; + let ready = argument(&arguments, "--restore-ready-path").unwrap(); + let sent = send_ready(Path::new(&ready)).is_ok(); + let staged = directory.join(format!(".{STARTED}")); + fs::write(&staged, format!("{} {sent}{problem}", hex(&capability))).unwrap(); + fs::rename(&staged, directory.join(STARTED)).unwrap(); + let started = Instant::now(); + while !directory.join(EXIT_MARKER).exists() && started.elapsed() < VMM_LIFETIME { + thread::sleep(Duration::from_millis(10)); + } + if let Some(held) = held { + held.release(); + } +} + +/// Returns the value of the option `name` among `arguments`, if they name it. +fn argument(arguments: &[String], name: &str) -> Option { + arguments + .iter() + .position(|argument| argument == name) + .and_then(|at| arguments.get(at + 1).cloned()) +} + +/// What a [`restore_vmm_role`] holds of its restore while it runs. +struct Held { + /// The listener at the control endpoint, on Linux, and the path at which it listens. + #[cfg(unix)] + listener: (std::os::unix::net::UnixListener, PathBuf), + /// The snapshot's directory and files, open, on Windows. + #[cfg(windows)] + _files: Vec, +} + +impl Held { + /// Lets go of the restore, as OpenVMM does at a normal exit, which removes its control socket. + fn release(self) { + #[cfg(unix)] + { + let (listener, path) = self.listener; + drop(listener); + fs::remove_file(path).unwrap(); + } + } +} + +/// Holds the snapshot that `arguments` restore as OpenVMM does, or returns why OpenVMM would +/// refuse them. On Linux, OpenVMM requires the parent of its control endpoint to be a directory +/// of its user with mode 0700, not a link, whose path with its links resolved is that of the +/// snapshot's parent, and listens at the endpoint's path with its parent's links resolved. +#[cfg(unix)] +fn hold_snapshot(arguments: &[String]) -> std::result::Result { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let endpoint = argument(arguments, "--microvm-control-console") + .and_then(|console| console.strip_prefix("listen=").map(PathBuf::from)) + .ok_or("no control endpoint")?; + let snapshot = PathBuf::from(argument(arguments, "--restore-snapshot").ok_or("no snapshot")?); + let (Some(parent), Some(name), Some(snapshot_parent)) = + (endpoint.parent(), endpoint.file_name(), snapshot.parent()) + else { + return Err("the endpoint or the snapshot names no directory".to_owned()); + }; + let metadata = fs::symlink_metadata(parent).map_err(|error| error.to_string())?; + if !metadata.is_dir() + || metadata.uid() != crate::host::sys::effective_ids().0 + || metadata.permissions().mode() & 0o7777 != 0o700 + { + return Err(format!( + "the endpoint's directory {} is not one that OpenVMM takes", + parent.display() + )); + } + let canonical = fs::canonicalize(parent).map_err(|error| error.to_string())?; + if fs::canonicalize(snapshot_parent).map_err(|error| error.to_string())? != canonical { + return Err(format!( + "the endpoint {} is not beside the snapshot {}", + endpoint.display(), + snapshot.display() + )); + } + let path = canonical.join(name); + let listener = std::os::unix::net::UnixListener::bind(&path) + .map_err(|error| format!("cannot listen at {}: {error}", path.display()))?; + Ok(Held { + listener: (listener, path), + }) +} + +/// Holds the snapshot that `arguments` restore as OpenVMM does, or returns why OpenVMM would +/// refuse them: on Windows, OpenVMM opens the snapshot's directory and files for reading, sharing +/// nothing but reading, so that a restore of another clone, which shares reading, reads them too, +/// while no one changes them. +#[cfg(windows)] +fn hold_snapshot(arguments: &[String]) -> std::result::Result { + use std::os::windows::fs::OpenOptionsExt; + + /// `FILE_SHARE_READ`. + const SHARE_READ: u32 = 0x1; + /// `FILE_FLAG_BACKUP_SEMANTICS`, which opens a directory. + const BACKUP_SEMANTICS: u32 = 0x0200_0000; + let snapshot = PathBuf::from(argument(arguments, "--restore-snapshot").ok_or("no snapshot")?); + let mut files = Vec::new(); + let paths = SNAPSHOT_ARTIFACTS.map(|name| (snapshot.join(name), 0)); + for (path, flags) in std::iter::once((snapshot.clone(), BACKUP_SEMANTICS)).chain(paths) { + let file = fs::OpenOptions::new() + .read(true) + .share_mode(SHARE_READ) + .custom_flags(flags) + .open(&path) + .map_err(|error| format!("cannot open {}: {error}", path.display()))?; + files.push(file); + } + Ok(Held { _files: files }) +} + +/// Sends the restore-ready event to the endpoint at `path`, as OpenVMM does: through a Unix +/// socket, or a named pipe on Windows. +fn send_ready(path: &Path) -> io::Result<()> { + #[cfg(unix)] + let mut sink = std::os::unix::net::UnixStream::connect(path)?; + #[cfg(windows)] + let mut sink = fs::OpenOptions::new().write(true).open(path)?; + sink.write_all(READY_EVENT) +} + +#[test] +fn a_clone_restores_its_leased_snapshot_with_its_devices_and_activation_record() { + let fixture = Fixture::new(); + let (snapshot, contents) = complete_snapshot(&fixture); + let sealed = seals(&snapshot); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let request = tmpfs_request(&image, &binding); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let (key, activation) = (plan.key().clone(), plan.activation().clone()); + let capability = *plan.capability(); + let sandbox = acquired.locked.dir().to_owned(); + let lease = acquired.lease.clone(); + let mut vmm = VmmDir::new(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(&mut acquired, Path::new(OPENVMM), key.runtime(), hooks) + .unwrap(); + let reference = restoring.starting().process().reference().clone(); + vmm.reference = Some(reference.clone()); + // The VMM read the plan's capability, took its arguments, and sent its event to the + // restore-ready endpoint, which listened before the VMM started. + assert_eq!(vmm.started(), format!("{} true", hex(&capability))); + // The VMM started once the sandbox held the clone binding and the launch's journal, in the + // sandbox's directory, with nothing of the caller's besides its standard streams. + let made = made.take().unwrap(); + assert!(made.bound && made.begun, "{made:?}"); + assert_eq!( + made.program, + artifacts::absolute(Path::new(OPENVMM)).unwrap() + ); + assert_eq!(made.working_dir, sandbox); + assert_eq!(made.inherited, 0); + // It restores the generation's snapshot in place, with the clone's devices: its image, and + // the sparse file of its `scratch` block device. On Linux, its control console listens beside + // the snapshot, at the socket that the lease reserved, named after the launch. + let endpoint = restoring.starting().endpoint().to_owned(); + let ready = restoring.ready_endpoint().to_owned(); + let socket = fixture.store.clone_socket_path(&lease); + if cfg!(target_os = "linux") { + assert_eq!( + socket, + snapshot + .parent() + .unwrap() + .join(format!("{}.sock", lease.launch())) + ); + assert_eq!(endpoint, socket.to_str().unwrap()); + assert!(exists(&socket)); + assert_eq!(ready, sandbox.join(RESTORE_READY_NAME).to_str().unwrap()); + } else { + assert!( + endpoint.starts_with("//./pipe/openvmm-microvm-"), + "{endpoint}" + ); + assert!(!exists(&socket)); + assert!(ready.starts_with("//./pipe/nvx-restore-ready-"), "{ready}"); + } + assert_eq!(restoring.starting().journal().endpoint(), endpoint); + let distro = format!("distro:file:{},ro", shown(&image)); + let scratch = format!("scratch:file:{}", shown(&sandbox.join(CLONE_SCRATCH_NAME))); + let (profile, processors) = (key.cpu_profile(), key.processors().to_string()); + let listen = format!("listen={endpoint}"); + let report = sandbox.join(OUTCOME_NAME); + let expected = [ + "--machine", + "microvm", + "--microvm-sandbox-block", + &distro, + "--microvm-sandbox-block", + &scratch, + "--single-process", + "--hypervisor", + key.hypervisor().as_str(), + "--cpu-profile", + &profile, + "--processors", + &processors, + "--restore-snapshot", + snapshot.to_str().unwrap(), + "--restore-memory", + "512M", + "--restore-ready-path", + &ready, + "--virtio-console", + "none", + "--microvm-control-console", + &listen, + "--microvm-control-auth-stdin", + "--microvm-report", + report.to_str().unwrap(), + ]; + assert_eq!(made.arguments, expected); + // The sandbox holds the clone's activation record, in its canonical encoding, which the + // launch keeps. + let recorded = fs::read(sandbox.join(CLONE_ACTIVATION_NAME)).unwrap(); + assert_eq!(recorded, activation.encode()); + assert_eq!(ActivationRecord::decode(&recorded).unwrap(), activation); + assert_eq!(restoring.activation(), &activation); + assert_eq!(restoring.capability(), &capability); + assert_eq!(restoring.image().path().to_str().unwrap(), shown(&image)); + // The launch never shows its capability. + let shown_launch = format!("{restoring:?}"); + assert!(shown_launch.starts_with("Restoring {"), "{shown_launch}"); + assert!(!shown_launch.contains("capability"), "{shown_launch}"); + assert!( + !shown_launch.contains(&format!("{capability:?}")), + "{shown_launch}" + ); + // Abandoning the launch ends its VMM, which leaves its control socket, and the end of the + // launch removes it. The release then removes the clone's files, and nothing of the + // generation, whose snapshot the restore left as it was. + assert_eq!( + restoring.abandon().unwrap(), + Recovery::Terminated(reference) + ); + assert!(!exists(&socket)); + assert_eq!( + fixture.store.release(&mut acquired.locked).unwrap(), + Some(acquired.lease.clone()) + ); + assert_released(&sandbox); + assert_untouched(&fixture, &snapshot, &contents, &sealed); +} + +#[test] +fn clones_of_one_generation_restore_its_snapshot_in_place_at_once() { + let fixture = Fixture::new(); + let (snapshot, contents) = complete_snapshot(&fixture); + let sealed = seals(&snapshot); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let request = tmpfs_request(&image, &binding); + let mut clones = [fixture.acquire("first"), fixture.acquire("second")]; + let leases = clones.each_ref().map(|acquired| acquired.lease.clone()); + let mut vmms = [VmmDir::new(), VmmDir::new()]; + // Both clones plan and launch at once, each under its own sandbox's lock. + let launched: Vec<(Restoring<'_>, Made)> = thread::scope(|scope| { + let launches: Vec<_> = clones + .iter_mut() + .zip(&vmms) + .map(|(acquired, vmm)| { + let (fixture, request, dir) = (&fixture, &request, vmm.path().to_owned()); + scope.spawn(move || { + let plan = fixture.plan(acquired, request, NO_CONTENT).unwrap(); + let runtime = plan.key().runtime().clone(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + (restoring, made.take().unwrap()) + }) + }) + .collect(); + launches + .into_iter() + .map(|launch| launch.join().unwrap()) + .collect() + }); + for ((restoring, _), vmm) in launched.iter().zip(&mut vmms) { + vmm.reference = Some(restoring.starting().process().reference().clone()); + } + // Both VMMs run, restoring the one snapshot in place, each listening at its own endpoint. + for ((restoring, made), vmm) in launched.iter().zip(&vmms) { + let capability = hex(restoring.capability()); + assert_eq!(vmm.started(), format!("{capability} true")); + let restored = argument(&made.arguments, "--restore-snapshot").unwrap(); + assert_eq!(restored, snapshot.to_str().unwrap()); + } + let endpoints: Vec<&str> = launched + .iter() + .map(|(restoring, _)| restoring.starting().endpoint()) + .collect(); + assert_ne!(endpoints[0], endpoints[1]); + if cfg!(target_os = "linux") { + for (endpoint, lease) in endpoints.iter().zip(&leases) { + let socket = fixture.store.clone_socket_path(lease); + assert_eq!(*endpoint, socket.to_str().unwrap()); + assert!(exists(&socket)); + } + } + // Each VMM exits as OpenVMM does, removing its socket, and each lease is released. + for ((restoring, _), vmm) in launched.into_iter().zip(&vmms) { + vmm.end_and_wait(); + assert_eq!(restoring.abandon().unwrap(), Recovery::Exited); + } + for (acquired, lease) in clones.iter_mut().zip(leases) { + assert_eq!( + fixture.store.release(&mut acquired.locked).unwrap(), + Some(lease) + ); + } + assert_untouched(&fixture, &snapshot, &contents, &sealed); +} + +/// Launches the clone of `acquired`, whose plan `fixture` makes, with a [`restore_vmm_role`] in +/// `vmm`, and commits the launch once the VMM runs, as a clone's restore does. +fn committed( + fixture: &Fixture, + acquired: &mut Acquired, + image: &(PathBuf, DistroBinding), + vmm: &mut VmmDir, +) -> supervisor::Vmm { + let plan = fixture + .plan(acquired, &tmpfs_request(&image.0, &image.1), NO_CONTENT) + .unwrap(); + let runtime = plan.key().runtime().clone(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + vmm.reference = Some(restoring.starting().process().reference().clone()); + let capability = hex(restoring.capability()); + assert_eq!(vmm.started(), format!("{capability} true")); + restoring.into_starting().commit().unwrap() +} + +#[test] +fn the_control_socket_of_a_clone_goes_once_its_vm_has_ended_and_for_good_with_its_lease() { + let fixture = Fixture::new(); + let (snapshot, contents) = complete_snapshot(&fixture); + let sealed = seals(&snapshot); + let image = fixture.image("distro.erofs", IMAGE_LEN); + // A clone's VM whose VMM a termination ends, which leaves its control socket: the teardown + // removes it with the runtime record. + let mut acquired = fixture.acquire("terminated"); + let mut vmm = VmmDir::new(); + let running = committed(&fixture, &mut acquired, &image, &mut vmm); + let socket = fixture.store.clone_socket_path(&acquired.lease); + assert_eq!( + running.endpoint(), + acquired.locked.runtime().unwrap().unwrap().endpoint() + ); + assert_eq!(exists(&socket), cfg!(target_os = "linux")); + assert_eq!( + running.terminate(&mut acquired.locked, TIMEOUT).unwrap(), + StopOutcome::Forced + ); + assert!(acquired.locked.runtime().unwrap().is_none()); + assert!(!exists(&socket)); + // A socket that comes back, as an unsynced removal can after a crash of the host, goes with + // the release, which makes its absence durable before the lease ends. + if cfg!(target_os = "linux") { + fs::write(&socket, b"").unwrap(); + } + assert_eq!( + fixture.store.release(&mut acquired.locked).unwrap(), + Some(acquired.lease.clone()) + ); + assert!(!exists(&socket)); + // A clone whose VMM exits on its own removes its socket itself, and a stop that finds it + // gone ends the VM. + let mut acquired = fixture.acquire("stopped"); + let mut vmm = VmmDir::new(); + let running = committed(&fixture, &mut acquired, &image, &mut vmm); + let socket = fixture.store.clone_socket_path(&acquired.lease); + let deadline = Instant::now() + TIMEOUT; + let stopped = running + .stop(&mut acquired.locked, deadline, None, |_, _| { + vmm.end(); + Ok::<_, ()>(()) + }) + .unwrap(); + assert_eq!(stopped.outcome, StopOutcome::Graceful); + assert!(acquired.locked.runtime().unwrap().is_none()); + assert!(!exists(&socket)); + assert_eq!( + fixture.store.release(&mut acquired.locked).unwrap(), + Some(acquired.lease.clone()) + ); + assert_untouched(&fixture, &snapshot, &contents, &sealed); +} + +#[test] +fn a_clone_of_a_template_with_a_nic_restores_its_policy_without_its_address() { + let fixture = Fixture::with(|class, spec, physical| { + *class = network_class(); + *spec = ManifestSpec::with_network(); + let (distro, scratch) = ( + geometry(IMAGE_LEN, physical), + geometry(SCRATCH_LEN, physical), + ); + set_geometries(class, spec, distro, scratch); + }); + complete_snapshot(&fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let attached = network(NetworkAccess::Allow); + let request = with_network(tmpfs_request(&image, &binding), Some(attached.clone())); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let runtime = plan.key().runtime().clone(); + let mut vmm = VmmDir::new(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + vmm.reference = Some(restoring.starting().process().reference().clone()); + // The devices end with the arguments of the network's policy, without the guest's address, + // which OpenVMM takes from the snapshot, and refuses beside it. + let mut policy = attached.openvmm_arguments().unwrap(); + assert_eq!(policy[..2], ["--net", "10.0.0.2/24"]); + policy.drain(..2); + let arguments = made.take().unwrap().arguments; + let devices = arguments + .iter() + .position(|argument| argument == "--single-process") + .unwrap(); + assert_eq!(arguments[6..devices], policy); + assert!(!arguments.iter().any(|argument| argument == "--net")); +} + +#[test] +fn a_clone_on_a_scratch_image_passes_its_claim_to_its_vmm() { + let fixture = Fixture::new(); + complete_snapshot(&fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); + let mut acquired = fixture.acquire("sandbox"); + let request = scratch_request(&image, &binding, &scratch, &scratch_binding); + let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); + let runtime = plan.key().runtime().clone(); + let mut vmm = VmmDir::new(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + let reference = restoring.starting().process().reference().clone(); + vmm.reference = Some(reference.clone()); + vmm.started(); + // The VMM opens the image at its path, as the `scratch` block device, and inherits the one + // descriptor or handle that holds the claim, which no argument names. + let made = made.take().unwrap(); + assert_eq!(made.inherited, 1); + let device = format!("scratch:file:{}", shown(&scratch)); + assert!(made.arguments.contains(&device), "{:?}", made.arguments); + // On Windows, no handle that may write the image is open any more, so OpenVMM's open, which + // shares no writing, succeeds. + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + + /// `FILE_SHARE_READ | FILE_SHARE_DELETE`. + const SHARE_READ_DELETE: u32 = 0x1 | 0x4; + /// `FILE_FLAG_OPEN_REPARSE_POINT`. + const OPEN_REPARSE_POINT: u32 = 0x0020_0000; + let opened = fs::OpenOptions::new() + .read(true) + .write(true) + .share_mode(SHARE_READ_DELETE) + .custom_flags(OPEN_REPARSE_POINT) + .open(&scratch); + assert!(opened.is_ok(), "{opened:?}"); + } + // The VMM holds the claim: another check finds the image claimed. + let error = scratch_binding.check(&scratch, Instant::now()).unwrap_err(); + assert_eq!(error.code(), ErrorCode::ScratchClaimed, "{error}"); + // The claim is the VMM's alone: once the VMM has exited, the image serves another check, + // while the launch lasts. + vmm.end_and_wait(); + assert!(released(|| scratch_binding + .check(&scratch, Instant::now()) + .is_ok())); + assert_eq!(restoring.abandon().unwrap(), Recovery::Exited); +} + +#[test] +fn a_clone_launches_only_as_the_launch_that_its_durable_binding_reserves() { + let fixture = Fixture::new(); + complete_snapshot(&fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let request = tmpfs_request(&image, &binding); + let runtime = runtime_of(&fixture); + let openvmm = Path::new(OPENVMM); + let hooks = Hooks { + command: &never_spawned, + ..Hooks::LAUNCH + }; + // A sandbox that no longer records the clone binding starts no VMM of the clone, and keeps + // nothing of its launch. + let mut unbound = fixture.acquire("unbound"); + let plan = fixture.plan(&mut unbound, &request, NO_CONTENT).unwrap(); + fs::remove_file(unbound.locked.binding_path()).unwrap(); + assert_eq!( + failure(plan.launch_with(&mut unbound, openvmm, &runtime, hooks)), + ( + ErrorKind::Other, + format!( + "the sandbox in {} records no clone binding of the lease {}, which its \ + acquisition makes durable before any VMM of the clone starts", + unbound.locked.dir().display(), + unbound.lease.id() + ) + ) + ); + assert_unlaunched(&fixture, &unbound); + // Nor does an acquisition of another lease. + let mut first = fixture.acquire("first"); + let mut second = fixture.acquire("second"); + let plan = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); + assert_eq!( + failure(plan.launch_with(&mut second, openvmm, &runtime, hooks)), + ( + ErrorKind::Other, + format!( + "a clone is launched under the lock that acquired the lease of its plan, {}, \ + before the launch that the lease reserves begins, but the lock of the sandbox in \ + {} holds the lease {}, and reserves no launch {} of it", + first.lease.id(), + second.locked.dir().display(), + second.lease.id(), + first.lease.launch() + ) + ) + ); + assert_unlaunched(&fixture, &first); + assert_unlaunched(&fixture, &second); + // Nor a runtime other than the one that captured the template. + let other = RuntimeIdentity::new(artifact('d'), artifact('e'), artifact('f')); + assert_ne!(other, runtime); + let plan = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); + let (kind, message) = failure(plan.launch_with(&mut first, openvmm, &other, hooks)); + assert_eq!(kind, ErrorKind::Unsupported, "{message}"); + assert!( + message.starts_with(&format!( + "the template's generation {} restores under the runtime that captured it", + fixture.generation.generation() + )), + "{message}" + ); + assert_unlaunched(&fixture, &first); + // Nor a plan in RAM whose scratch device's file a later plan of the clone, which failed, + // removed: OpenVMM would find no file at the path of its scratch device. + let stale = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); + let failing = Access { + create: &|path: &Path| -> io::Result { + fs::remove_file(path)?; + Err(io::Error::other("the disk failed")) + }, + ..NO_CONTENT + }; + assert!(fixture.plan(&mut first, &request, failing).is_err()); + let sparse = first.locked.clone_scratch_path(); + assert!(!exists(&sparse)); + let (kind, message) = failure(stale.launch_with(&mut first, openvmm, &runtime, hooks)); + assert_eq!(kind, ErrorKind::Unavailable, "{message}"); + assert!( + message.starts_with(&format!( + "the scratch device of the clone {}, which the clone's plan created, cannot be opened", + sparse.display() + )), + "{message}" + ); + assert_unlaunched(&fixture, &first); + // On Linux, a sandbox whose restore-ready socket's path would not fit a Unix socket's + // address is refused. + #[cfg(target_os = "linux")] + { + let mut deep = fixture.acquire(&"d".repeat(100)); + let plan = fixture.plan(&mut deep, &request, NO_CONTENT).unwrap(); + assert_eq!( + failure(plan.launch_with(&mut deep, openvmm, &runtime, hooks)), + ( + ErrorKind::Unsupported, + too_long(&deep.locked.restore_ready_path()) + ) + ); + assert_unlaunched(&fixture, &deep); + } + // The acquisition launches once its plan is its own. + let plan = fixture.plan(&mut second, &request, NO_CONTENT).unwrap(); + let mut vmm = VmmDir::new(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let restoring = plan + .launch_with(&mut second, openvmm, &runtime, hooks) + .unwrap(); + vmm.reference = Some(restoring.starting().process().reference().clone()); +} + +/// Returns the message of the refusal of a clone's launch whose socket at `path` would not fit a +/// Unix socket's address. +#[cfg(target_os = "linux")] +fn too_long(path: &Path) -> String { + format!( + "the path of the socket {} of a clone's launch must be shorter than 108 bytes, as a Unix \ + socket's address requires", + path.display() + ) +} + +#[cfg(target_os = "linux")] +#[test] +fn a_clone_whose_control_socket_openvmm_or_the_host_cannot_use_starts_nothing() { + use std::os::unix::fs::PermissionsExt; + + /// Returns a directory of 60 characters in `directory`, private to the current user. + fn deep(directory: &Path) -> PathBuf { + let deep = directory.join("d".repeat(60)); + assert_eq!(platform::create_owner_only_dir(&deep).unwrap(), None); + deep + } + + let hooks = Hooks { + command: &never_spawned, + ..Hooks::LAUNCH + }; + let refused = |fixture: &Fixture, sandbox: &str| { + complete_snapshot(fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire(sandbox); + let plan = fixture + .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) + .unwrap(); + let runtime = runtime_of(fixture); + let refusal = failure(plan.launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks)); + assert_unlaunched(fixture, &acquired); + (refusal, fixture.store.clone_socket_path(&acquired.lease)) + }; + // A store whose path leaves no room for the socket of a clone's VMM in a payload. + let fixture = Fixture::in_store(|directory| deep(directory).join("templates"), |_, _, _| {}); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert!(socket.as_os_str().len() >= 108); + assert_eq!((kind, message), (ErrorKind::Unsupported, too_long(&socket))); + // A store whose path is short, but long once its links are resolved, at which OpenVMM listens. + let fixture = Fixture::in_store( + |directory| { + let link = directory.join("l"); + std::os::unix::fs::symlink(deep(directory), &link).unwrap(); + link.join("templates") + }, + |_, _, _| {}, + ); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert!(socket.as_os_str().len() < 108, "{}", socket.display()); + let resolved = filesystem::canonicalize(socket.parent().unwrap()) + .unwrap() + .join(socket.file_name().unwrap()); + assert_eq!( + (kind, message), + (ErrorKind::Unsupported, too_long(&resolved)) + ); + // A store whose path holds a comma or an equals sign, which OpenVMM's option `listen=` + // takes for the end of the path. + for name in ["a,b", "a=b"] { + let fixture = Fixture::in_store( + |directory| { + let parent = directory.join(name); + assert_eq!(platform::create_owner_only_dir(&parent).unwrap(), None); + parent.join("templates") + }, + |_, _, _| {}, + ); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert_eq!( + (kind, message), + ( + ErrorKind::Unsupported, + format!( + "the path of the control socket {} of a clone's VMM must hold no ',' or '=', \ + which end the path in OpenVMM's option `listen=`", + socket.display() + ) + ) + ); + } + // A store whose path is not valid UTF-8, as OpenVMM takes the paths of its sockets. + let fixture = Fixture::in_store( + |directory| { + use std::os::unix::ffi::OsStrExt; + + let parent = directory.join(std::ffi::OsStr::from_bytes(b"a\xffb")); + assert_eq!(platform::create_owner_only_dir(&parent).unwrap(), None); + parent.join("templates") + }, + |_, _, _| {}, + ); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert!(socket.to_str().is_none(), "{}", socket.display()); + assert_eq!( + (kind, message), + ( + ErrorKind::Unsupported, + format!( + "the path of the socket {} of a clone's launch must be valid UTF-8, as OpenVMM \ + takes it", + socket.display() + ) + ) + ); + // A store whose path is valid UTF-8, but not once its links are resolved, as OpenVMM takes + // the path at which it listens. + let fixture = Fixture::in_store( + |directory| { + use std::os::unix::ffi::OsStrExt; + + let target = directory.join(std::ffi::OsStr::from_bytes(b"a\xffb")); + assert_eq!(platform::create_owner_only_dir(&target).unwrap(), None); + let link = directory.join("u"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + link.join("templates") + }, + |_, _, _| {}, + ); + let ((kind, message), socket) = refused(&fixture, "sandbox"); + assert!(socket.to_str().is_some(), "{}", socket.display()); + let resolved = filesystem::canonicalize(socket.parent().unwrap()) + .unwrap() + .join(socket.file_name().unwrap()); + assert!(resolved.to_str().is_none(), "{}", resolved.display()); + assert_eq!( + (kind, message), + ( + ErrorKind::Unsupported, + format!( + "the path of the socket {} of a clone's launch must be valid UTF-8, as OpenVMM \ + takes it", + resolved.display() + ) + ) + ); + // A payload private to the current user, as a plan requires, but whose mode is not 0700, as + // OpenVMM requires of the directory of its control socket. An acquisition refuses such a + // payload too, so its mode changes here once the lease is granted. + let fixture = Fixture::new(); + complete_snapshot(&fixture); + let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); + let mut acquired = fixture.acquire("sandbox"); + let plan = fixture + .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) + .unwrap(); + let payload = fixture.store.payload_dir(fixture.generation.generation()); + fs::set_permissions(&payload, fs::Permissions::from_mode(0o500)).unwrap(); + let refusal = failure(plan.launch_with( + &mut acquired, + Path::new(OPENVMM), + &runtime_of(&fixture), + hooks, + )); + fs::set_permissions(&payload, fs::Permissions::from_mode(0o700)).unwrap(); + assert_unlaunched(&fixture, &acquired); + assert_eq!( + refusal, + ( + ErrorKind::Io, + format!( + "the payload of the generation {} has mode 500, not the mode 700 that the \ + directory of a control socket must have", + payload.display() + ) + ) + ); +} + +/// A clone's launch in a process of its own, which [`run_launch_role`] runs: it leases the store +/// generation [`ROLE_GENERATION`] of the store in [`ROLE_STORE`] to the sandbox in +/// [`ROLE_SANDBOX`], plans its clone of the image in [`ROLE_IMAGE`] in RAM, and launches it with a +/// [`restore_vmm_role`] in [`ROLE_VMM`] as its VMM; and it exits without unwinding where +/// [`ROLE_CRASH`] says, as a crash would, once it has printed the reference of the VMM that it +/// spawned, if any. +#[test] +#[ignore = "a clone's launch that the launch tests run in a process of its own"] +fn launch_role() { + let (Some(dir), Ok(generation), Some(sandbox), Some(image), Ok(step), Some(vmm)) = ( + env::var_os(ROLE_STORE), + env::var(ROLE_GENERATION), + env::var_os(ROLE_SANDBOX).map(PathBuf::from), + env::var_os(ROLE_IMAGE).map(PathBuf::from), + env::var(ROLE_CRASH), + env::var_os(ROLE_VMM).map(PathBuf::from), + ) else { + return; + }; + let store = Store::open(dir).unwrap(); + let generation = StoreGeneration::new(generation.parse().unwrap()); + let indexed = store.index().unwrap().get(generation).unwrap().clone(); + let mut acquired = store + .acquire(generation, sandbox, &files(), &after(&indexed, 1), None) + .unwrap(); + let binding = DistroBinding::import(&layer(&image), PolicyVerdict::ACCEPTED).unwrap(); + let plan = ClonePlan::new_with( + &store, + &mut acquired, + &tmpfs_request(&image, &binding), + Instant::now() + CLAIM_RELEASE_WAIT, + NO_CONTENT, + ) + .unwrap(); + let runtime = plan.key().runtime().clone(); + let made = RefCell::new(None); + let command = fake_vmm(&vmm, &made); + let interrupt = |point: Point| -> Result<()> { + if format!("{point:?}").to_lowercase() == step { + crash(&step); + } + Ok(()) + }; + let hooks = Hooks { + command: &command, + interrupt: &interrupt, + }; + let restoring = plan + .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + println!( + "\n{REFERENCE}{}", + serde_json::to_string(restoring.starting().process().reference()).unwrap() + ); + if step == "launched" { + crash(&step); + } + panic!("the clone's launch did not crash at {step}"); +} + +/// Ends this process at once, once it has printed that it crashes at `step`, without unwinding +/// or running destructors, as a crash would. +fn crash(step: &str) -> ! { + println!("\n{CRASHED}{step}"); + std::io::stdout().flush().unwrap(); + std::process::exit(0) +} + +/// Runs [`launch_role`] for the generation of `fixture`, the sandbox in `sandbox`, the image +/// `image`, and a VMM in `vmm`, until it crashes at `step`, and returns the reference of the VMM +/// that it spawned, if it spawned one. +fn run_launch_role( + step: &str, + fixture: &Fixture, + sandbox: &Path, + image: &Path, + vmm: &Path, +) -> Option { + let output = Command::new(env::current_exe().unwrap()) + .args(role("launch_role")) + .env(ROLE_STORE, fixture.store.dir()) + .env(ROLE_GENERATION, fixture.generation.generation().to_string()) + .env(ROLE_SANDBOX, sandbox) + .env(ROLE_IMAGE, image) + .env(ROLE_CRASH, step) + .env(ROLE_VMM, vmm) + .stdin(Stdio::null()) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + let crashed = format!("{CRASHED}{step}"); + assert!( + output.status.success() && stdout.lines().any(|line| line == crashed), + "{}: {stdout}{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + stdout + .lines() + .find_map(|line| line.strip_prefix(REFERENCE)) + .map(|reference| serde_json::from_str(reference).unwrap()) +} + +/// A clone whose launch's caller, a [`launch_role`], crashed, in a fresh fixture: the VMM's +/// working directory, the sandbox, and the lease that the sandbox holds. +struct Crashed { + /// The working directory of the clone's VMM, which ends the VMM as it drops, before the rest. + vmm: VmmDir, + /// The template store, its generation, and the clone's image. + fixture: Fixture, + /// The sandbox's directory. + sandbox: PathBuf, + /// The lease that the sandbox holds. + lease: Lease, + /// The runtime identity under which the clone launched. + runtime: RuntimeIdentity, +} + +impl Crashed { + /// Leases a fresh fixture's generation to a new sandbox, whose clone a [`launch_role`] + /// launches until it crashes at `step`. + fn at(step: &str) -> Self { + let fixture = Fixture::new(); + complete_snapshot(&fixture); + let (image, _) = fixture.image("distro.erofs", IMAGE_LEN); + let sandbox = fixture.provision("sandbox"); + let mut vmm = VmmDir::new(); + vmm.reference = run_launch_role(step, &fixture, &sandbox, &image, vmm.path()); + let lease = Lease::of(&SandboxStore::open(&sandbox).unwrap()) + .unwrap() + .expect("the crash kept the clone's lease"); + let runtime = runtime_of(&fixture); + Self { + vmm, + fixture, + sandbox, + lease, + runtime, + } + } + + /// Takes the lock of the clone's sandbox, as the restarted caller does. + fn lock(&self) -> LockedStore { + SandboxStore::lock_existing(&self.sandbox, &files(), Some(Instant::now() + TIMEOUT)) + .unwrap() + } + + /// Checks that the sandbox holds the clone's activation record, which names the restore of + /// the leased generation, and, on Linux, the restore-ready socket. + fn kept_its_files(&self, locked: &LockedStore) { + let recorded = fs::read(locked.clone_activation_path()).unwrap(); + let record = ActivationRecord::decode(&recorded).unwrap(); + assert_eq!(record.restore().unwrap().template, self.lease.template()); + if cfg!(target_os = "linux") { + assert!(exists(&locked.restore_ready_path())); + } + } + + /// Returns the path of the control socket of the clone's VMM, beside the snapshot that it + /// restores. + fn socket(&self) -> PathBuf { + self.fixture.store.clone_socket_path(&self.lease) + } + + /// Reattaches to the VM of the sandbox of `locked` until the result is determinate, as a + /// caller retries an indeterminate reattach, and returns it. + fn reattach(&self, locked: &mut LockedStore) -> Reattached { + let deadline = Instant::now() + TIMEOUT; + loop { + match supervisor::reattach(locked, &self.runtime).unwrap() { + Reattached::Indeterminate(reason) if Instant::now() < deadline => { + eprintln!("reattach is indeterminate: {reason}"); + thread::sleep(Duration::from_millis(100)); + } + reattached => return reattached, + } + } + } + + /// Checks that the store refuses to release the lease of the sandbox of `locked`, whose + /// launch has not ended, and that the sandbox keeps it. + fn refuses_release(&self, locked: &mut LockedStore) { + let error = self.fixture.store.release(locked).unwrap_err(); + assert_eq!(error.code(), ErrorCode::SandboxRunning, "{error}"); + assert_eq!(Lease::of(locked).unwrap(), Some(self.lease.clone())); + } + + /// Checks that the store releases the lease of the sandbox of `locked`, which removes the + /// clone's files, and nothing of the generation, whose payload holds no socket of the clone. + fn releases(&self, locked: &mut LockedStore) { + assert_eq!( + self.fixture.store.release(locked).unwrap(), + Some(self.lease.clone()) + ); + assert_released(&self.sandbox); + assert!(!exists(&self.socket())); + let snapshot = self.fixture.store.snapshot_dir(&self.fixture.generation); + for name in SNAPSHOT_ARTIFACTS { + assert!(snapshot.as_ref().unwrap().join(name).is_file(), "{name}"); + } + } +} + +#[test] +fn a_launch_whose_caller_crashes_before_it_begins_leaves_files_that_the_release_removes() { + let crashed = Crashed::at("prepared"); + assert_eq!(crashed.vmm.reference, None); + let mut locked = crashed.lock(); + // The launch never began: the sandbox records no launch, and keeps the clone's files. + assert!(locked.journal().unwrap().is_none()); + crashed.kept_its_files(&locked); + match crashed.reattach(&mut locked) { + Reattached::Exited { + record: None, + recovery: Recovery::Clean, + } => {} + other => panic!("unexpected reattach: {other:?}"), + } + crashed.releases(&mut locked); +} + +#[test] +fn a_launch_whose_caller_crashes_once_it_began_ends_as_reattach_recovers_it() { + let crashed = Crashed::at("begun"); + assert_eq!(crashed.vmm.reference, None); + let mut locked = crashed.lock(); + // The journal names the launch that the lease reserved, which spawned nothing, and keeps the + // lease until a reattach ends it. + let journal = locked.journal().unwrap().unwrap(); + assert_eq!(journal.launch(), crashed.lease.launch()); + assert_eq!(journal.state(), LaunchState::Prepared); + crashed.kept_its_files(&locked); + crashed.refuses_release(&mut locked); + match crashed.reattach(&mut locked) { + Reattached::Exited { + record: None, + recovery: Recovery::Exited, + } => {} + other => panic!("unexpected reattach: {other:?}"), + } + crashed.releases(&mut locked); +} + +#[test] +fn a_launch_whose_caller_crashes_once_its_vmm_started_ends_the_vmm_as_reattach_recovers_it() { + let crashed = Crashed::at("launched"); + let reference = crashed.vmm.reference.clone().unwrap(); + let process = Process::open(&reference).unwrap().unwrap(); + // The VMM listens beside the snapshot on Linux, as it has once it tried to send its event. + crashed.vmm.started(); + assert_eq!(exists(&crashed.socket()), cfg!(target_os = "linux")); + let mut locked = crashed.lock(); + // The journal records the VMM, which never committed: reattach ends it through its verified + // reference, and the lease lasts until then. The VMM, which the termination ends, leaves its + // control socket, which the end of the launch removes. + let journal = locked.journal().unwrap().unwrap(); + assert_eq!(journal.launch(), crashed.lease.launch()); + assert_eq!(journal.state(), LaunchState::Spawned); + assert_eq!(journal.process(), Some(&reference)); + if cfg!(target_os = "linux") { + assert_eq!(journal.endpoint(), crashed.socket().to_str().unwrap()); + } + crashed.kept_its_files(&locked); + crashed.refuses_release(&mut locked); + match crashed.reattach(&mut locked) { + Reattached::Exited { + record: None, + recovery: Recovery::Terminated(terminated), + } => assert_eq!(terminated, reference), + other => panic!("unexpected reattach: {other:?}"), + } + assert!(process.wait(Instant::now() + TIMEOUT).unwrap()); + assert!(!exists(&crashed.socket())); + crashed.releases(&mut locked); +} diff --git a/nvx-rs/src/host/template/key.rs b/nvx-rs/src/host/template/key.rs index a28c0d68a..419bb658f 100644 --- a/nvx-rs/src/host/template/key.rs +++ b/nvx-rs/src/host/template/key.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use std::fmt; -use std::net::{Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use sha2::{Digest, Sha256}; @@ -34,6 +34,10 @@ const CLASS_DOMAIN: &[u8] = b"nvx capture class\0"; /// The generation of a CPU profile that a host derives, rather than a built-in one. const HOST_GENERATION: &str = "host"; +/// The prefix of the command line's token that names the DNS server of the guest's resolver, +/// which OpenVMM writes once at most, with an IP address in its canonical spelling. +const DNS_TOKEN: &str = "virtnet_dns="; + /// The versions of the contracts of a template's clones that its runtime identity does not /// record: the snapshot format, the saved device state, the time ABI, the restore packet, and the /// activation record. @@ -362,6 +366,12 @@ fn cpu_profile_problem(profile: &str) -> Option { }) } +/// Returns the guest's network on `nic`, its IPv4 address and prefix length, in the canonical form +/// of a [`SandboxNetwork`]. +fn guest_network_of(nic: &Nic) -> String { + format!("{}/{}", nic.ipv4, nic.prefix_length) +} + /// The invariant inputs of a template's capture, known before it: what a template's adapter /// requests, and the class of the templates that the capture makes. /// @@ -660,10 +670,7 @@ impl TemplateKey { .filter(|_| attached) .map(|network| network.guest_network().to_owned()); // A network that `SandboxNetwork` accepts has the canonical form that this writes. - let found_network = machine - .nic - .as_ref() - .map(|nic| format!("{}/{}", nic.ipv4, nic.prefix_length)); + let found_network = machine.nic.as_ref().map(guest_network_of); if found_network != requested_network { let describe = |network: Option| network.unwrap_or_else(|| "none".to_owned()); return mismatch( @@ -759,6 +766,59 @@ impl TemplateKey { self.machine.nic.is_some() } + /// Returns the guest's network on the template's NIC, its IPv4 address and prefix length, in + /// the canonical form of a [`SandboxNetwork`], such as `10.0.0.2/24`, which OpenVMM restores + /// from the snapshot with the NIC, or `None` for a template without a NIC. + pub(crate) fn guest_network(&self) -> Option { + self.machine.nic.as_ref().map(guest_network_of) + } + + /// Returns the SHA-256 digest of the canonical encoding of the network policy that OpenVMM + /// enforces on the template's NIC, which OpenVMM computed and the manifest records, or `None` + /// for a template without a NIC. + pub(crate) fn network_policy_digest(&self) -> Option<[u8; 32]> { + self.machine.nic.as_ref().map(|nic| nic.policy.digest) + } + + /// Returns the address of the DNS server with which the template's guest configured its + /// resolver at boot: the one that the `virtnet_dns=` token of the command line names, which + /// OpenVMM adds when the policy of the template's NIC lets the guest reach the gateway's DNS + /// server, or `None` for a command line without the token. + /// + /// Fails with [`ErrorKind::Io`](crate::ErrorKind::Io) for a command line that OpenVMM does + /// not write: one with more than one such token, with a token for a template without a NIC, + /// or with a token that does not name one IP address in its canonical spelling. + pub(crate) fn dns_server(&self) -> Result> { + let mut tokens = self + .machine + .command_line + .split_ascii_whitespace() + .filter_map(|token| token.strip_prefix(DNS_TOKEN)); + let Some(token) = tokens.next() else { + return Ok(None); + }; + let malformed = |problem: &str| { + Err(Error::io(format!( + "the command line of the template's key is malformed: {problem}" + ))) + }; + if tokens.next().is_some() { + return malformed(&format!("it holds more than one {DNS_TOKEN} token")); + } + if self.machine.nic.is_none() { + return malformed(&format!( + "it names a DNS server, {token:?}, for a template without a NIC" + )); + } + match token.parse::() { + Ok(server) if server.to_string() == token => Ok(Some(server)), + _ => malformed(&format!( + "its {DNS_TOKEN} token names {token:?}, which is not an IP address in its \ + canonical spelling" + )), + } + } + /// Returns the tag of the filesystem slot, on which a clone's export is bound. pub fn filesystem_tag(&self) -> &str { &self.filesystem_tag diff --git a/nvx-rs/src/host/template/key/tests.rs b/nvx-rs/src/host/template/key/tests.rs index 5203c4292..7a665b828 100644 --- a/nvx-rs/src/host/template/key/tests.rs +++ b/nvx-rs/src/host/template/key/tests.rs @@ -365,6 +365,74 @@ fn a_key_holds_what_its_capture_made() { assert_eq!(made.invariants(), &Invariants::new()); } +#[test] +fn a_key_names_the_dns_server_that_its_command_line_names() { + // Neither manifest of the tests names a DNS server. + assert_eq!(network_key().dns_server().unwrap(), None); + let template = key(&support::class(), &ManifestSpec::template()); + assert_eq!(template.dns_server().unwrap(), None); + let with_tokens = |class: CaptureClass, mut spec: ManifestSpec, tokens: &str| { + spec.contract().command_line.push_str(tokens); + key(&class, &spec) + }; + let with_dns = |tokens| with_tokens(network_class(), ManifestSpec::with_network(), tokens); + for (tokens, server) in [ + ( + " virtnet_dns=10.0.0.1", + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), + ), + ( + " virtnet_dns=fd00::a00:1", + IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0xa00, 1)), + ), + ] { + assert_eq!(with_dns(tokens).dns_server().unwrap(), Some(server)); + } + // OpenVMM writes one token at most, with one address in its canonical spelling, and only + // for a NIC. + let noncanonical = |value: &str| { + format!( + "its virtnet_dns= token names {value:?}, which is not an IP address in its canonical \ + spelling" + ) + }; + let without_nic = with_tokens( + support::class(), + ManifestSpec::template(), + " virtnet_dns=10.0.0.1", + ); + for (made, problem) in [ + ( + with_dns(" virtnet_dns=10.0.0.1 virtnet_dns=10.0.0.1"), + "it holds more than one virtnet_dns= token".to_owned(), + ), + (with_dns(" virtnet_dns="), noncanonical("")), + ( + with_dns(" virtnet_dns=010.0.0.1"), + noncanonical("010.0.0.1"), + ), + ( + with_dns(" virtnet_dns=fd00:0::a00:1"), + noncanonical("fd00:0::a00:1"), + ), + ( + with_dns(" virtnet_dns=10.0.0.1,fd00::a00:1"), + noncanonical("10.0.0.1,fd00::a00:1"), + ), + ( + without_nic, + "it names a DNS server, \"10.0.0.1\", for a template without a NIC".to_owned(), + ), + ] { + let error = made.dns_server().unwrap_err(); + assert_eq!(error.code(), ErrorCode::Io, "{error}"); + assert_eq!( + error.message(), + format!("the command line of the template's key is malformed: {problem}") + ); + } +} + /// Asserts that the capture of `class` whose manifest is `spec` does not match its class, with /// a message that holds `expected`. fn assert_mismatched(class: &CaptureClass, spec: &ManifestSpec, expected: &str) { diff --git a/nvx-rs/src/host/template/mod.rs b/nvx-rs/src/host/template/mod.rs index f0c8ae886..3feb60cd3 100644 --- a/nvx-rs/src/host/template/mod.rs +++ b/nvx-rs/src/host/template/mod.rs @@ -37,6 +37,9 @@ //! its memory target against the template, and refuses a request that the template cannot //! serve, such as an image of another length or block geometry than the template's placeholder, //! before any VMM starts. +//! - [`ClonePlan::launch`] launches the clone, as the launch that its lease reserves: OpenVMM +//! restores the generation's snapshot in place with the clone's devices, and the clone's +//! activation record waits in its sandbox, as a [`Restoring`] describes. //! //! # Canonical encoding //! @@ -68,7 +71,7 @@ mod store; mod test_support; mod wire; -pub use self::clone::{ClonePlan, CloneRequest, CloneUpper}; +pub use self::clone::{ClonePlan, CloneRequest, CloneUpper, Restoring}; pub use self::eligibility::{Downtime, DowntimeSource, Eligibility, HostTime, MAX_DOWNTIME_NS}; pub use self::key::{CaptureClass, ClassIdentity, Invariants, KeyIdentity, TemplateKey}; pub use self::manifest::{ diff --git a/nvx-rs/src/host/template/store.rs b/nvx-rs/src/host/template/store.rs index 8dd9cf4dd..762f45e8c 100644 --- a/nvx-rs/src/host/template/store.rs +++ b/nvx-rs/src/host/template/store.rs @@ -24,20 +24,23 @@ use crate::error::{Error, ErrorCode, ErrorKind, Result}; use crate::host::platform::{self, parse_hex_id}; use crate::host::process::BootId; use crate::host::store::files::{open_lock, write_private}; -use crate::host::store::lease::{ID_LEN, hex, read_private, read_record, store_error}; +use crate::host::store::lease::{ + ID_LEN, StoreRecord, hex, read_identity, read_private, read_record, store_error, +}; +#[cfg(all(test, any(target_os = "linux", windows)))] +pub(super) use self::capture::SNAPSHOT_ARTIFACTS; +pub(super) use self::capture::SOCKET_PATH_LIMIT; pub use self::capture::{CAPTURE_RECORD_NAME, CAPTURES_NAME, Capture, Created, TRANSACTION_NAME}; pub use self::leases::{Acquired, LEASES_NAME, Retirement, TOMBSTONES_NAME}; -pub use crate::host::store::lease::{Lease, LeaseId, STORE_FORMAT, StoreId}; +pub use crate::host::store::lease::{ + GENERATIONS_NAME, Lease, LeaseId, STORE_FORMAT, STORE_RECORD_NAME, StoreId, +}; -/// Name of the store's identity record. -pub const STORE_RECORD_NAME: &str = "store.json"; /// Name of the store's index. pub const INDEX_NAME: &str = "index.json"; /// Name of the lock of the store's index. pub const INDEX_LOCK_NAME: &str = "index.lock"; -/// Name of the directory of the generations' payloads. -pub const GENERATIONS_NAME: &str = "generations"; /// Name of the canonical encoding of a generation's key in its payload. pub const KEY_NAME: &str = "key.bin"; /// Name of a generation's snapshot in its payload. @@ -48,9 +51,6 @@ pub const MAX_GENERATIONS: usize = 1024; /// The store generation that a new store assigns first. const FIRST_GENERATION: StoreGeneration = StoreGeneration::new(NonZeroU64::MIN); -/// Longest identity record, in bytes. -const MAX_STORE_RECORD_LEN: u64 = 4 * 1024; - /// Longest index, in bytes: more than an index of [`MAX_GENERATIONS`] generations takes. const MAX_INDEX_LEN: u64 = 4 * 1024 * 1024; @@ -248,6 +248,8 @@ impl Index { /// capture.json the record of the capture that made it /// key.bin the canonical encoding of its compatibility key /// snapshot/ its snapshot, as OpenVMM published it +/// .sock on Linux, the control socket of the VMM of a clone that leases it, +/// named after the clone's launch, until the lease's release /// leases/ the records of the leases that the store granted /// .json the record of the lease L /// tombstones/ the tombstones of the generations that take no new leases @@ -279,7 +281,9 @@ impl Index { /// of its key and of its capture class, its storage generation, and what its manifest records of /// the capture itself, its creation time, its time contract, and the lengths of its artifacts. /// The payload of a generation never changes once the index lists it: the record of its capture, -/// the canonical encoding of its key, which [`Store::read_key`] reads, and its snapshot. The index +/// the canonical encoding of its key, which [`Store::read_key`] reads, and its snapshot, which its +/// clones restore in place. Only the control sockets of the VMMs of the clones that lease it come +/// and go beside them, on Linux, as [`Store::acquire`] describes. The index /// changes as generations are published and retired, and the records of leases and tombstones as /// clones lease generations, and as generations are quarantined and retired. /// @@ -422,6 +426,14 @@ impl Index { /// verifies through the VM's recorded process, also after a reattach. A caller that reads a /// generation's payload, as a restore does, holds a lease of it. /// +/// On Linux, a payload holds the control socket of the VMM of each clone that leases it, +/// `.sock`, named after the launch that the lease reserves, beside the snapshot that the +/// clone restores, and of no clone whose lease has ended: the release makes the socket's absence +/// durable before it ends the lease. So a payload holds entries other than its capture's while +/// its generation has leases, which nothing that reads an indexed payload may refuse, and a +/// retirement deletes the payload with any socket that a crash, or a removal of a sandbox that +/// never released its lease, left in it. +/// /// A tombstone, which [`Store::tombstone`] writes, quarantines a generation: it takes no new /// lease, and selection skips it, while the leases that it granted keep working. /// [`Store::retire`] tombstones a generation, and deletes it once its leases have drained: its @@ -878,8 +890,8 @@ fn take_lock(dir: &Path, deadline: Option) -> Result { } }, } - let locked = platform::opened_file_identity(&file).map_err(failed)?; - match platform::file_identity(&path) { + let locked = platform::opened_file_id(&file).map_err(failed)?; + match platform::file_id(&path) { Ok(current) if current == locked => return Ok(file), Ok(_) => {} Err(error) if error.kind() == io::ErrorKind::NotFound => {} @@ -904,33 +916,6 @@ fn write_record(path: &Path, record: &T, description: &str) -> Res write_private(path, &text) } -/// Reads the identity of the store in `dir` from its identity record, or returns `None` if the -/// directory holds none. -fn read_identity(dir: &Path) -> Result> { - let path = dir.join(STORE_RECORD_NAME); - let description = "identity record of the template store"; - let Some(record) = read_record::(&path, description, MAX_STORE_RECORD_LEN)? else { - return Ok(None); - }; - record.store.parse().map(Some).map_err(|error: Error| { - Error::io(format!( - "the {description} {} is malformed: {}", - path.display(), - error.message() - )) - }) -} - -/// The identity record of a store, as JSON. -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct StoreRecord { - /// [`STORE_FORMAT`]. - format: u32, - /// The store's identity. - store: String, -} - /// The index of a store, as JSON. #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] diff --git a/nvx-rs/src/host/template/store/capture.rs b/nvx-rs/src/host/template/store/capture.rs index 204ecb438..11b6d95f1 100644 --- a/nvx-rs/src/host/template/store/capture.rs +++ b/nvx-rs/src/host/template/store/capture.rs @@ -52,7 +52,8 @@ const STATE_NAME: &str = "state.bin"; const MEMORY_NAME: &str = "memory.bin"; /// The artifacts of a platform-tier snapshot, in the order of their names: its manifest, its RAM, /// and its saved state. -const SNAPSHOT_ARTIFACTS: [&str; 3] = [MANIFEST_NAME, MEMORY_NAME, STATE_NAME]; +pub(in crate::host::template) const SNAPSHOT_ARTIFACTS: [&str; 3] = + [MANIFEST_NAME, MEMORY_NAME, STATE_NAME]; /// Prefix of the name of the file that backs the RAM of a capture's VM, which OpenVMM creates /// beside the snapshot's destination, and removes as the VM ends. @@ -67,7 +68,7 @@ const STAGING_PREFIX: &str = ".snapshot.staging-"; const SLOT_NAME_LEN: usize = 16; /// Most bytes of a Unix socket's path on Linux, `sun_path`, with its terminating NUL. -const SOCKET_PATH_LIMIT: usize = 108; +pub(in crate::host::template) const SOCKET_PATH_LIMIT: usize = 108; /// Longest record of a capture, in bytes. const MAX_CAPTURE_RECORD_LEN: u64 = 4 * 1024; diff --git a/nvx-rs/src/host/template/store/leases.rs b/nvx-rs/src/host/template/store/leases.rs index 87672428b..e19d73967 100644 --- a/nvx-rs/src/host/template/store/leases.rs +++ b/nvx-rs/src/host/template/store/leases.rs @@ -24,10 +24,14 @@ use crate::host::filesystem; use crate::host::platform; use crate::host::store::files::{remove_if_present, sync_dir}; use crate::host::store::lease::{ - Lease, LeaseId, LeaseRecord, MAX_LEASE_RECORD_LEN, read_binding, read_lease, + Lease, LeaseId, LeaseRecord, MAX_LEASE_RECORD_LEN, clone_socket_name, open_payload, + read_binding, read_binding_in, read_lease, +}; +use crate::host::store::regular::{open_regular_in, was_removed}; +use crate::host::store::{ + CLONE_BINDING_NAME, CallerFiles, JOURNAL_NAME, LaunchId, LockedStore, RUNTIME_NAME, + Reservation, SandboxStore, }; -use crate::host::store::regular::was_removed; -use crate::host::store::{CLONE_BINDING_NAME, CallerFiles, LaunchId, LockedStore, SandboxStore}; use crate::host::template::eligibility::HostTime; use crate::host::template::key::KeyIdentity; @@ -132,22 +136,37 @@ impl Store { /// In one transaction, under the store's lock and then the sandbox's /// [lifecycle lock](crate::host::store#lifecycle-locks), it checks that the store indexes the /// generation, that no tombstone quarantines it, that it is eligible at `now`, as - /// [`Generation::eligibility`] decides it, and that the sandbox is provisioned, records no VM - /// and no launch, and holds no lease; and it writes the lease's record in the store, and then - /// the same record as the sandbox's clone binding, beside its launch journal, each durable - /// before the next step, so that no binding outlasts, through a crash of the host, the record - /// that keeps its generation in the store. It resolves the links of the sandbox's path before - /// it takes the sandbox's lock, and the lease, the binding, and the lock that it returns all - /// name the sandbox by that path, as [`Lease::sandbox`] returns it, which no later change of a - /// link redirects to another directory. It returns the lease, with the sandbox's lock, the only - /// one under which the caller launches the clone, once, as the launch that the lease reserves, - /// [`Lease::launch`], and releases the store's lock. + /// [`Generation::eligibility`] decides it, that its payload, on Linux, is one that a launch of + /// the clone and a release of its lease can use, and that the sandbox is provisioned, records + /// no VM and no launch, and holds no lease; and it writes the lease's record in the store, and + /// then the same record as the sandbox's clone binding, beside its launch journal, each + /// durable before the next step, so that no binding outlasts, through a crash of the host, the + /// record that keeps its generation in the store. It resolves the links of the sandbox's path + /// before it takes the sandbox's lock, and the lease, the binding, and the lock that it + /// returns all name the sandbox by that path, as [`Lease::sandbox`] returns it, which no later + /// change of a link redirects to another directory. It returns the lease, with the sandbox's + /// lock, the only one under which the caller launches the clone, once, as the launch that the + /// lease reserves, [`Lease::launch`], and releases the store's lock. + /// + /// The launch that the lease reserves restores the generation's snapshot in place, and its VMM + /// listens beside the snapshot, as OpenVMM requires of a restore on Linux: at the control + /// socket `.sock`, in the generation's payload, after the launch's identity, rather + /// than at the sandbox's own socket. So a payload holds the sockets of the clones of its + /// generation, each named after a launch, while they lease it, and none once its leases have + /// drained: the VMM removes its socket as it exits, a stop or a recovery of the sandbox + /// removes the one that an abrupt exit leaves, and a [release](Self::release) makes the + /// socket's absence durable before the lease ends. On Windows, the VMM listens at a named + /// pipe, as every launch does. /// /// Fails, and leases nothing: /// /// - with [`ErrorKind::NotFound`] if the store does not index the generation; /// - with [`ErrorCode::TemplateTombstoned`] if a tombstone quarantines it; /// - with [`ErrorCode::TemplateIneligible`] if it is not eligible at `now`; + /// - on Linux, with [`ErrorKind::Io`] if the generation's payload is not a directory of the + /// current user with mode 0700, which a launch of the clone requires, as OpenVMM does of the + /// directory of a control socket, and which a release needs, to remove the clone's socket + /// from the payload and make its absence durable; /// - with [`ErrorCode::SandboxNotProvisioned`] if the directory does not exist, or holds no /// configuration record of the caller, as `files` declares it; /// - with [`ErrorCode::SandboxRunning`] if the sandbox records a VM, or a launch, which may @@ -209,6 +228,12 @@ impl Store { ), )); } + // A release removes the socket of the clone's VMM through a handle of the payload, which + // it opens for reading, so that it can make the removal durable: a payload that it could + // not open, or remove an entry from, would keep the lease and its binding past it. + if cfg!(unix) { + self.check_payload(generation)?; + } let not_provisioned = || { Error::new( ErrorCode::SandboxNotProvisioned, @@ -307,10 +332,29 @@ impl Store { return Err(error); } drop(lock); - locked.reserve(Some(lease.launch().clone())); + locked.reserve(Some(Reservation { + socket: self.clone_socket_path(&lease), + launch: lease.launch().clone(), + })); Ok(Acquired { lease, locked }) } + /// Returns the path of the Linux socket at which the VMM of the clone that holds `lease` + /// listens, its control endpoint: `.sock`, after the launch that the lease reserves, + /// in the payload of the leased generation, beside the snapshot that the VMM restores in + /// place, since OpenVMM requires a restore's control socket in the snapshot's parent. The + /// clone's launch refuses a path that a Unix socket, or OpenVMM's option `listen=`, + /// cannot take. + /// + /// A payload holds the sockets of the clones of its generation only while they lease it: the + /// VMM removes its socket as it exits, a stop or a recovery removes what an abrupt exit + /// leaves, and a [release](Self::release) makes the socket's absence durable before the lease + /// ends. A [retirement](Self::retire) removes the payload with whatever a crash left in it. + pub(crate) fn clone_socket_path(&self, lease: &Lease) -> PathBuf { + self.payload_dir(lease.generation()) + .join(clone_socket_name(lease.launch())) + } + /// Releases the lease that the sandbox of `locked` holds, and returns it, or returns `None` if /// the sandbox holds none. /// @@ -318,32 +362,77 @@ impl Store { /// [`ErrorCode::SandboxRunning`] while the sandbox records a VM or a launch, which may still /// run. The sandbox's store clears those records only once it has verified that the VM's /// process exited, through its recorded [`ProcessRef`](crate::host::process::ProcessRef), as - /// a stop or a [`reattach`](crate::host::supervisor::reattach) does. The release removes the - /// file of the scratch device of a clone whose upper is in RAM, + /// a stop or a [`reattach`](crate::host::supervisor::reattach) does. The release first ends + /// the reservation of the launch that `locked` holds, if it acquired the lease, so that one + /// that fails part of the way, once it removed files that a launch of the clone's plan needs, + /// leaves no launch that the lock may begin. It then removes what the clone kept of its + /// launch: the file of the scratch device of a clone whose upper is in RAM, /// [`CLONE_SCRATCH_NAME`](crate::host::store::CLONE_SCRATCH_NAME), which the clone's - /// [plan](crate::host::template::ClonePlan) created, then the clone binding, and once their - /// removal is durable, the lease's record in the store, so that an interrupted release leaves - /// a record that no binding names, which a [retirement](Self::retire) removes, and never a - /// binding without its record. It takes no lock of the store. + /// [plan](crate::host::template::ClonePlan) created, the clone's activation record, + /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME), and the socket of + /// its restore-ready event, [`RESTORE_READY_NAME`](crate::host::store::RESTORE_READY_NAME); + /// then, on Linux, the control socket of the clone's VMM, `.sock` in the leased + /// generation's payload, after the launch that the lease reserved, whose absence it makes + /// durable; then the clone binding, and once their removal is durable, the lease's record in + /// the store, so that an interrupted release leaves a record that no binding names, which a + /// [retirement](Self::retire) removes, and never a binding without its record. It takes no + /// lock of the store. /// /// The caller takes the sandbox's lock through the lease's path, as [`Lease::sandbox`] - /// returns it, and as [`acquire`](Self::acquire) takes it, which has no links: a link in the - /// path of the lock could change as the release runs, and make it remove another sandbox's - /// binding, whose lock it does not hold. + /// returns it, and as [`acquire`](Self::acquire) takes it, with its links resolved. The + /// release names the entries of the sandbox's directory through one handle of the directory + /// that the path names, once it checked through that handle that the directory holds the + /// lock's file: through it, it checks that the sandbox records no VM and no launch, reads the + /// binding, removes the clone's files and the binding, and makes their absence durable. No + /// change of the path as the release runs, of a link in it or of one of its directories, then + /// makes it check or remove the files of another sandbox, whose lock it does not hold. /// - /// Fails with [`ErrorKind::Other`] if the sandbox holds a lease of another store, or if the - /// lock does not name the sandbox by its lease's path, and then removes nothing. + /// Likewise, on Linux, the release opens the leased generation's payload before it removes + /// anything, through a handle of the store's directory: following no link at the store's + /// directory, at its directory of the generations, or at the payload, and once it checked + /// through that handle that the directory holds this store. It removes the clone's socket and + /// makes its absence durable through the payload's handle, so that neither a link that takes + /// the place of one of them nor a directory that takes the place of the store ever leads it to + /// remove the files of another directory. A directory that holds another store, or none, holds + /// no payload of this store's, so no socket goes. + /// + /// Fails with [`ErrorKind::Other`] if the sandbox holds a lease of another store, if the lock + /// does not name the sandbox by its lease's path, or if the directory that the path names does + /// not hold the lock's file, and with [`ErrorKind::Io`] if the store's directory, its + /// directory of the generations, or the payload is a link, or not a directory, and then + /// removes nothing. pub fn release(&self, locked: &mut LockedStore) -> Result> { - self.release_with(locked, &sync_dir) + self.release_with( + locked, + &|| {}, + &platform::Directory::sync, + &platform::Directory::sync, + ) } - /// Releases a lease as [`release`](Self::release) does, making the removal of its binding - /// durable with `sync`. - fn release_with(&self, locked: &mut LockedStore, sync: SyncDir<'_>) -> Result> { - if locked.journal()?.is_some() || locked.runtime()?.is_some() { - return Err(may_run(locked.dir())); + /// Releases a lease as [`release`](Self::release) does, running `checked` once its checks + /// pass, before it removes anything, and making the removal of the clone's socket durable with + /// `sync_payload`, through the handle of the leased generation's payload, and that of the + /// binding with `sync_sandbox`, through the handle of the sandbox's directory. + fn release_with( + &self, + locked: &mut LockedStore, + checked: &dyn Fn(), + sync_payload: &dyn Fn(&platform::Directory) -> io::Result<()>, + sync_sandbox: &dyn Fn(&platform::Directory) -> io::Result<()>, + ) -> Result> { + let directory = locked.open_dir()?; + // A record whose deletion is pending, on Windows, is removed, as its readers take it. + for name in [JOURNAL_NAME, RUNTIME_NAME] { + let present = open_regular_in(&directory, name).map_err(store_error(format!( + "cannot examine {}", + locked.dir().join(name).display() + )))?; + if present.is_some() { + return Err(may_run(locked.dir())); + } } - let Some(lease) = read_binding(locked.dir())? else { + let Some(lease) = read_binding_in(&directory, locked.dir())? else { return Ok(None); }; if lease.store() != self.id { @@ -363,18 +452,94 @@ impl Store { ))); } let recorded = self.check_records_dir(LEASES_NAME, LEASES_WHAT)?; - // The clone's VM has exited, and the sandbox launches nothing else under the lease, so no - // VM holds the scratch device's file of a RAM upper. The binding's removal makes the - // file's durable too, since both are entries of the sandbox's directory. - remove_if_present(&locked.clone_scratch_path())?; - unbind(locked.dir(), sync)?; + // The payload opens before anything goes, so that a link in its place refuses the release. + let payload = self.open_clone_payload(&lease)?; + checked(); + // A plan of the clone launches under the reservation that the lease's acquisition left in + // the lock, which the release ends before it removes anything, so that a release that + // fails once it removed the clone's files, such as the scratch device's file of an upper + // in RAM, leaves no launch that the lock may begin. locked.reserve(None); + // The clone's VM has exited, and the sandbox launches nothing else under the lease, so no + // VM holds the scratch device's file of a RAM upper, or listens at the clone's control + // socket. The binding's removal makes the removal of the clone's files durable too, since + // all are entries of the sandbox's directory, but not that of the socket, an entry of the + // leased generation's payload, which must be durable before the lease ends, so that a + // generation whose leases have drained holds no clone's socket. + locked.remove_clone_files(&directory)?; + if let Some(payload) = &payload { + self.remove_clone_socket(&lease, payload, sync_payload)?; + } + let binding = locked.dir().join(CLONE_BINDING_NAME); + directory + .remove_file(CLONE_BINDING_NAME) + .map_err(store_error(format!("cannot remove {}", binding.display())))?; + // The binding's absence is durable before its record goes: a binding that outlasted the + // record through a crash would let a retirement delete the generation that it names. + sync_sandbox(&directory).map_err(store_error(format!( + "cannot sync the directory {}", + locked.dir().display() + )))?; if recorded { remove_lease_record(&self.lease_path(lease.id()))?; } Ok(Some(lease)) } + /// Refuses the payload of the generation `generation` unless it is a directory of the current + /// user with mode 0700, as a launch of a clone requires it, OpenVMM requiring that mode of the + /// directory of a control socket, and as a release of the clone's lease needs it on Linux: the + /// release opens the payload for reading, and removes the clone's socket from it. + fn check_payload(&self, generation: StoreGeneration) -> Result<()> { + let payload = self.payload_dir(generation); + let problem = platform::check_store_dir(&payload).map_err(store_error(format!( + "cannot open the payload of the generation {}", + payload.display() + )))?; + match problem { + Some(problem) => Err(Error::io(format!( + "the payload of the generation {} {problem}", + payload.display() + ))), + None => Ok(()), + } + } + + /// Opens the payload of the generation that `lease` leases, which on Linux holds the control + /// socket of the clone's VMM, through the store's directory, following no link, as + /// [`open_payload`] opens it, or returns `None` elsewhere, where the VMM listens at a named + /// pipe, or if the payload is gone, or the store's directory holds another store, or none, + /// since neither holds a socket of this store's clones. + fn open_clone_payload(&self, lease: &Lease) -> Result> { + if !cfg!(unix) { + return Ok(None); + } + open_payload(self.dir(), self.id, lease.generation()) + } + + /// Removes the control socket of the VMM of the clone that holds `lease`, beside the snapshot + /// that the VMM restored, from `payload`, the handle of the leased generation's payload, if it + /// is there, and makes its absence durable with `sync`, through that handle. The VMM removes + /// its socket as it exits, and a stop or a recovery removes the one that an abrupt exit + /// leaves, but neither makes the removal durable. + fn remove_clone_socket( + &self, + lease: &Lease, + payload: &platform::Directory, + sync: &dyn Fn(&platform::Directory) -> io::Result<()>, + ) -> Result<()> { + payload + .remove_file(&clone_socket_name(lease.launch())) + .map_err(store_error(format!( + "cannot remove {}", + self.clone_socket_path(lease).display() + )))?; + sync(payload).map_err(store_error(format!( + "cannot sync the directory {}", + self.payload_dir(lease.generation()).display() + ))) + } + /// Returns the leases of the generation `generation` that the store's records list, in the /// order of their identities, including those that an interrupted release or removal of /// their sandboxes left, which a [retirement](Self::retire) removes. diff --git a/nvx-rs/src/host/template/store/leases/tests.rs b/nvx-rs/src/host/template/store/leases/tests.rs index 1c8c26d9e..eeee09695 100644 --- a/nvx-rs/src/host/template/store/leases/tests.rs +++ b/nvx-rs/src/host/template/store/leases/tests.rs @@ -18,12 +18,15 @@ use crate::activation::TemplateReference; use crate::host::process::{Process, ProcessRef}; use crate::host::store::files::write_private; use crate::host::store::{ - CLONE_SCRATCH_NAME, FORMAT, LaunchState, RUNTIME_NAME, Recovery, RuntimeIdentity, + CLONE_ACTIVATION_NAME, CLONE_SCRATCH_NAME, FORMAT, LOCK_NAME, LaunchState, RESTORE_READY_NAME, + RUNTIME_NAME, Recovery, RuntimeIdentity, }; use crate::host::supervisor::{self, Reattached, StopOutcome, VmmCommand}; use crate::host::template::eligibility::MAX_DOWNTIME_NS; +use crate::host::template::store::SNAPSHOT_NAME; use crate::host::template::store::tests::{ - link_dir, link_file, publish, publish_with, remove, storage, store as new_store, unlink_dir, + cause, link_dir, link_file, publish, publish_with, remove, storage, store as new_store, + unlink_dir, }; use crate::host::template::test_support::{ManifestSpec, artifact, network_class}; use crate::host::test_support::released; @@ -282,6 +285,29 @@ fn acquisitions_refuse_what_they_cannot_lease_and_lease_nothing() { assert_eq!(error.code(), ErrorCode::TemplateIneligible, "{error}"); assert!(error.message().contains(condition), "{error}"); } + // On Unix, a payload that is not a directory of the current user with mode 0700, which a + // launch of the clone requires, and through which a release ends the lease: it opens the + // payload for reading, and removes the clone's socket from it. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let payload = store.payload_dir(generation); + for mode in [0o300, 0o500] { + fs::set_permissions(&payload, fs::Permissions::from_mode(mode)).unwrap(); + let error = refusal(&store, generation, &sandbox, &now); + fs::set_permissions(&payload, fs::Permissions::from_mode(0o700)).unwrap(); + assert_eq!(error.kind(), ErrorKind::Io, "{error}"); + assert_eq!( + error.message(), + format!( + "the payload of the generation {} has mode {mode:o}, not the mode 700 that \ + the directory of a control socket must have", + payload.display() + ) + ); + } + } // A sandbox that is not provisioned: its directory is missing, or holds no configuration. let unprovisioned = directory.path().join("unprovisioned"); drop(SandboxStore::lock_for_provision(&unprovisioned, &files(), None).unwrap()); @@ -786,32 +812,115 @@ fn a_release_makes_the_removal_of_its_binding_durable_before_it_removes_its_reco .dir() .join(LEASES_NAME) .join(format!("{}.json", lease.id())); - // Each sync notes its directory, and whether the scratch device's file of a clone in RAM, - // the binding, and the record existed then. + let payload = store.payload_dir(generation); + let socket = store.clone_socket_path(&lease); + // Each sync notes its directory, and whether any file that the clone kept of its launch, the + // control socket of its VMM included, the binding, and the record existed then. + let kept = [ + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, + ]; let synced = RefCell::new(Vec::new()); - let sync = |dir: &Path| -> Result<()> { - let scratch = sandbox.join(CLONE_SCRATCH_NAME).exists(); + let note = |dir: &Path| { + let clone = kept + .iter() + .any(|name| fs::symlink_metadata(sandbox.join(name)).is_ok()) + || fs::symlink_metadata(&socket).is_ok(); let bound = sandbox.join(CLONE_BINDING_NAME).exists(); synced .borrow_mut() - .push((dir.to_owned(), scratch, bound, record.exists())); + .push((dir.to_owned(), clone, bound, record.exists())); + }; + // The release syncs the payload through its handle, which holds the generation's snapshot. + let snapshot = platform::file_id(&payload.join(SNAPSHOT_NAME)).unwrap(); + let sync_payload = |directory: &platform::Directory| -> io::Result<()> { + assert_eq!(directory.entry_id(SNAPSHOT_NAME).unwrap(), Some(snapshot)); + note(&payload); Ok(()) }; - fs::write(sandbox.join(CLONE_SCRATCH_NAME), b"").unwrap(); + // The release syncs the sandbox's directory through its handle, which holds the lock's file. + let lock = platform::file_id(&sandbox.join(LOCK_NAME)).unwrap(); + let sync_sandbox = |directory: &platform::Directory| -> io::Result<()> { + assert_eq!(directory.entry_id(LOCK_NAME).unwrap(), Some(lock)); + note(&sandbox); + Ok(()) + }; + for name in kept { + fs::write(sandbox.join(name), b"").unwrap(); + } + // A socket that a VMM on Linux left in the payload, which a file stands for. + if cfg!(unix) { + fs::write(&socket, b"").unwrap(); + } let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); - assert_eq!(store.release_with(&mut locked, &sync).unwrap(), Some(lease)); + assert_eq!( + store + .release_with(&mut locked, &|| {}, &sync_payload, &sync_sandbox) + .unwrap(), + Some(lease) + ); drop(locked); - assert_eq!(*synced.borrow(), [(sandbox.clone(), false, false, true)]); + let mut expected = vec![(sandbox.clone(), false, false, true)]; + if cfg!(unix) { + expected.insert(0, (payload.clone(), false, true, true)); + } + assert_eq!(*synced.borrow(), expected); assert!(!record.exists()); + assert!(fs::symlink_metadata(&socket).is_err()); + // A release under the lock that acquired the lease ends the lock's reservation before it + // removes the clone's files, such as the scratch device's file of an upper in RAM, which a + // launch of the clone's plan needs, so that one that fails after their removal leaves no + // launch to begin. + let Acquired { + lease: again, + mut locked, + } = store + .acquire( + generation, + sandbox.as_path(), + &files(), + &after(&published, 1), + None, + ) + .unwrap(); + assert_eq!(locked.reserved(), Some(again.launch())); + let scratch = sandbox.join(CLONE_SCRATCH_NAME); + fs::write(&scratch, b"").unwrap(); + // A release that cannot make the socket's removal durable keeps the binding, and the lease. + if cfg!(unix) { + let error = store + .release_with( + &mut locked, + &|| {}, + &|_| Err(io::Error::other("the disk failed")), + &|_| Ok(()), + ) + .unwrap_err(); + assert_eq!( + error.message(), + format!("cannot sync the directory {}", payload.display()) + ); + assert_eq!(cause(&error), "the disk failed"); + assert_eq!(binding(&sandbox), Some(again.clone())); + assert_eq!(locked.reserved(), None); + assert!(fs::symlink_metadata(&scratch).is_err()); + } // A release that cannot make the binding's removal durable keeps the record, which a // retirement removes once no binding names it. - let again = leased(&store, &published, &sandbox); - let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); let error = store - .release_with(&mut locked, &|_: &Path| Err(Error::io("the disk failed"))) + .release_with(&mut locked, &|| {}, &|_| Ok(()), &|_| { + Err(io::Error::other("the disk failed")) + }) .unwrap_err(); + assert_eq!(locked.reserved(), None); + assert!(fs::symlink_metadata(&scratch).is_err()); drop(locked); - assert_eq!(error.message(), "the disk failed"); + assert_eq!( + error.message(), + format!("cannot sync the directory {}", sandbox.display()) + ); + assert_eq!(cause(&error), "the disk failed"); assert_eq!(binding(&sandbox), None); assert_eq!(store.leases(generation).unwrap(), [again]); assert_eq!(store.retire(generation, None).unwrap(), Retirement::Retired); @@ -1088,21 +1197,20 @@ fn a_release_takes_the_lock_of_its_sandbox_through_the_path_of_its_lease() { let link = root.join("link"); link_dir(&first, &link); // A lock taken through a link refuses a release, which removes nothing, also once the link - // names another sandbox, whose lease a release would otherwise end without its lock. + // names another sandbox, whose lease a release would otherwise end without its lock: the + // directory that the link names then holds another lock file than the one locked. let mut locked = SandboxStore::lock_existing(link.join("sandbox"), &files(), None).unwrap(); - for retargeted in [false, true] { + for (retargeted, refusal) in [ + (false, "does not name it by the path of its lease"), + (true, "does not hold the lock of its sandbox"), + ] { if retargeted { unlink_dir(&link); link_dir(&second, &link); } let error = store.release(&mut locked).unwrap_err(); assert_eq!(error.kind(), ErrorKind::Other, "{error}"); - assert!( - error - .message() - .contains("does not name it by the path of its lease"), - "{error}" - ); + assert!(error.message().contains(refusal), "{error}"); assert_eq!(binding(&leased_sandbox), Some(lease.clone())); assert_eq!(binding(&other), Some(other_lease.clone())); assert_eq!(store.leases(generation).unwrap(), both); @@ -1114,6 +1222,389 @@ fn a_release_takes_the_lock_of_its_sandbox_through_the_path_of_its_lease() { assert_eq!(store.leases(generation).unwrap(), [other_lease]); } +/// A release whose path comes to name another sandbox, once a directory in the path is renamed, +/// and a link to the other sandbox's parent takes its place: only on Unix, since Windows refuses +/// to rename a directory that holds an open file, such as the lock's. +#[cfg(unix)] +#[test] +fn a_release_ends_the_lease_of_the_sandbox_whose_lock_it_holds_whatever_its_path_names_meanwhile() { + let (directory, store) = new_store(); + let root = filesystem::canonicalize(directory.path()).unwrap(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let (first, second) = (root.join("first"), root.join("second")); + fs::create_dir(&first).unwrap(); + fs::create_dir(&second).unwrap(); + let leased_sandbox = provision(&first, "sandbox"); + let other = provision(&second, "sandbox"); + let lease = leased(&store, &published, &leased_sandbox); + let other_lease = leased(&store, &published, &other); + let kept = [ + CLONE_SCRATCH_NAME, + CLONE_ACTIVATION_NAME, + RESTORE_READY_NAME, + ]; + for sandbox in [&leased_sandbox, &other] { + for name in kept { + fs::write(sandbox.join(name), b"").unwrap(); + } + } + // Moves `parent` to `to`, and links `parent` to the other sandbox's parent. + let swap = |parent: &Path, to: &Path| { + fs::rename(parent, to).unwrap(); + link_dir(&second, parent); + }; + // The other sandbox's files, and its lease, stay. + let other_stays = || { + for name in kept { + assert!(other.join(name).exists(), "{name}"); + } + assert_eq!(binding(&other), Some(other_lease.clone())); + }; + // Once the release has checked the sandbox, its path comes to name the other sandbox, from + // the removal of its files on. + let moved = root.join("moved"); + let mut locked = SandboxStore::lock_existing(&leased_sandbox, &files(), None).unwrap(); + assert_eq!( + store + .release_with( + &mut locked, + &|| swap(&first, &moved), + &platform::Directory::sync, + &platform::Directory::sync + ) + .unwrap(), + Some(lease) + ); + drop(locked); + // The release removed the files and the binding of the sandbox whose lock it held. + let released = moved.join("sandbox"); + for name in kept { + assert!(fs::symlink_metadata(released.join(name)).is_err(), "{name}"); + } + assert_eq!(binding(&released), None); + other_stays(); + assert_eq!( + store.leases(generation).unwrap(), + std::slice::from_ref(&other_lease) + ); + // A release whose path names the other sandbox once the lock is taken, before the release + // begins, refuses, and removes nothing. + let again = leased(&store, &published, &released); + for name in kept { + fs::write(released.join(name), b"").unwrap(); + } + let mut locked = SandboxStore::lock_existing(&released, &files(), None).unwrap(); + let away = root.join("away"); + swap(&moved, &away); + let error = store.release(&mut locked).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Other, "{error}"); + assert!( + error + .message() + .contains("does not hold the lock of its sandbox"), + "{error}" + ); + drop(locked); + let kept_sandbox = away.join("sandbox"); + for name in kept { + assert!(kept_sandbox.join(name).exists(), "{name}"); + } + assert_eq!(binding(&kept_sandbox), Some(again.clone())); + other_stays(); + let mut both = vec![again, other_lease]; + both.sort_by_key(Lease::id); + assert_eq!(store.leases(generation).unwrap(), both); +} + +/// The payload of a generation, which holds the control sockets of the VMMs of its clones on +/// Linux, opens through the store's directory, following no link, or junction on Windows, at the +/// store's directory, at its directory of the generations, or at the payload, once it has found +/// that the store's directory holds the store of the caller's identity. On Unix, a store's +/// directory and its directory of the generations need only the right to be searched, which an +/// owner-only directory may grant without the right to read it. +#[test] +fn the_payload_of_a_generation_opens_through_no_link() { + let (directory, store) = new_store(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let id = store.id(); + let payload = store.payload_dir(generation); + let snapshot = platform::file_id(&payload.join(SNAPSHOT_NAME)).unwrap(); + let opened = open_payload(store.dir(), id, generation).unwrap().unwrap(); + assert_eq!(opened.entry_id(SNAPSHOT_NAME).unwrap(), Some(snapshot)); + drop(opened); + // A generation without a payload has none, nor has a store that is gone, or one of another + // identity. + assert!( + open_payload(store.dir(), id, store_generation(9)) + .unwrap() + .is_none() + ); + let missing = directory.path().join("missing"); + assert!(open_payload(&missing, id, generation).unwrap().is_none()); + let foreign = StoreId::from_bytes([0x5a; 16]); + assert!( + open_payload(store.dir(), foreign, generation) + .unwrap() + .is_none() + ); + // Another store in the store's place, whose generation of the same number has a payload, has + // none of the store's, nor has a directory that holds no store. + let other = Store::create(directory.path().join("other")).unwrap(); + fs::create_dir(other.payload_dir(generation)).unwrap(); + let moved = directory.path().join("moved"); + fs::rename(store.dir(), &moved).unwrap(); + fs::rename(other.dir(), store.dir()).unwrap(); + assert!(open_payload(store.dir(), id, generation).unwrap().is_none()); + assert!( + open_payload(store.dir(), other.id(), generation) + .unwrap() + .is_some() + ); + fs::rename(store.dir(), other.dir()).unwrap(); + fs::create_dir(store.dir()).unwrap(); + assert!(open_payload(store.dir(), id, generation).unwrap().is_none()); + fs::remove_dir(store.dir()).unwrap(); + // A link in the place of the store's directory, even to the store, of the directory of the + // generations, or of the payload, and a file in the place of the store's directory or of the + // payload, are refused. + let generations = store.dir().join(GENERATIONS_NAME); + let refused = |entry: &dyn std::fmt::Display| { + let error = open_payload(store.dir(), id, generation).err().unwrap(); + assert_eq!(error.kind(), ErrorKind::Io, "{error}"); + assert_eq!( + error.message(), + format!( + "cannot open the payload of the generation {}", + payload.display() + ) + ); + assert_eq!( + cause(&error), + format!("{entry} is a link, or not a directory") + ); + }; + link_dir(&moved, store.dir()); + refused(&store.dir().display()); + unlink_dir(store.dir()); + fs::write(store.dir(), b"").unwrap(); + refused(&store.dir().display()); + fs::remove_file(store.dir()).unwrap(); + fs::rename(&moved, store.dir()).unwrap(); + for (path, name, link) in [ + (&payload, generation.to_string(), true), + (&generations, GENERATIONS_NAME.to_owned(), true), + (&payload, generation.to_string(), false), + ] { + fs::rename(path, &moved).unwrap(); + if link { + link_dir(&moved, path); + } else { + fs::write(path, b"").unwrap(); + } + refused(&name); + if link { + unlink_dir(path); + } else { + fs::remove_file(path).unwrap(); + } + fs::rename(&moved, path).unwrap(); + } + assert!(open_payload(store.dir(), id, generation).unwrap().is_some()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mode = |dir: &Path, mode| { + fs::set_permissions(dir, fs::Permissions::from_mode(mode)).unwrap(); + }; + for dir in [&generations, store.dir()] { + mode(dir, 0o300); + } + let opened = open_payload(store.dir(), id, generation); + for dir in [&generations, store.dir()] { + mode(dir, 0o700); + } + let opened = opened.unwrap().unwrap(); + assert_eq!(opened.entry_id(SNAPSHOT_NAME).unwrap(), Some(snapshot)); + opened.sync().unwrap(); + } +} + +/// A release whose generation's payload, or store's directory, a link takes the place of: before +/// the release begins, it refuses, and removes nothing; once it has opened the payload, it removes +/// the socket from the payload that it opened, and the link's target keeps its file. Only on Unix, +/// where the payload holds the socket of the clone's VMM. +#[cfg(unix)] +#[test] +fn a_release_removes_the_socket_from_the_payload_that_it_opened_whatever_its_path_names_meanwhile() +{ + let (directory, store) = new_store(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let sandbox = provision(directory.path(), "sandbox"); + let lease = leased(&store, &published, &sandbox); + let payload = store.payload_dir(generation); + let name = clone_socket_name(lease.launch()); + // A socket that the VMM left in the payload, which a file stands for, and a directory that + // holds a file of its name, to which a link takes the payload's place. + fs::write(payload.join(&name), b"").unwrap(); + let decoy = directory.path().join("decoy"); + fs::create_dir(&decoy).unwrap(); + fs::write(decoy.join(&name), b"").unwrap(); + // A directory of the store's layout, whose generation of the same number holds a file of the + // socket's name, to which a link takes the store's place. + let decoy_store = directory.path().join("decoy-store"); + let decoy_socket = decoy_store + .join(GENERATIONS_NAME) + .join(generation.to_string()) + .join(&name); + fs::create_dir_all(decoy_socket.parent().unwrap()).unwrap(); + fs::write(&decoy_socket, b"").unwrap(); + let moved = directory.path().join("moved"); + let swap = || { + fs::rename(&payload, &moved).unwrap(); + link_dir(&decoy, &payload); + }; + let restore = || { + unlink_dir(&payload); + fs::rename(&moved, &payload).unwrap(); + }; + let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); + let refused = |locked: &mut LockedStore, entry: &dyn std::fmt::Display| { + let error = store.release(locked).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io, "{error}"); + assert_eq!( + error.message(), + format!( + "cannot open the payload of the generation {}", + payload.display() + ) + ); + assert_eq!( + cause(&error), + format!("{entry} is a link, or not a directory") + ); + assert!(decoy.join(&name).exists()); + assert!(decoy_socket.exists()); + assert_eq!(binding(&sandbox), Some(lease.clone())); + }; + swap(); + refused(&mut locked, &generation); + assert!(moved.join(&name).exists()); + restore(); + fs::rename(store.dir(), &moved).unwrap(); + link_dir(&decoy_store, store.dir()); + refused(&mut locked, &store.dir().display()); + unlink_dir(store.dir()); + fs::rename(&moved, store.dir()).unwrap(); + assert!(payload.join(&name).exists()); + assert_eq!( + store + .release_with( + &mut locked, + &swap, + &platform::Directory::sync, + &platform::Directory::sync + ) + .unwrap(), + Some(lease) + ); + drop(locked); + assert!(fs::symlink_metadata(moved.join(&name)).is_err()); + assert!(decoy.join(&name).exists()); + assert!(decoy_socket.exists()); + assert_eq!(binding(&sandbox), None); + restore(); + assert!(store.leases(generation).unwrap().is_empty()); +} + +/// A sandbox's store that clears the runtime record of its clone's VM removes the VMM's control +/// socket through the payload of its generation, as a recovery of its launch does: it refuses a +/// link in the place of the payload or of the template store's directory, keeping the record, and +/// leaves the files of another store in the template store's place. Only on Unix, where the socket +/// is a file. +#[cfg(unix)] +#[test] +fn a_sandbox_s_store_removes_the_socket_of_its_clone_through_no_link() { + let (directory, store) = new_store(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let sandbox = provision(directory.path(), "sandbox"); + let lease = leased(&store, &published, &sandbox); + let payload = store.payload_dir(generation); + let socket = store.clone_socket_path(&lease); + let name = clone_socket_name(lease.launch()); + fs::write(&socket, b"").unwrap(); + let decoy = directory.path().join("decoy"); + fs::create_dir(&decoy).unwrap(); + fs::write(decoy.join(&name), b"").unwrap(); + // Another store, whose generation of the same number holds a file of the socket's name. + let other = Store::create(directory.path().join("other")).unwrap(); + fs::create_dir(other.payload_dir(generation)).unwrap(); + let other_socket = other.payload_dir(generation).join(&name); + fs::write(&other_socket, b"").unwrap(); + // The runtime record of the clone's VM, whose control endpoint is the socket. + let record = json!({ + "format": FORMAT, + "launch": lease.launch().to_string(), + "process": { "pid": 42, "startTime": 4242, "boot": "ab".repeat(16) }, + "endpoint": socket.to_str().unwrap(), + "runtime": serde_json::to_value(runtime()).unwrap(), + }); + let write_record = || { + write_private(&sandbox.join(RUNTIME_NAME), record.to_string().as_bytes()).unwrap(); + }; + write_record(); + let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); + let refused = |locked: &mut LockedStore, entry: &dyn std::fmt::Display| { + let error = locked.clear_runtime().unwrap_err(); + assert_eq!( + error.message(), + format!( + "cannot open the payload of the generation {}", + payload.display() + ) + ); + assert_eq!( + cause(&error), + format!("{entry} is a link, or not a directory") + ); + assert!(decoy.join(&name).exists()); + assert!(other_socket.exists()); + assert!(sandbox.join(RUNTIME_NAME).exists()); + }; + let moved = directory.path().join("moved"); + fs::rename(&payload, &moved).unwrap(); + link_dir(&decoy, &payload); + refused(&mut locked, &generation); + assert!(moved.join(&name).exists()); + unlink_dir(&payload); + fs::rename(&moved, &payload).unwrap(); + fs::rename(store.dir(), &moved).unwrap(); + link_dir(other.dir(), store.dir()); + refused(&mut locked, &store.dir().display()); + unlink_dir(store.dir()); + // Another store in the store's place holds no socket of the clone's, so the record goes, and + // the other store's file stays, as does the socket in the store's own directory. + fs::rename(other.dir(), store.dir()).unwrap(); + locked.clear_runtime().unwrap(); + assert!(!sandbox.join(RUNTIME_NAME).exists()); + fs::rename(store.dir(), other.dir()).unwrap(); + fs::rename(&moved, store.dir()).unwrap(); + assert!(other_socket.exists()); + assert!(socket.exists()); + write_record(); + locked.clear_runtime().unwrap(); + assert!(fs::symlink_metadata(&socket).is_err()); + assert!(!sandbox.join(RUNTIME_NAME).exists()); + assert!(decoy.join(&name).exists()); + assert!(other_socket.exists()); + drop(locked); + assert_eq!(release(&store, &sandbox).unwrap(), Some(lease)); +} + #[test] fn a_retirement_that_finds_its_generation_unindexed_makes_the_index_durable_first() { let (_directory, store) = new_store(); @@ -1744,10 +2235,11 @@ fn clone_role() { }; let store = Store::open(dir).unwrap(); let generation = StoreGeneration::new(generation.parse().unwrap()); - // The step's transaction crashes as it makes its write in `target` durable. + // The step's transaction crashes as it makes its write in `target` durable, or, for a + // release, the removal of the binding, in the sandbox's directory. let target = match step.as_str() { "after-record" => Some(store.dir().join(LEASES_NAME)), - "after-binding" | "releasing" => Some(sandbox.clone()), + "after-binding" => Some(sandbox.clone()), _ => None, }; let sync = |dir: &Path| -> Result<()> { @@ -1758,7 +2250,11 @@ fn clone_role() { }; if step == "releasing" { let mut locked = SandboxStore::lock_existing(&sandbox, &files(), None).unwrap(); - store.release_with(&mut locked, &sync).unwrap(); + store + .release_with(&mut locked, &|| {}, &platform::Directory::sync, &|_| { + crash(&step) + }) + .unwrap(); panic!("the release did not make the removal of its binding durable"); } let now = after(store.index().unwrap().get(generation).unwrap(), 1); @@ -2383,4 +2879,36 @@ mod windows { drop(held_record); assert!(store.leases(generation).unwrap().is_empty()); } + + #[test] + fn a_release_takes_the_files_of_its_sandbox_whose_deletion_is_pending_for_removed() { + let (directory, store) = new_store(); + let published = publish(&store, &ManifestSpec::template(), storage(1)); + let generation = published.generation(); + let sandbox = provision(directory.path(), "sandbox"); + let lease = leased(&store, &published, &sandbox); + // A stop removed the runtime record of the clone's VM, and the journal of its launch, + // which a scanner, say, holds: the release, which reads them through its handle of the + // sandbox's directory, finds that the sandbox records no VM and no launch. + let held: Vec = [JOURNAL_NAME, RUNTIME_NAME] + .into_iter() + .map(|name| { + fs::write(sandbox.join(name), b"{}").unwrap(); + delete_pending(&sandbox.join(name)) + }) + .collect(); + assert_eq!(release(&store, &sandbox).unwrap(), Some(lease)); + assert_eq!(binding(&sandbox), None); + assert!(store.leases(generation).unwrap().is_empty()); + drop(held); + // A release that removed the binding, which a reader holds, and failed before its record + // went, leaves a binding whose deletion is pending, which a release retried finds removed. + let again = leased(&store, &published, &sandbox); + let held_binding = delete_pending(&sandbox.join(CLONE_BINDING_NAME)); + assert_eq!(release(&store, &sandbox).unwrap(), None); + drop(held_binding); + assert_eq!(binding(&sandbox), None); + assert_eq!(store.leases(generation).unwrap(), [again]); + assert_eq!(store.retire(generation, None).unwrap(), Retirement::Retired); + } }