From a7c7226ab7beaa866804abe613b8c1fe5e02ea34 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Narajala Date: Mon, 2 Nov 2026 18:00:09 +0000 Subject: [PATCH 1/3] feat(vm): load signed macOS runtime without dyld overrides Signed-off-by: Vineeth Sai Narajala --- crates/openshell-driver-vm/README.md | 10 +++ crates/openshell-driver-vm/src/lib.rs | 4 +- crates/openshell-driver-vm/src/main.rs | 10 +-- crates/openshell-driver-vm/src/runtime.rs | 79 +++++++++++++++++++- docs/how-it-works/gateways/configuration.mdx | 2 + docs/how-it-works/sandboxes/runtimes.mdx | 43 +++++++++++ skills/debug-openshell-cluster/SKILL.md | 7 ++ 7 files changed, 147 insertions(+), 8 deletions(-) diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index a0371bacfb..d3c354a597 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -337,6 +337,16 @@ driver with the Hypervisor entitlement in `post_install` and owns the `brew services` gateway lifecycle. The service also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless the user explicitly overrides it. +For organization-signed macOS packages, `OPENSHELL_VM_RUNTIME_DIR` selects a +separately signed runtime instead of extracting the embedded libraries. The +managed VM worker temporarily uses this directory as its working directory +while libkrun opens `libkrunfw.5.dylib` by name, then restores its original +working directory. This avoids `DYLD_*` loader variables for the override and +leaves the embedded runtime path unchanged. The directory must contain +`libkrun.dylib` and `libkrunfw.5.dylib` from the matching VM runtime build; +keep it writable only by administrators. See the +[macOS re-signing recipe](../../docs/how-it-works/sandboxes/runtimes.mdx#re-sign-the-macos-vm-driver). + ## TODOs - The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` note in `crates/openshell-gateway/src/vm.rs`. diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index 66041ea034..af53d3ceef 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -47,6 +47,6 @@ pub use lifecycle::{ }; #[cfg(feature = "compute-driver")] pub use runtime::{ - VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, VmMount, VsockPortMap, configured_runtime_dir, - run_vm, + VM_RUNTIME_DIR_ENV, VM_RUNTIME_REEXEC_ENV, VmBackend, VmLaunchConfig, VmMount, VsockPortMap, + configured_runtime_dir, run_vm, }; diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 0ab4743771..9b352287f8 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -7,7 +7,7 @@ use miette::{IntoDiagnostic, Result}; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; #[cfg(target_os = "macos")] -use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; +use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, VM_RUNTIME_REEXEC_ENV, configured_runtime_dir}; use openshell_driver_vm::{ VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, VmMount, VsockPortMap, procguard, run_vm, }; @@ -748,9 +748,9 @@ fn parse_vm_mount_arg(arg: &str) -> std::result::Result { fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { use std::os::unix::process::CommandExt as _; - const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; - - if std::env::var_os(REEXEC_ENV).is_some() { + if std::env::var_os(VM_RUNTIME_REEXEC_ENV).is_some() + || std::env::var_os(VM_RUNTIME_DIR_ENV).is_some() + { return Ok(()); } @@ -783,7 +783,7 @@ fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { .args(&args) .env("DYLD_LIBRARY_PATH", &joined) .env(VM_RUNTIME_DIR_ENV, runtime_dir) - .env(REEXEC_ENV, "1") + .env(VM_RUNTIME_REEXEC_ENV, "1") .exec(); // `exec()` only returns on failure. Err(miette::miette!("failed to re-exec with runtime env: {err}")) diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index ad300d98cf..676d5ad3c0 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -13,6 +13,7 @@ use std::time::Duration; use crate::{embedded_runtime, ffi, procguard, rootfs}; pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR"; +pub const VM_RUNTIME_REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1"; /// PID of the VM worker process (libkrun fork or QEMU). Zero when not running. @@ -333,7 +334,7 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { configure_runtime_loader_env(&runtime_dir)?; raise_nofile_limit(); - let vm = VmContext::create(&runtime_dir, config.log_level)?; + let vm = create_vm_context(&runtime_dir, config.log_level)?; vm.set_vm_config(config.vcpus, config.mem_mib)?; vm.set_disks( &config.root_disk, @@ -439,6 +440,14 @@ pub fn configured_runtime_dir() -> Result { embedded_runtime::ensure_runtime_extracted() } +#[cfg(target_os = "macos")] +fn external_runtime_selected() -> bool { + // The embedded-runtime re-exec sets VM_RUNTIME_DIR_ENV to keep the cache + // path stable. Its marker distinguishes that path from an operator override. + std::env::var_os(VM_RUNTIME_DIR_ENV).is_some() + && std::env::var_os(VM_RUNTIME_REEXEC_ENV).is_none() +} + fn qemu_runtime_dir() -> Result { configured_runtime_dir().map_err(|_| { "QEMU backend requires OPENSHELL_VM_RUNTIME_DIR to be set (pointing to a directory \ @@ -449,6 +458,10 @@ fn qemu_runtime_dir() -> Result { #[cfg(target_os = "macos")] fn configure_runtime_loader_env(runtime_dir: &Path) -> Result<(), String> { + if external_runtime_selected() { + return Ok(()); + } + let existing = std::env::var_os("DYLD_FALLBACK_LIBRARY_PATH"); let mut paths = vec![runtime_dir.to_path_buf()]; if let Some(existing) = existing { @@ -510,6 +523,39 @@ struct VmContext { ctx_id: u32, } +fn create_vm_context(runtime_dir: &Path, log_level: u32) -> Result { + #[cfg(target_os = "macos")] + if external_runtime_selected() { + // The pinned libkrun opens libkrunfw by its bare filename. dyld finds + // that name in the worker's current directory without DYLD_* settings. + // Only the isolated VM launcher changes cwd, and only while libkrun + // initializes its firmware handle. + let runtime_dir = std::fs::canonicalize(runtime_dir).map_err(|e| { + format!( + "resolve VM runtime directory {}: {e}", + runtime_dir.display() + ) + })?; + return with_runtime_cwd(&runtime_dir, || VmContext::create(&runtime_dir, log_level)); + } + + VmContext::create(runtime_dir, log_level) +} + +#[cfg(any(target_os = "macos", test))] +fn with_runtime_cwd( + runtime_dir: &Path, + load: impl FnOnce() -> Result, +) -> Result { + let previous = std::env::current_dir().map_err(|e| format!("read working directory: {e}"))?; + std::env::set_current_dir(runtime_dir) + .map_err(|e| format!("enter VM runtime directory {}: {e}", runtime_dir.display()))?; + let result = load(); + std::env::set_current_dir(&previous) + .map_err(|e| format!("restore working directory {}: {e}", previous.display()))?; + result +} + impl VmContext { fn create(runtime_dir: &Path, log_level: u32) -> Result { let krun = ffi::libkrun(runtime_dir)?; @@ -803,6 +849,37 @@ fn check_kvm_access() -> Result<(), String> { mod tests { use super::*; + #[test] + fn external_runtime_cwd_restores_after_load_error() { + const PROBE_ENV: &str = "__OPENSHELL_TEST_RUNTIME_CWD"; + + if let Some(runtime_dir) = std::env::var_os(PROBE_ENV) { + let previous = std::env::current_dir().expect("working directory"); + let runtime_dir = PathBuf::from(runtime_dir); + let result = with_runtime_cwd(&runtime_dir, || { + assert_eq!(std::env::current_dir().unwrap(), runtime_dir); + Err::<(), String>("load failed".to_string()) + }); + assert_eq!(result.unwrap_err(), "load failed"); + assert_eq!(std::env::current_dir().unwrap(), previous); + return; + } + + // cwd is process-wide, so exercise it in a child test process. + let runtime_dir = tempfile::tempdir().expect("runtime directory"); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("external_runtime_cwd_restores_after_load_error") + .env(PROBE_ENV, runtime_dir.path()) + .output() + .expect("run cwd probe"); + assert!( + output.status.success(), + "cwd probe failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + } + fn qemu_config() -> VmLaunchConfig { VmLaunchConfig { root_disk: PathBuf::from("/rootfs.ext4"), diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 3d5680301d..8515c43bc3 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1586,6 +1586,8 @@ gateway_id = "openshell" state_dir = "/var/lib/openshell/vm" # Where the gateway looks for the openshell-driver-vm subprocess binary. driver_dir = "/usr/local/libexec/openshell" +# For an organization-signed macOS VM driver, set OPENSHELL_VM_RUNTIME_DIR in +# the gateway service environment. It is not a gateway.toml field. default_image = "nvcr.io/nvidia/base/ubuntu:24.04" # Optional override. When omitted, the gateway derives # https://host.openshell.internal: for the VM driver. diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 70f6891ce1..adf14b1f9e 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -214,6 +214,49 @@ Podman also supports `image` mounts, which mount a container image read-only at The MicroVM driver runs each sandbox in its own lightweight VM. It requires host virtualization: Apple Hypervisor on macOS or KVM on Linux. +### Re-sign the macOS VM driver + +For a managed Mac, place the matching `libkrun.dylib` and +`libkrunfw.5.dylib` in an administrator-owned runtime directory. Sign both +libraries and `openshell-driver-vm` with the same Developer ID Application +identity and the hardened runtime. Give the driver only the +`com.apple.security.hypervisor` entitlement. The repository's +[VM driver entitlement file](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-vm/entitlements.plist) +contains that entitlement. + +This example stages the two libraries from the VM runtime archive matching the +driver build. Replace the identity, archive, and paths for your package: + +```shell +mkdir -p runtime +zstd -dc vm-runtime-darwin-aarch64.tar.zst | + tar -xf - -C runtime ./libkrun.dylib ./libkrunfw.5.dylib + +identity='Developer ID Application: Your Organization (TEAMID)' +codesign --force --options runtime --timestamp -s "$identity" runtime/libkrunfw.5.dylib +codesign --force --options runtime --timestamp -s "$identity" runtime/libkrun.dylib +codesign --force --options runtime --timestamp \ + --entitlements crates/openshell-driver-vm/entitlements.plist \ + -s "$identity" openshell-driver-vm + +codesign --verify --strict runtime/libkrunfw.5.dylib +codesign --verify --strict runtime/libkrun.dylib +codesign --verify --strict openshell-driver-vm +``` + +Set `OPENSHELL_VM_RUNTIME_DIR` to the absolute path of that directory in the +gateway's service environment, for example: + +```shell +OPENSHELL_VM_RUNTIME_DIR=/opt/example/openshell/runtime \ + openshell-gateway --config /etc/openshell/gateway.toml +``` + +The managed VM driver inherits the setting. With this override, the VM worker +loads the signed libraries from that directory and resolves firmware without +`DYLD_*` environment variables. Without it, the driver uses its embedded +runtime as before. Keep the runtime directory writable only by administrators. + The VM driver is opt-in. Enable it in the gateway TOML file: ```toml diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 9e56f7117c..ec0ae805a4 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1492,6 +1492,13 @@ Use the VM driver logs and host diagnostics available in the user's environment. upper layer, prepared rootfs, explicit config, or current image. Do not assign `10001:10001` unless the persisted state reports that legacy identity. - Host virtualization support is enabled. +- On macOS with `OPENSHELL_VM_RUNTIME_DIR` set in the gateway service + environment, the directory contains `libkrun.dylib` and + `libkrunfw.5.dylib` from the matching runtime build. Use + `codesign -dv --verbose=2` on both libraries and `openshell-driver-vm` to + compare their Team IDs; an organization-signed hardened driver cannot load + ad-hoc-signed runtime libraries. See the + [macOS signing recipe](https://docs.nvidia.com/openshell/latest/how-it-works/sandboxes/runtimes#re-sign-the-macos-vm-driver). - The sandbox supervisor can establish its authenticated gateway session. For `vm.mounts` (virtiofs host bind mounts) failures, also verify: From 26bb2a2fda42c1a54b1d4eb6504e3b4da9764655 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Narajala Date: Mon, 2 Nov 2026 19:51:36 +0000 Subject: [PATCH 2/3] fix(vm): harden external macOS runtime loading Signed-off-by: Vineeth Sai Narajala --- crates/openshell-driver-vm/src/main.rs | 64 +++++++++++++++++++---- crates/openshell-driver-vm/src/runtime.rs | 51 +++++++++++++++++- 2 files changed, 103 insertions(+), 12 deletions(-) diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 9b352287f8..900cb7896e 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -287,23 +287,50 @@ struct Args { vm_mount: Vec, } -#[tokio::main] -async fn main() -> Result<()> { +fn main() -> Result<()> { let args = Args::parse(); + if args.internal_run_vm { + return run_internal_vm(&args); + } + + // The re-exec marker belongs only to an embedded-runtime VM launcher. + // A gateway service must not pass it through the outer driver to a new + // launcher using an operator-supplied runtime. + #[cfg(target_os = "macos")] + unsafe { + std::env::remove_var(VM_RUNTIME_REEXEC_ENV); + } + + run_async_driver(args) +} + +fn run_internal_vm(args: &Args) -> Result<()> { + ensure_vm_worker_outside_tokio()?; + // This launcher can temporarily change cwd while libkrun loads firmware. + // Keep it outside Tokio so no runtime worker resolves a path during that + // process-wide change. It also forks libkrun before any Tokio threads exist. + maybe_reexec_internal_vm_with_runtime_env()?; + let config = build_vm_launch_config(args).map_err(|err| miette::miette!("{err}"))?; + run_vm(&config).map_err(|err| miette::miette!("{err}")) +} + +fn ensure_vm_worker_outside_tokio() -> Result<()> { + if tokio::runtime::Handle::try_current().is_ok() { + return Err(miette::miette!( + "internal VM worker must start before the Tokio runtime" + )); + } + Ok(()) +} + +#[tokio::main] +async fn run_async_driver(args: Args) -> Result<()> { if let Some(request) = args.internal_prepare_image { openshell_driver_vm::driver::run_image_preparation_worker(&request) .await .map_err(|error| miette::miette!("{error}"))?; return Ok(()); } - if args.internal_run_vm { - // The VM launcher arms procguard after resolving its runtime so its - // libkrun worker cannot outlive the launcher. - maybe_reexec_internal_vm_with_runtime_env()?; - let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; - run_vm(&config).map_err(|err| miette::miette!("{err}"))?; - return Ok(()); - } let _tracing = openshell_otel::install_driver_tracing( openshell_driver_vm::otel_tracing::TRACING, @@ -805,6 +832,23 @@ mod tests { use clap::Parser; use std::path::PathBuf; + #[test] + fn internal_vm_worker_rejects_tokio_context() { + assert!(super::ensure_vm_worker_outside_tokio().is_ok()); + + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("Tokio runtime"); + runtime.block_on(async { + let error = super::ensure_vm_worker_outside_tokio().unwrap_err(); + assert!( + error + .to_string() + .contains("must start before the Tokio runtime") + ); + }); + } + #[test] fn corporate_proxy_flags_parse_into_driver_settings() { let args = Args::parse_from([ diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index 676d5ad3c0..d000ae45c5 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -547,12 +547,24 @@ fn with_runtime_cwd( runtime_dir: &Path, load: impl FnOnce() -> Result, ) -> Result { + use std::os::fd::AsRawFd as _; + let previous = std::env::current_dir().map_err(|e| format!("read working directory: {e}"))?; + // A deployment can rename the launcher's original directory while libkrun + // loads. Keep an open handle so restoration does not depend on its pathname. + // Opening "." can fail for an execute-only directory; retain the path as a + // fallback so that case keeps working. + let previous_fd = std::fs::File::open(".").ok(); std::env::set_current_dir(runtime_dir) .map_err(|e| format!("enter VM runtime directory {}: {e}", runtime_dir.display()))?; let result = load(); - std::env::set_current_dir(&previous) - .map_err(|e| format!("restore working directory {}: {e}", previous.display()))?; + let restored = previous_fd + .as_ref() + .is_some_and(|fd| unsafe { libc::fchdir(fd.as_raw_fd()) == 0 }); + if !restored { + std::env::set_current_dir(&previous) + .map_err(|e| format!("restore working directory {}: {e}", previous.display()))?; + } result } @@ -880,6 +892,41 @@ mod tests { assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); } + #[test] + fn external_runtime_cwd_restores_after_original_dir_is_renamed() { + const PROBE_ENV: &str = "__OPENSHELL_TEST_RENAMED_CWD"; + + if let Some(root) = std::env::var_os(PROBE_ENV) { + let root = PathBuf::from(root); + let previous = root.join("previous"); + let moved = root.join("moved"); + let runtime = root.join("runtime"); + std::env::set_current_dir(&previous).expect("enter original directory"); + let result = with_runtime_cwd(&runtime, || { + std::fs::rename(&previous, &moved).expect("rename original directory"); + Ok::<(), String>(()) + }); + assert!(result.is_ok(), "{result:?}"); + assert_eq!(std::env::current_dir().unwrap(), moved); + return; + } + + let root = tempfile::tempdir().expect("test root"); + std::fs::create_dir(root.path().join("previous")).unwrap(); + std::fs::create_dir(root.path().join("runtime")).unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("external_runtime_cwd_restores_after_original_dir_is_renamed") + .env(PROBE_ENV, root.path()) + .output() + .expect("run cwd probe"); + assert!( + output.status.success(), + "cwd probe failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); + } + fn qemu_config() -> VmLaunchConfig { VmLaunchConfig { root_disk: PathBuf::from("/rootfs.ext4"), From a5a78cb33a9daa11eb51443a39d935cd22248988 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Narajala Date: Mon, 2 Nov 2026 21:05:57 +0000 Subject: [PATCH 3/3] fix(vm): load external macOS firmware by absolute path Signed-off-by: Vineeth Sai Narajala --- crates/openshell-driver-vm/README.md | 15 +-- crates/openshell-driver-vm/src/ffi.rs | 105 +++++++++++++----- crates/openshell-driver-vm/src/main.rs | 4 +- crates/openshell-driver-vm/src/runtime.rs | 100 +---------------- docs/how-it-works/sandboxes/runtimes.mdx | 38 ++++--- skills/debug-openshell-cluster/SKILL.md | 7 +- tasks/scripts/vm/build-libkrun-macos.sh | 10 ++ .../libkrun-macos-absolute-firmware.patch | 46 ++++++++ 8 files changed, 176 insertions(+), 149 deletions(-) create mode 100644 tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index d3c354a597..ed8f0e756d 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -339,13 +339,14 @@ so driver choice remains automatic unless the user explicitly overrides it. For organization-signed macOS packages, `OPENSHELL_VM_RUNTIME_DIR` selects a separately signed runtime instead of extracting the embedded libraries. The -managed VM worker temporarily uses this directory as its working directory -while libkrun opens `libkrunfw.5.dylib` by name, then restores its original -working directory. This avoids `DYLD_*` loader variables for the override and -leaves the embedded runtime path unchanged. The directory must contain -`libkrun.dylib` and `libkrunfw.5.dylib` from the matching VM runtime build; -keep it writable only by administrators. See the -[macOS re-signing recipe](../../docs/how-it-works/sandboxes/runtimes.mdx#re-sign-the-macos-vm-driver). +driver resolves both dylibs to canonical absolute paths inside that directory +and passes the firmware path to libkrun before creating a VM context. This +avoids `DYLD_*` loader variables and process-wide working-directory changes. +The runtime must be built with the `krun_set_fw_path` entry point added by +`build-libkrun-macos.sh`; older runtime archives lack it and are rejected. +The directory must contain `libkrun.dylib` and `libkrunfw.5.dylib` from +the matching VM runtime build. Keep it writable only by administrators. See +the [macOS re-signing recipe](../../docs/how-it-works/sandboxes/runtimes.mdx#re-sign-the-macos-vm-driver). ## TODOs diff --git a/crates/openshell-driver-vm/src/ffi.rs b/crates/openshell-driver-vm/src/ffi.rs index db1deb619c..27ae3f7784 100644 --- a/crates/openshell-driver-vm/src/ffi.rs +++ b/crates/openshell-driver-vm/src/ffi.rs @@ -12,7 +12,11 @@ use std::sync::OnceLock; use libc::c_char; use libloading::Library; +#[cfg(target_os = "macos")] +use crate::runtime::external_runtime_selected; use crate::runtime::validate_runtime_dir; +#[cfg(target_os = "macos")] +use std::os::unix::ffi::OsStrExt as _; pub const KRUN_LOG_TARGET_DEFAULT: i32 = -1; pub const KRUN_LOG_LEVEL_OFF: u32 = 0; @@ -27,6 +31,8 @@ pub const KRUN_LOG_OPTION_NO_ENV: u32 = 1; type KrunInitLog = unsafe extern "C" fn(target_fd: i32, level: u32, style: u32, options: u32) -> i32; type KrunCreateCtx = unsafe extern "C" fn() -> i32; +#[cfg(target_os = "macos")] +type KrunSetFwPath = unsafe extern "C" fn(*const c_char) -> i32; type KrunFreeCtx = unsafe extern "C" fn(ctx_id: u32) -> i32; type KrunSetVmConfig = unsafe extern "C" fn(ctx_id: u32, num_vcpus: u8, ram_mib: u32) -> i32; type KrunAddDisk = unsafe extern "C" fn( @@ -112,13 +118,39 @@ pub fn required_runtime_lib_name() -> &'static str { impl LibKrun { fn load(runtime_dir: &Path) -> Result { let libkrun_path = runtime_dir.join(required_runtime_lib_name()); - preload_runtime_support_libraries(runtime_dir)?; + #[cfg(target_os = "macos")] + let (libkrun_path, firmware_path) = if external_runtime_selected() { + ( + canonical_runtime_library(runtime_dir, required_runtime_lib_name())?, + Some(canonical_runtime_library(runtime_dir, "libkrunfw.5.dylib")?), + ) + } else { + (libkrun_path, None) + }; + #[cfg(not(target_os = "macos"))] + let firmware_path: Option = None; + + preload_runtime_support_libraries(runtime_dir, firmware_path.as_deref())?; let library = Box::leak(Box::new(unsafe { Library::new(&libkrun_path) .map_err(|e| format!("load libkrun from {}: {e}", libkrun_path.display()))? })); + #[cfg(target_os = "macos")] + if let Some(firmware_path) = &firmware_path { + let set_fw_path: KrunSetFwPath = + load_symbol(library, b"krun_set_fw_path\0", &libkrun_path).map_err(|e| { + format!("{e}; rebuild the macOS VM runtime with absolute firmware path support") + })?; + let path = CString::new(firmware_path.as_os_str().as_bytes()) + .map_err(|e| format!("invalid firmware path {}: {e}", firmware_path.display()))?; + let result = unsafe { set_fw_path(path.as_ptr()) }; + if result < 0 { + return Err(format!("krun_set_fw_path failed with error code {result}")); + } + } + Ok(Self { krun_init_log: load_symbol(library, b"krun_init_log\0", &libkrun_path)?, krun_create_ctx: load_symbol(library, b"krun_create_ctx\0", &libkrun_path)?, @@ -150,31 +182,52 @@ impl LibKrun { } } -fn preload_runtime_support_libraries(runtime_dir: &Path) -> Result, String> { - let entries = std::fs::read_dir(runtime_dir) - .map_err(|e| format!("read {}: {e}", runtime_dir.display()))?; - - let mut support_libs: Vec = entries - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| { - #[cfg(target_os = "macos")] - { - name.starts_with("libkrunfw") - && Path::new(name) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("dylib")) - } - #[cfg(not(target_os = "macos"))] - { - name.starts_with("libkrunfw") && name.contains(".so") - } - }) - }) - .collect(); +#[cfg(target_os = "macos")] +fn canonical_runtime_library(runtime_dir: &Path, name: &str) -> Result { + let root = std::fs::canonicalize(runtime_dir) + .map_err(|e| format!("resolve runtime directory {}: {e}", runtime_dir.display()))?; + let path = std::fs::canonicalize(root.join(name)) + .map_err(|e| format!("resolve runtime library {}: {e}", root.join(name).display()))?; + if !path.starts_with(&root) || !path.is_file() { + return Err(format!( + "runtime library must be a file inside {}", + root.display() + )); + } + Ok(path) +} + +fn preload_runtime_support_libraries( + runtime_dir: &Path, + explicit_firmware: Option<&Path>, +) -> Result, String> { + let mut support_libs: Vec = if let Some(path) = explicit_firmware { + vec![path.to_path_buf()] + } else { + let entries = std::fs::read_dir(runtime_dir) + .map_err(|e| format!("read {}: {e}", runtime_dir.display()))?; + entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + #[cfg(target_os = "macos")] + { + name.starts_with("libkrunfw") + && Path::new(name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("dylib")) + } + #[cfg(not(target_os = "macos"))] + { + name.starts_with("libkrunfw") && name.contains(".so") + } + }) + }) + .collect() + }; support_libs.sort(); for path in &support_libs { diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 900cb7896e..e1b7446804 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -306,9 +306,7 @@ fn main() -> Result<()> { fn run_internal_vm(args: &Args) -> Result<()> { ensure_vm_worker_outside_tokio()?; - // This launcher can temporarily change cwd while libkrun loads firmware. - // Keep it outside Tokio so no runtime worker resolves a path during that - // process-wide change. It also forks libkrun before any Tokio threads exist. + // libkrun forks its worker before any Tokio threads exist. maybe_reexec_internal_vm_with_runtime_env()?; let config = build_vm_launch_config(args).map_err(|err| miette::miette!("{err}"))?; run_vm(&config).map_err(|err| miette::miette!("{err}")) diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index d000ae45c5..06a3388c51 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -441,7 +441,7 @@ pub fn configured_runtime_dir() -> Result { } #[cfg(target_os = "macos")] -fn external_runtime_selected() -> bool { +pub(crate) fn external_runtime_selected() -> bool { // The embedded-runtime re-exec sets VM_RUNTIME_DIR_ENV to keep the cache // path stable. Its marker distinguishes that path from an operator override. std::env::var_os(VM_RUNTIME_DIR_ENV).is_some() @@ -526,48 +526,18 @@ struct VmContext { fn create_vm_context(runtime_dir: &Path, log_level: u32) -> Result { #[cfg(target_os = "macos")] if external_runtime_selected() { - // The pinned libkrun opens libkrunfw by its bare filename. dyld finds - // that name in the worker's current directory without DYLD_* settings. - // Only the isolated VM launcher changes cwd, and only while libkrun - // initializes its firmware handle. let runtime_dir = std::fs::canonicalize(runtime_dir).map_err(|e| { format!( "resolve VM runtime directory {}: {e}", runtime_dir.display() ) })?; - return with_runtime_cwd(&runtime_dir, || VmContext::create(&runtime_dir, log_level)); + return VmContext::create(&runtime_dir, log_level); } VmContext::create(runtime_dir, log_level) } -#[cfg(any(target_os = "macos", test))] -fn with_runtime_cwd( - runtime_dir: &Path, - load: impl FnOnce() -> Result, -) -> Result { - use std::os::fd::AsRawFd as _; - - let previous = std::env::current_dir().map_err(|e| format!("read working directory: {e}"))?; - // A deployment can rename the launcher's original directory while libkrun - // loads. Keep an open handle so restoration does not depend on its pathname. - // Opening "." can fail for an execute-only directory; retain the path as a - // fallback so that case keeps working. - let previous_fd = std::fs::File::open(".").ok(); - std::env::set_current_dir(runtime_dir) - .map_err(|e| format!("enter VM runtime directory {}: {e}", runtime_dir.display()))?; - let result = load(); - let restored = previous_fd - .as_ref() - .is_some_and(|fd| unsafe { libc::fchdir(fd.as_raw_fd()) == 0 }); - if !restored { - std::env::set_current_dir(&previous) - .map_err(|e| format!("restore working directory {}: {e}", previous.display()))?; - } - result -} - impl VmContext { fn create(runtime_dir: &Path, log_level: u32) -> Result { let krun = ffi::libkrun(runtime_dir)?; @@ -861,72 +831,6 @@ fn check_kvm_access() -> Result<(), String> { mod tests { use super::*; - #[test] - fn external_runtime_cwd_restores_after_load_error() { - const PROBE_ENV: &str = "__OPENSHELL_TEST_RUNTIME_CWD"; - - if let Some(runtime_dir) = std::env::var_os(PROBE_ENV) { - let previous = std::env::current_dir().expect("working directory"); - let runtime_dir = PathBuf::from(runtime_dir); - let result = with_runtime_cwd(&runtime_dir, || { - assert_eq!(std::env::current_dir().unwrap(), runtime_dir); - Err::<(), String>("load failed".to_string()) - }); - assert_eq!(result.unwrap_err(), "load failed"); - assert_eq!(std::env::current_dir().unwrap(), previous); - return; - } - - // cwd is process-wide, so exercise it in a child test process. - let runtime_dir = tempfile::tempdir().expect("runtime directory"); - let output = std::process::Command::new(std::env::current_exe().unwrap()) - .arg("external_runtime_cwd_restores_after_load_error") - .env(PROBE_ENV, runtime_dir.path()) - .output() - .expect("run cwd probe"); - assert!( - output.status.success(), - "cwd probe failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); - } - - #[test] - fn external_runtime_cwd_restores_after_original_dir_is_renamed() { - const PROBE_ENV: &str = "__OPENSHELL_TEST_RENAMED_CWD"; - - if let Some(root) = std::env::var_os(PROBE_ENV) { - let root = PathBuf::from(root); - let previous = root.join("previous"); - let moved = root.join("moved"); - let runtime = root.join("runtime"); - std::env::set_current_dir(&previous).expect("enter original directory"); - let result = with_runtime_cwd(&runtime, || { - std::fs::rename(&previous, &moved).expect("rename original directory"); - Ok::<(), String>(()) - }); - assert!(result.is_ok(), "{result:?}"); - assert_eq!(std::env::current_dir().unwrap(), moved); - return; - } - - let root = tempfile::tempdir().expect("test root"); - std::fs::create_dir(root.path().join("previous")).unwrap(); - std::fs::create_dir(root.path().join("runtime")).unwrap(); - let output = std::process::Command::new(std::env::current_exe().unwrap()) - .arg("external_runtime_cwd_restores_after_original_dir_is_renamed") - .env(PROBE_ENV, root.path()) - .output() - .expect("run cwd probe"); - assert!( - output.status.success(), - "cwd probe failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed")); - } - fn qemu_config() -> VmLaunchConfig { VmLaunchConfig { root_disk: PathBuf::from("/rootfs.ext4"), diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index adf14b1f9e..5001febf68 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -217,15 +217,27 @@ The MicroVM driver runs each sandbox in its own lightweight VM. It requires host ### Re-sign the macOS VM driver For a managed Mac, place the matching `libkrun.dylib` and -`libkrunfw.5.dylib` in an administrator-owned runtime directory. Sign both -libraries and `openshell-driver-vm` with the same Developer ID Application -identity and the hardened runtime. Give the driver only the -`com.apple.security.hypervisor` entitlement. The repository's -[VM driver entitlement file](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-vm/entitlements.plist) -contains that entitlement. - -This example stages the two libraries from the VM runtime archive matching the -driver build. Replace the identity, archive, and paths for your package: +`libkrunfw.5.dylib` in an administrator-owned runtime directory. Use a +`vm-runtime-darwin-aarch64` archive built with this driver revision. The +macOS runtime build adds `krun_set_fw_path` to the pinned libkrun so the +driver can pass a canonical absolute firmware path. Earlier archives lack +that entry point and cannot boot a hardened driver through this override. + +Sign both libraries and `openshell-driver-vm` with the same Developer ID +Application identity and the hardened runtime. Apple's +[library validation rule](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.security.cs.disable-library-validation) +allows a hardened process to load libraries signed with its Team ID. The +repository's [VM driver entitlement file](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-vm/entitlements.plist) +provides `com.apple.security.hypervisor`. A VM boot with matching Developer +ID signatures has not yet been verified. + +If the libraries are ad-hoc signed or signed by a different Team ID, the +driver also needs `com.apple.security.cs.disable-library-validation`. The +ad-hoc hardened-runtime VM proof used both entitlements. It did not need +`com.apple.security.cs.allow-dyld-environment-variables`. + +This example stages the two libraries from a newly built matching archive. +Replace the identity, archive, and paths for your package: ```shell mkdir -p runtime @@ -252,10 +264,10 @@ OPENSHELL_VM_RUNTIME_DIR=/opt/example/openshell/runtime \ openshell-gateway --config /etc/openshell/gateway.toml ``` -The managed VM driver inherits the setting. With this override, the VM worker -loads the signed libraries from that directory and resolves firmware without -`DYLD_*` environment variables. Without it, the driver uses its embedded -runtime as before. Keep the runtime directory writable only by administrators. +The managed VM driver inherits the setting. It opens both dylibs by canonical +absolute path inside the selected directory and does not depend on `DYLD_*` +environment variables. Without the override, it uses its embedded runtime. +Keep the runtime directory writable only by administrators. The VM driver is opt-in. Enable it in the gateway TOML file: diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ec0ae805a4..241c0dc54d 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1496,8 +1496,11 @@ Use the VM driver logs and host diagnostics available in the user's environment. environment, the directory contains `libkrun.dylib` and `libkrunfw.5.dylib` from the matching runtime build. Use `codesign -dv --verbose=2` on both libraries and `openshell-driver-vm` to - compare their Team IDs; an organization-signed hardened driver cannot load - ad-hoc-signed runtime libraries. See the + compare their Team IDs. A hardened driver needs + `com.apple.security.cs.disable-library-validation` for ad-hoc libraries or + libraries signed by another Team ID. The runtime must export + `krun_set_fw_path`; older archives lack the absolute firmware path support. + See the [macOS signing recipe](https://docs.nvidia.com/openshell/latest/how-it-works/sandboxes/runtimes#re-sign-the-macos-vm-driver). - The sandbox supervisor can establish its authenticated gateway session. diff --git a/tasks/scripts/vm/build-libkrun-macos.sh b/tasks/scripts/vm/build-libkrun-macos.sh index 1afee1c937..be573f3afe 100755 --- a/tasks/scripts/vm/build-libkrun-macos.sh +++ b/tasks/scripts/vm/build-libkrun-macos.sh @@ -170,6 +170,16 @@ cd .. LIBKRUN_COMMIT=$(git -C libkrun rev-parse HEAD) echo " Commit: ${LIBKRUN_COMMIT}" +# The pinned libkrun opens firmware by a bare name. Hardened dyld rejects +# that lookup, so expose a setter for the driver's canonical absolute path. +PATCH="${ROOT}/tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch" +if git -C libkrun apply --unidiff-zero --check "$PATCH"; then + git -C libkrun apply --unidiff-zero "$PATCH" +elif ! git -C libkrun apply --unidiff-zero --reverse --check "$PATCH"; then + echo "Error: libkrun firmware path patch does not match ${LIBKRUN_COMMIT}" >&2 + exit 1 +fi + cd libkrun # ── Build libkrun ─────────────────────────────────────────────────────── diff --git a/tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch b/tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch new file mode 100644 index 0000000000..b2e0c3f9a0 --- /dev/null +++ b/tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch @@ -0,0 +1,46 @@ +diff --git a/src/libkrun/src/lib.rs b/src/libkrun/src/lib.rs +index a797fd1..3c6f9eb 100644 +--- a/src/libkrun/src/lib.rs ++++ b/src/libkrun/src/lib.rs +@@ -29,0 +30,2 @@ use std::ffi::{c_void, CStr}; ++#[cfg(target_os = "macos")] ++use std::ffi::OsStr; +@@ -36,0 +39,2 @@ use std::os::fd::{BorrowedFd, FromRawFd, RawFd}; ++#[cfg(target_os = "macos")] ++use std::os::unix::ffi::OsStrExt; +@@ -40,0 +45,2 @@ use std::sync::LazyLock; ++#[cfg(target_os = "macos")] ++use std::sync::OnceLock; +@@ -116,2 +122,30 @@ fn init_virtual_entry() -> VirtualDirEntry { +-static KRUNFW: LazyLock> = +- LazyLock::new(|| unsafe { libloading::Library::new(KRUNFW_NAME).ok() }); ++// Set by the VM driver before the first context is created. A full path is ++// required because hardened dyld rejects the bare firmware filename. ++#[cfg(target_os = "macos")] ++static KRUNFW_PATH: OnceLock = OnceLock::new(); ++ ++#[cfg(target_os = "macos")] ++#[no_mangle] ++pub unsafe extern "C" fn krun_set_fw_path(path: *const c_char) -> i32 { ++ if path.is_null() { ++ return -libc::EINVAL; ++ } ++ ++ let path = PathBuf::from(OsStr::from_bytes(unsafe { CStr::from_ptr(path) }.to_bytes())); ++ if !path.is_absolute() { ++ return -libc::EINVAL; ++ } ++ ++ match KRUNFW_PATH.set(path) { ++ Ok(()) => KRUN_SUCCESS, ++ Err(_) => -libc::EALREADY, ++ } ++} ++ ++static KRUNFW: LazyLock> = LazyLock::new(|| { ++ #[cfg(target_os = "macos")] ++ if let Some(path) = KRUNFW_PATH.get() { ++ return unsafe { libloading::Library::new(path).ok() }; ++ } ++ unsafe { libloading::Library::new(KRUNFW_NAME).ok() } ++});