diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index a0371bacfb..ed8f0e756d 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -337,6 +337,17 @@ driver with the Hypervisor entitlement in `post_install` and owns the `brew services` gateway lifecycle. The service also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless the user explicitly overrides it. +For organization-signed macOS packages, `OPENSHELL_VM_RUNTIME_DIR` selects a +separately signed runtime instead of extracting the embedded libraries. The +driver resolves both dylibs to canonical absolute paths inside that directory +and passes the firmware path to libkrun before creating a VM context. This +avoids `DYLD_*` loader variables and process-wide working-directory changes. +The runtime must be built with the `krun_set_fw_path` entry point added by +`build-libkrun-macos.sh`; older runtime archives lack it and are rejected. +The directory must contain `libkrun.dylib` and `libkrunfw.5.dylib` from +the matching VM runtime build. Keep it writable only by administrators. See +the [macOS re-signing recipe](../../docs/how-it-works/sandboxes/runtimes.mdx#re-sign-the-macos-vm-driver). + ## TODOs - The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` note in `crates/openshell-gateway/src/vm.rs`. diff --git a/crates/openshell-driver-vm/src/ffi.rs b/crates/openshell-driver-vm/src/ffi.rs index db1deb619c..27ae3f7784 100644 --- a/crates/openshell-driver-vm/src/ffi.rs +++ b/crates/openshell-driver-vm/src/ffi.rs @@ -12,7 +12,11 @@ use std::sync::OnceLock; use libc::c_char; use libloading::Library; +#[cfg(target_os = "macos")] +use crate::runtime::external_runtime_selected; use crate::runtime::validate_runtime_dir; +#[cfg(target_os = "macos")] +use std::os::unix::ffi::OsStrExt as _; pub const KRUN_LOG_TARGET_DEFAULT: i32 = -1; pub const KRUN_LOG_LEVEL_OFF: u32 = 0; @@ -27,6 +31,8 @@ pub const KRUN_LOG_OPTION_NO_ENV: u32 = 1; type KrunInitLog = unsafe extern "C" fn(target_fd: i32, level: u32, style: u32, options: u32) -> i32; type KrunCreateCtx = unsafe extern "C" fn() -> i32; +#[cfg(target_os = "macos")] +type KrunSetFwPath = unsafe extern "C" fn(*const c_char) -> i32; type KrunFreeCtx = unsafe extern "C" fn(ctx_id: u32) -> i32; type KrunSetVmConfig = unsafe extern "C" fn(ctx_id: u32, num_vcpus: u8, ram_mib: u32) -> i32; type KrunAddDisk = unsafe extern "C" fn( @@ -112,13 +118,39 @@ pub fn required_runtime_lib_name() -> &'static str { impl LibKrun { fn load(runtime_dir: &Path) -> Result { let libkrun_path = runtime_dir.join(required_runtime_lib_name()); - preload_runtime_support_libraries(runtime_dir)?; + #[cfg(target_os = "macos")] + let (libkrun_path, firmware_path) = if external_runtime_selected() { + ( + canonical_runtime_library(runtime_dir, required_runtime_lib_name())?, + Some(canonical_runtime_library(runtime_dir, "libkrunfw.5.dylib")?), + ) + } else { + (libkrun_path, None) + }; + #[cfg(not(target_os = "macos"))] + let firmware_path: Option = None; + + preload_runtime_support_libraries(runtime_dir, firmware_path.as_deref())?; let library = Box::leak(Box::new(unsafe { Library::new(&libkrun_path) .map_err(|e| format!("load libkrun from {}: {e}", libkrun_path.display()))? })); + #[cfg(target_os = "macos")] + if let Some(firmware_path) = &firmware_path { + let set_fw_path: KrunSetFwPath = + load_symbol(library, b"krun_set_fw_path\0", &libkrun_path).map_err(|e| { + format!("{e}; rebuild the macOS VM runtime with absolute firmware path support") + })?; + let path = CString::new(firmware_path.as_os_str().as_bytes()) + .map_err(|e| format!("invalid firmware path {}: {e}", firmware_path.display()))?; + let result = unsafe { set_fw_path(path.as_ptr()) }; + if result < 0 { + return Err(format!("krun_set_fw_path failed with error code {result}")); + } + } + Ok(Self { krun_init_log: load_symbol(library, b"krun_init_log\0", &libkrun_path)?, krun_create_ctx: load_symbol(library, b"krun_create_ctx\0", &libkrun_path)?, @@ -150,31 +182,52 @@ impl LibKrun { } } -fn preload_runtime_support_libraries(runtime_dir: &Path) -> Result, String> { - let entries = std::fs::read_dir(runtime_dir) - .map_err(|e| format!("read {}: {e}", runtime_dir.display()))?; - - let mut support_libs: Vec = entries - .filter_map(Result::ok) - .map(|entry| entry.path()) - .filter(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| { - #[cfg(target_os = "macos")] - { - name.starts_with("libkrunfw") - && Path::new(name) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("dylib")) - } - #[cfg(not(target_os = "macos"))] - { - name.starts_with("libkrunfw") && name.contains(".so") - } - }) - }) - .collect(); +#[cfg(target_os = "macos")] +fn canonical_runtime_library(runtime_dir: &Path, name: &str) -> Result { + let root = std::fs::canonicalize(runtime_dir) + .map_err(|e| format!("resolve runtime directory {}: {e}", runtime_dir.display()))?; + let path = std::fs::canonicalize(root.join(name)) + .map_err(|e| format!("resolve runtime library {}: {e}", root.join(name).display()))?; + if !path.starts_with(&root) || !path.is_file() { + return Err(format!( + "runtime library must be a file inside {}", + root.display() + )); + } + Ok(path) +} + +fn preload_runtime_support_libraries( + runtime_dir: &Path, + explicit_firmware: Option<&Path>, +) -> Result, String> { + let mut support_libs: Vec = if let Some(path) = explicit_firmware { + vec![path.to_path_buf()] + } else { + let entries = std::fs::read_dir(runtime_dir) + .map_err(|e| format!("read {}: {e}", runtime_dir.display()))?; + entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + #[cfg(target_os = "macos")] + { + name.starts_with("libkrunfw") + && Path::new(name) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("dylib")) + } + #[cfg(not(target_os = "macos"))] + { + name.starts_with("libkrunfw") && name.contains(".so") + } + }) + }) + .collect() + }; support_libs.sort(); for path in &support_libs { diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index 66041ea034..af53d3ceef 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -47,6 +47,6 @@ pub use lifecycle::{ }; #[cfg(feature = "compute-driver")] pub use runtime::{ - VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, VmMount, VsockPortMap, configured_runtime_dir, - run_vm, + VM_RUNTIME_DIR_ENV, VM_RUNTIME_REEXEC_ENV, VmBackend, VmLaunchConfig, VmMount, VsockPortMap, + configured_runtime_dir, run_vm, }; diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 0ab4743771..e1b7446804 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -7,7 +7,7 @@ use miette::{IntoDiagnostic, Result}; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; #[cfg(target_os = "macos")] -use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; +use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, VM_RUNTIME_REEXEC_ENV, configured_runtime_dir}; use openshell_driver_vm::{ VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, VmMount, VsockPortMap, procguard, run_vm, }; @@ -287,23 +287,48 @@ struct Args { vm_mount: Vec, } -#[tokio::main] -async fn main() -> Result<()> { +fn main() -> Result<()> { let args = Args::parse(); + if args.internal_run_vm { + return run_internal_vm(&args); + } + + // The re-exec marker belongs only to an embedded-runtime VM launcher. + // A gateway service must not pass it through the outer driver to a new + // launcher using an operator-supplied runtime. + #[cfg(target_os = "macos")] + unsafe { + std::env::remove_var(VM_RUNTIME_REEXEC_ENV); + } + + run_async_driver(args) +} + +fn run_internal_vm(args: &Args) -> Result<()> { + ensure_vm_worker_outside_tokio()?; + // libkrun forks its worker before any Tokio threads exist. + maybe_reexec_internal_vm_with_runtime_env()?; + let config = build_vm_launch_config(args).map_err(|err| miette::miette!("{err}"))?; + run_vm(&config).map_err(|err| miette::miette!("{err}")) +} + +fn ensure_vm_worker_outside_tokio() -> Result<()> { + if tokio::runtime::Handle::try_current().is_ok() { + return Err(miette::miette!( + "internal VM worker must start before the Tokio runtime" + )); + } + Ok(()) +} + +#[tokio::main] +async fn run_async_driver(args: Args) -> Result<()> { if let Some(request) = args.internal_prepare_image { openshell_driver_vm::driver::run_image_preparation_worker(&request) .await .map_err(|error| miette::miette!("{error}"))?; return Ok(()); } - if args.internal_run_vm { - // The VM launcher arms procguard after resolving its runtime so its - // libkrun worker cannot outlive the launcher. - maybe_reexec_internal_vm_with_runtime_env()?; - let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; - run_vm(&config).map_err(|err| miette::miette!("{err}"))?; - return Ok(()); - } let _tracing = openshell_otel::install_driver_tracing( openshell_driver_vm::otel_tracing::TRACING, @@ -748,9 +773,9 @@ fn parse_vm_mount_arg(arg: &str) -> std::result::Result { fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { use std::os::unix::process::CommandExt as _; - const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; - - if std::env::var_os(REEXEC_ENV).is_some() { + if std::env::var_os(VM_RUNTIME_REEXEC_ENV).is_some() + || std::env::var_os(VM_RUNTIME_DIR_ENV).is_some() + { return Ok(()); } @@ -783,7 +808,7 @@ fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { .args(&args) .env("DYLD_LIBRARY_PATH", &joined) .env(VM_RUNTIME_DIR_ENV, runtime_dir) - .env(REEXEC_ENV, "1") + .env(VM_RUNTIME_REEXEC_ENV, "1") .exec(); // `exec()` only returns on failure. Err(miette::miette!("failed to re-exec with runtime env: {err}")) @@ -805,6 +830,23 @@ mod tests { use clap::Parser; use std::path::PathBuf; + #[test] + fn internal_vm_worker_rejects_tokio_context() { + assert!(super::ensure_vm_worker_outside_tokio().is_ok()); + + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("Tokio runtime"); + runtime.block_on(async { + let error = super::ensure_vm_worker_outside_tokio().unwrap_err(); + assert!( + error + .to_string() + .contains("must start before the Tokio runtime") + ); + }); + } + #[test] fn corporate_proxy_flags_parse_into_driver_settings() { let args = Args::parse_from([ diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index ad300d98cf..06a3388c51 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -13,6 +13,7 @@ use std::time::Duration; use crate::{embedded_runtime, ffi, procguard, rootfs}; pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR"; +pub const VM_RUNTIME_REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1"; /// PID of the VM worker process (libkrun fork or QEMU). Zero when not running. @@ -333,7 +334,7 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { configure_runtime_loader_env(&runtime_dir)?; raise_nofile_limit(); - let vm = VmContext::create(&runtime_dir, config.log_level)?; + let vm = create_vm_context(&runtime_dir, config.log_level)?; vm.set_vm_config(config.vcpus, config.mem_mib)?; vm.set_disks( &config.root_disk, @@ -439,6 +440,14 @@ pub fn configured_runtime_dir() -> Result { embedded_runtime::ensure_runtime_extracted() } +#[cfg(target_os = "macos")] +pub(crate) fn external_runtime_selected() -> bool { + // The embedded-runtime re-exec sets VM_RUNTIME_DIR_ENV to keep the cache + // path stable. Its marker distinguishes that path from an operator override. + std::env::var_os(VM_RUNTIME_DIR_ENV).is_some() + && std::env::var_os(VM_RUNTIME_REEXEC_ENV).is_none() +} + fn qemu_runtime_dir() -> Result { configured_runtime_dir().map_err(|_| { "QEMU backend requires OPENSHELL_VM_RUNTIME_DIR to be set (pointing to a directory \ @@ -449,6 +458,10 @@ fn qemu_runtime_dir() -> Result { #[cfg(target_os = "macos")] fn configure_runtime_loader_env(runtime_dir: &Path) -> Result<(), String> { + if external_runtime_selected() { + return Ok(()); + } + let existing = std::env::var_os("DYLD_FALLBACK_LIBRARY_PATH"); let mut paths = vec![runtime_dir.to_path_buf()]; if let Some(existing) = existing { @@ -510,6 +523,21 @@ struct VmContext { ctx_id: u32, } +fn create_vm_context(runtime_dir: &Path, log_level: u32) -> Result { + #[cfg(target_os = "macos")] + if external_runtime_selected() { + let runtime_dir = std::fs::canonicalize(runtime_dir).map_err(|e| { + format!( + "resolve VM runtime directory {}: {e}", + runtime_dir.display() + ) + })?; + return VmContext::create(&runtime_dir, log_level); + } + + VmContext::create(runtime_dir, log_level) +} + impl VmContext { fn create(runtime_dir: &Path, log_level: u32) -> Result { let krun = ffi::libkrun(runtime_dir)?; diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 3d5680301d..8515c43bc3 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1586,6 +1586,8 @@ gateway_id = "openshell" state_dir = "/var/lib/openshell/vm" # Where the gateway looks for the openshell-driver-vm subprocess binary. driver_dir = "/usr/local/libexec/openshell" +# For an organization-signed macOS VM driver, set OPENSHELL_VM_RUNTIME_DIR in +# the gateway service environment. It is not a gateway.toml field. default_image = "nvcr.io/nvidia/base/ubuntu:24.04" # Optional override. When omitted, the gateway derives # https://host.openshell.internal: for the VM driver. diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 70f6891ce1..5001febf68 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -214,6 +214,61 @@ Podman also supports `image` mounts, which mount a container image read-only at The MicroVM driver runs each sandbox in its own lightweight VM. It requires host virtualization: Apple Hypervisor on macOS or KVM on Linux. +### Re-sign the macOS VM driver + +For a managed Mac, place the matching `libkrun.dylib` and +`libkrunfw.5.dylib` in an administrator-owned runtime directory. Use a +`vm-runtime-darwin-aarch64` archive built with this driver revision. The +macOS runtime build adds `krun_set_fw_path` to the pinned libkrun so the +driver can pass a canonical absolute firmware path. Earlier archives lack +that entry point and cannot boot a hardened driver through this override. + +Sign both libraries and `openshell-driver-vm` with the same Developer ID +Application identity and the hardened runtime. Apple's +[library validation rule](https://developer.apple.com/documentation/bundleresources/entitlements/com.apple.security.cs.disable-library-validation) +allows a hardened process to load libraries signed with its Team ID. The +repository's [VM driver entitlement file](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-vm/entitlements.plist) +provides `com.apple.security.hypervisor`. A VM boot with matching Developer +ID signatures has not yet been verified. + +If the libraries are ad-hoc signed or signed by a different Team ID, the +driver also needs `com.apple.security.cs.disable-library-validation`. The +ad-hoc hardened-runtime VM proof used both entitlements. It did not need +`com.apple.security.cs.allow-dyld-environment-variables`. + +This example stages the two libraries from a newly built matching archive. +Replace the identity, archive, and paths for your package: + +```shell +mkdir -p runtime +zstd -dc vm-runtime-darwin-aarch64.tar.zst | + tar -xf - -C runtime ./libkrun.dylib ./libkrunfw.5.dylib + +identity='Developer ID Application: Your Organization (TEAMID)' +codesign --force --options runtime --timestamp -s "$identity" runtime/libkrunfw.5.dylib +codesign --force --options runtime --timestamp -s "$identity" runtime/libkrun.dylib +codesign --force --options runtime --timestamp \ + --entitlements crates/openshell-driver-vm/entitlements.plist \ + -s "$identity" openshell-driver-vm + +codesign --verify --strict runtime/libkrunfw.5.dylib +codesign --verify --strict runtime/libkrun.dylib +codesign --verify --strict openshell-driver-vm +``` + +Set `OPENSHELL_VM_RUNTIME_DIR` to the absolute path of that directory in the +gateway's service environment, for example: + +```shell +OPENSHELL_VM_RUNTIME_DIR=/opt/example/openshell/runtime \ + openshell-gateway --config /etc/openshell/gateway.toml +``` + +The managed VM driver inherits the setting. It opens both dylibs by canonical +absolute path inside the selected directory and does not depend on `DYLD_*` +environment variables. Without the override, it uses its embedded runtime. +Keep the runtime directory writable only by administrators. + The VM driver is opt-in. Enable it in the gateway TOML file: ```toml diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 9e56f7117c..241c0dc54d 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1492,6 +1492,16 @@ Use the VM driver logs and host diagnostics available in the user's environment. upper layer, prepared rootfs, explicit config, or current image. Do not assign `10001:10001` unless the persisted state reports that legacy identity. - Host virtualization support is enabled. +- On macOS with `OPENSHELL_VM_RUNTIME_DIR` set in the gateway service + environment, the directory contains `libkrun.dylib` and + `libkrunfw.5.dylib` from the matching runtime build. Use + `codesign -dv --verbose=2` on both libraries and `openshell-driver-vm` to + compare their Team IDs. A hardened driver needs + `com.apple.security.cs.disable-library-validation` for ad-hoc libraries or + libraries signed by another Team ID. The runtime must export + `krun_set_fw_path`; older archives lack the absolute firmware path support. + See the + [macOS signing recipe](https://docs.nvidia.com/openshell/latest/how-it-works/sandboxes/runtimes#re-sign-the-macos-vm-driver). - The sandbox supervisor can establish its authenticated gateway session. For `vm.mounts` (virtiofs host bind mounts) failures, also verify: diff --git a/tasks/scripts/vm/build-libkrun-macos.sh b/tasks/scripts/vm/build-libkrun-macos.sh index 1afee1c937..be573f3afe 100755 --- a/tasks/scripts/vm/build-libkrun-macos.sh +++ b/tasks/scripts/vm/build-libkrun-macos.sh @@ -170,6 +170,16 @@ cd .. LIBKRUN_COMMIT=$(git -C libkrun rev-parse HEAD) echo " Commit: ${LIBKRUN_COMMIT}" +# The pinned libkrun opens firmware by a bare name. Hardened dyld rejects +# that lookup, so expose a setter for the driver's canonical absolute path. +PATCH="${ROOT}/tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch" +if git -C libkrun apply --unidiff-zero --check "$PATCH"; then + git -C libkrun apply --unidiff-zero "$PATCH" +elif ! git -C libkrun apply --unidiff-zero --reverse --check "$PATCH"; then + echo "Error: libkrun firmware path patch does not match ${LIBKRUN_COMMIT}" >&2 + exit 1 +fi + cd libkrun # ── Build libkrun ─────────────────────────────────────────────────────── diff --git a/tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch b/tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch new file mode 100644 index 0000000000..b2e0c3f9a0 --- /dev/null +++ b/tasks/scripts/vm/patches/libkrun-macos-absolute-firmware.patch @@ -0,0 +1,46 @@ +diff --git a/src/libkrun/src/lib.rs b/src/libkrun/src/lib.rs +index a797fd1..3c6f9eb 100644 +--- a/src/libkrun/src/lib.rs ++++ b/src/libkrun/src/lib.rs +@@ -29,0 +30,2 @@ use std::ffi::{c_void, CStr}; ++#[cfg(target_os = "macos")] ++use std::ffi::OsStr; +@@ -36,0 +39,2 @@ use std::os::fd::{BorrowedFd, FromRawFd, RawFd}; ++#[cfg(target_os = "macos")] ++use std::os::unix::ffi::OsStrExt; +@@ -40,0 +45,2 @@ use std::sync::LazyLock; ++#[cfg(target_os = "macos")] ++use std::sync::OnceLock; +@@ -116,2 +122,30 @@ fn init_virtual_entry() -> VirtualDirEntry { +-static KRUNFW: LazyLock> = +- LazyLock::new(|| unsafe { libloading::Library::new(KRUNFW_NAME).ok() }); ++// Set by the VM driver before the first context is created. A full path is ++// required because hardened dyld rejects the bare firmware filename. ++#[cfg(target_os = "macos")] ++static KRUNFW_PATH: OnceLock = OnceLock::new(); ++ ++#[cfg(target_os = "macos")] ++#[no_mangle] ++pub unsafe extern "C" fn krun_set_fw_path(path: *const c_char) -> i32 { ++ if path.is_null() { ++ return -libc::EINVAL; ++ } ++ ++ let path = PathBuf::from(OsStr::from_bytes(unsafe { CStr::from_ptr(path) }.to_bytes())); ++ if !path.is_absolute() { ++ return -libc::EINVAL; ++ } ++ ++ match KRUNFW_PATH.set(path) { ++ Ok(()) => KRUN_SUCCESS, ++ Err(_) => -libc::EALREADY, ++ } ++} ++ ++static KRUNFW: LazyLock> = LazyLock::new(|| { ++ #[cfg(target_os = "macos")] ++ if let Some(path) = KRUNFW_PATH.get() { ++ return unsafe { libloading::Library::new(path).ok() }; ++ } ++ unsafe { libloading::Library::new(KRUNFW_NAME).ok() } ++});