feat(vm): allow an externally signed VM runtime on macOS

crates/openshell-driver-vm/README.md

@@ -337,6 +337,17 @@ driver with the Hypervisor entitlement in `post_install` and owns the `brew
 services` gateway lifecycle. The service also leaves `OPENSHELL_DRIVERS` unset
 so driver choice remains automatic unless the user explicitly overrides it.
 
+For organization-signed macOS packages, `OPENSHELL_VM_RUNTIME_DIR` selects a
+separately signed runtime instead of extracting the embedded libraries. The
+driver resolves both dylibs to canonical absolute paths inside that directory
+and passes the firmware path to libkrun before creating a VM context. This
+avoids `DYLD_*` loader variables and process-wide working-directory changes.
+The runtime must be built with the `krun_set_fw_path` entry point added by
+`build-libkrun-macos.sh`; older runtime archives lack it and are rejected.
+The directory must contain `libkrun.dylib` and `libkrunfw.5.dylib` from
+the matching VM runtime build. Keep it writable only by administrators. See
+the [macOS re-signing recipe](../../docs/how-it-works/sandboxes/runtimes.mdx#re-sign-the-macos-vm-driver).
+
 ## TODOs
 
 - The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` note in `crates/openshell-gateway/src/vm.rs`.

crates/openshell-driver-vm/src/ffi.rs

@@ -12,7 +12,11 @@ use std::sync::OnceLock;
 use libc::c_char;
 use libloading::Library;
 
+#[cfg(target_os = "macos")]
+use crate::runtime::external_runtime_selected;
 use crate::runtime::validate_runtime_dir;
+#[cfg(target_os = "macos")]
+use std::os::unix::ffi::OsStrExt as _;
 
 pub const KRUN_LOG_TARGET_DEFAULT: i32 = -1;
 pub const KRUN_LOG_LEVEL_OFF: u32 = 0;
@@ -27,6 +31,8 @@ pub const KRUN_LOG_OPTION_NO_ENV: u32 = 1;
 type KrunInitLog =
     unsafe extern "C" fn(target_fd: i32, level: u32, style: u32, options: u32) -> i32;
 type KrunCreateCtx = unsafe extern "C" fn() -> i32;
+#[cfg(target_os = "macos")]
+type KrunSetFwPath = unsafe extern "C" fn(*const c_char) -> i32;
 type KrunFreeCtx = unsafe extern "C" fn(ctx_id: u32) -> i32;
 type KrunSetVmConfig = unsafe extern "C" fn(ctx_id: u32, num_vcpus: u8, ram_mib: u32) -> i32;
 type KrunAddDisk = unsafe extern "C" fn(
@@ -112,13 +118,39 @@ pub fn required_runtime_lib_name() -> &'static str {
 impl LibKrun {
     fn load(runtime_dir: &Path) -> Result<Self, String> {
         let libkrun_path = runtime_dir.join(required_runtime_lib_name());
-        preload_runtime_support_libraries(runtime_dir)?;
+        #[cfg(target_os = "macos")]
+        let (libkrun_path, firmware_path) = if external_runtime_selected() {
+            (
+                canonical_runtime_library(runtime_dir, required_runtime_lib_name())?,
+                Some(canonical_runtime_library(runtime_dir, "libkrunfw.5.dylib")?),
+            )
+        } else {
+            (libkrun_path, None)
+        };
+        #[cfg(not(target_os = "macos"))]
+        let firmware_path: Option<PathBuf> = None;
+
+        preload_runtime_support_libraries(runtime_dir, firmware_path.as_deref())?;
 
         let library = Box::leak(Box::new(unsafe {
             Library::new(&libkrun_path)
                 .map_err(|e| format!("load libkrun from {}: {e}", libkrun_path.display()))?
         }));
 
+        #[cfg(target_os = "macos")]
+        if let Some(firmware_path) = &firmware_path {
+            let set_fw_path: KrunSetFwPath =
+                load_symbol(library, b"krun_set_fw_path\0", &libkrun_path).map_err(|e| {
+                    format!("{e}; rebuild the macOS VM runtime with absolute firmware path support")
+                })?;
+            let path = CString::new(firmware_path.as_os_str().as_bytes())
+                .map_err(|e| format!("invalid firmware path {}: {e}", firmware_path.display()))?;
+            let result = unsafe { set_fw_path(path.as_ptr()) };
+            if result < 0 {
+                return Err(format!("krun_set_fw_path failed with error code {result}"));
+            }
+        }
+
         Ok(Self {
             krun_init_log: load_symbol(library, b"krun_init_log\0", &libkrun_path)?,
             krun_create_ctx: load_symbol(library, b"krun_create_ctx\0", &libkrun_path)?,
@@ -150,31 +182,52 @@ impl LibKrun {
     }
 }
 
-fn preload_runtime_support_libraries(runtime_dir: &Path) -> Result<Vec<PathBuf>, String> {
-    let entries = std::fs::read_dir(runtime_dir)
-        .map_err(|e| format!("read {}: {e}", runtime_dir.display()))?;
-
-    let mut support_libs: Vec<PathBuf> = entries
-        .filter_map(Result::ok)
-        .map(|entry| entry.path())
-        .filter(|path| {
-            path.file_name()
-                .and_then(|name| name.to_str())
-                .is_some_and(|name| {
-                    #[cfg(target_os = "macos")]
-                    {
-                        name.starts_with("libkrunfw")
-                            && Path::new(name)
-                                .extension()
-                                .is_some_and(|ext| ext.eq_ignore_ascii_case("dylib"))
-                    }
-                    #[cfg(not(target_os = "macos"))]
-                    {
-                        name.starts_with("libkrunfw") && name.contains(".so")
-                    }
-                })
-        })
-        .collect();
+#[cfg(target_os = "macos")]
+fn canonical_runtime_library(runtime_dir: &Path, name: &str) -> Result<PathBuf, String> {
+    let root = std::fs::canonicalize(runtime_dir)
+        .map_err(|e| format!("resolve runtime directory {}: {e}", runtime_dir.display()))?;
+    let path = std::fs::canonicalize(root.join(name))
+        .map_err(|e| format!("resolve runtime library {}: {e}", root.join(name).display()))?;
+    if !path.starts_with(&root) || !path.is_file() {
+        return Err(format!(
+            "runtime library must be a file inside {}",
+            root.display()
+        ));
+    }
+    Ok(path)
+}
+
+fn preload_runtime_support_libraries(
+    runtime_dir: &Path,
+    explicit_firmware: Option<&Path>,
+) -> Result<Vec<PathBuf>, String> {
+    let mut support_libs: Vec<PathBuf> = if let Some(path) = explicit_firmware {
+        vec![path.to_path_buf()]
+    } else {
+        let entries = std::fs::read_dir(runtime_dir)
+            .map_err(|e| format!("read {}: {e}", runtime_dir.display()))?;
+        entries
+            .filter_map(Result::ok)
+            .map(|entry| entry.path())
+            .filter(|path| {
+                path.file_name()
+                    .and_then(|name| name.to_str())
+                    .is_some_and(|name| {
+                        #[cfg(target_os = "macos")]
+                        {
+                            name.starts_with("libkrunfw")
+                                && Path::new(name)
+                                    .extension()
+                                    .is_some_and(|ext| ext.eq_ignore_ascii_case("dylib"))
+                        }
+                        #[cfg(not(target_os = "macos"))]
+                        {
+                            name.starts_with("libkrunfw") && name.contains(".so")
+                        }
+                    })
+            })
+            .collect()
+    };
 
     support_libs.sort();
     for path in &support_libs {

crates/openshell-driver-vm/src/lib.rs

@@ -47,6 +47,6 @@ pub use lifecycle::{
 };
 #[cfg(feature = "compute-driver")]
 pub use runtime::{
-    VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, VmMount, VsockPortMap, configured_runtime_dir,
-    run_vm,
+    VM_RUNTIME_DIR_ENV, VM_RUNTIME_REEXEC_ENV, VmBackend, VmLaunchConfig, VmMount, VsockPortMap,
+    configured_runtime_dir, run_vm,
 };

crates/openshell-driver-vm/src/main.rs

@@ -7,7 +7,7 @@ use miette::{IntoDiagnostic, Result};
 use openshell_core::VERSION;
 use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
 #[cfg(target_os = "macos")]
-use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir};
+use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, VM_RUNTIME_REEXEC_ENV, configured_runtime_dir};
 use openshell_driver_vm::{
     VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, VmMount, VsockPortMap, procguard, run_vm,
 };
@@ -287,23 +287,48 @@ struct Args {
     vm_mount: Vec<String>,
 }
 
-#[tokio::main]
-async fn main() -> Result<()> {
+fn main() -> Result<()> {
     let args = Args::parse();
+    if args.internal_run_vm {
+        return run_internal_vm(&args);
+    }
+
+    // The re-exec marker belongs only to an embedded-runtime VM launcher.
+    // A gateway service must not pass it through the outer driver to a new
+    // launcher using an operator-supplied runtime.
+    #[cfg(target_os = "macos")]
+    unsafe {
+        std::env::remove_var(VM_RUNTIME_REEXEC_ENV);
+    }
+
+    run_async_driver(args)
+}
+
+fn run_internal_vm(args: &Args) -> Result<()> {
+    ensure_vm_worker_outside_tokio()?;
+    // libkrun forks its worker before any Tokio threads exist.
+    maybe_reexec_internal_vm_with_runtime_env()?;
+    let config = build_vm_launch_config(args).map_err(|err| miette::miette!("{err}"))?;
+    run_vm(&config).map_err(|err| miette::miette!("{err}"))
+}
+
+fn ensure_vm_worker_outside_tokio() -> Result<()> {
+    if tokio::runtime::Handle::try_current().is_ok() {
+        return Err(miette::miette!(
+            "internal VM worker must start before the Tokio runtime"
+        ));
+    }
+    Ok(())
+}
+
+#[tokio::main]
+async fn run_async_driver(args: Args) -> Result<()> {
     if let Some(request) = args.internal_prepare_image {
         openshell_driver_vm::driver::run_image_preparation_worker(&request)
             .await
             .map_err(|error| miette::miette!("{error}"))?;
         return Ok(());
     }
-    if args.internal_run_vm {
-        // The VM launcher arms procguard after resolving its runtime so its
-        // libkrun worker cannot outlive the launcher.
-        maybe_reexec_internal_vm_with_runtime_env()?;
-        let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?;
-        run_vm(&config).map_err(|err| miette::miette!("{err}"))?;
-        return Ok(());
-    }
 
     let _tracing = openshell_otel::install_driver_tracing(
         openshell_driver_vm::otel_tracing::TRACING,
@@ -748,9 +773,9 @@ fn parse_vm_mount_arg(arg: &str) -> std::result::Result<VmMount, String> {
 fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> {
     use std::os::unix::process::CommandExt as _;
 
-    const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC";
-
-    if std::env::var_os(REEXEC_ENV).is_some() {
+    if std::env::var_os(VM_RUNTIME_REEXEC_ENV).is_some()
+        || std::env::var_os(VM_RUNTIME_DIR_ENV).is_some()
+    {
         return Ok(());
     }
 
@@ -783,7 +808,7 @@ fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> {
         .args(&args)
         .env("DYLD_LIBRARY_PATH", &joined)
         .env(VM_RUNTIME_DIR_ENV, runtime_dir)
-        .env(REEXEC_ENV, "1")
+        .env(VM_RUNTIME_REEXEC_ENV, "1")
         .exec();
     // `exec()` only returns on failure.
     Err(miette::miette!("failed to re-exec with runtime env: {err}"))
@@ -805,6 +830,23 @@ mod tests {
     use clap::Parser;
     use std::path::PathBuf;
 
+    #[test]
+    fn internal_vm_worker_rejects_tokio_context() {
+        assert!(super::ensure_vm_worker_outside_tokio().is_ok());
+
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .build()
+            .expect("Tokio runtime");
+        runtime.block_on(async {
+            let error = super::ensure_vm_worker_outside_tokio().unwrap_err();
+            assert!(
+                error
+                    .to_string()
+                    .contains("must start before the Tokio runtime")
+            );
+        });
+    }
+
     #[test]
     fn corporate_proxy_flags_parse_into_driver_settings() {
         let args = Args::parse_from([

crates/openshell-driver-vm/src/runtime.rs

@@ -13,6 +13,7 @@ use std::time::Duration;
 use crate::{embedded_runtime, ffi, procguard, rootfs};
 
 pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR";
+pub const VM_RUNTIME_REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC";
 const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1";
 
 /// PID of the VM worker process (libkrun fork or QEMU). Zero when not running.
@@ -333,7 +334,7 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
     configure_runtime_loader_env(&runtime_dir)?;
     raise_nofile_limit();
 
-    let vm = VmContext::create(&runtime_dir, config.log_level)?;
+    let vm = create_vm_context(&runtime_dir, config.log_level)?;
     vm.set_vm_config(config.vcpus, config.mem_mib)?;
     vm.set_disks(
         &config.root_disk,
@@ -439,6 +440,14 @@ pub fn configured_runtime_dir() -> Result<PathBuf, String> {
     embedded_runtime::ensure_runtime_extracted()
 }
 
+#[cfg(target_os = "macos")]
+pub(crate) fn external_runtime_selected() -> bool {
+    // The embedded-runtime re-exec sets VM_RUNTIME_DIR_ENV to keep the cache
+    // path stable. Its marker distinguishes that path from an operator override.
+    std::env::var_os(VM_RUNTIME_DIR_ENV).is_some()
+        && std::env::var_os(VM_RUNTIME_REEXEC_ENV).is_none()
+}
+
 fn qemu_runtime_dir() -> Result<PathBuf, String> {
     configured_runtime_dir().map_err(|_| {
         "QEMU backend requires OPENSHELL_VM_RUNTIME_DIR to be set (pointing to a directory \
@@ -449,6 +458,10 @@ fn qemu_runtime_dir() -> Result<PathBuf, String> {
 
 #[cfg(target_os = "macos")]
 fn configure_runtime_loader_env(runtime_dir: &Path) -> Result<(), String> {
+    if external_runtime_selected() {
+        return Ok(());
+    }
+
     let existing = std::env::var_os("DYLD_FALLBACK_LIBRARY_PATH");
     let mut paths = vec![runtime_dir.to_path_buf()];
     if let Some(existing) = existing {
@@ -510,6 +523,21 @@ struct VmContext {
     ctx_id: u32,
 }
 
+fn create_vm_context(runtime_dir: &Path, log_level: u32) -> Result<VmContext, String> {
+    #[cfg(target_os = "macos")]
+    if external_runtime_selected() {
+        let runtime_dir = std::fs::canonicalize(runtime_dir).map_err(|e| {
+            format!(
+                "resolve VM runtime directory {}: {e}",
+                runtime_dir.display()
+            )
+        })?;
+        return VmContext::create(&runtime_dir, log_level);
+    }
+
+    VmContext::create(runtime_dir, log_level)
+}
+
 impl VmContext {
     fn create(runtime_dir: &Path, log_level: u32) -> Result<Self, String> {
         let krun = ffi::libkrun(runtime_dir)?;

docs/how-it-works/gateways/configuration.mdx

@@ -1586,6 +1586,8 @@ gateway_id       = "openshell"
 state_dir        = "/var/lib/openshell/vm"
 # Where the gateway looks for the openshell-driver-vm subprocess binary.
 driver_dir       = "/usr/local/libexec/openshell"
+# For an organization-signed macOS VM driver, set OPENSHELL_VM_RUNTIME_DIR in
+# the gateway service environment. It is not a gateway.toml field.
 default_image    = "nvcr.io/nvidia/base/ubuntu:24.04"
 # Optional override. When omitted, the gateway derives
 # https://host.openshell.internal:<gateway-port> for the VM driver.