From 2dae91642a6da7861c88b86c7c6c9b0efb90c144 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 14 Oct 2026 21:25:49 +0200 Subject: [PATCH 1/6] feat(core): add the CDI context to the sandbox boundary protocol Define the shared CDI context schema: the selected CDI device IDs and the workload-local paths at which a compute driver projects CDI spec directories. Carry it as an optional field in the sandbox boundary configuration, omitted from serialized output when absent so non-GPU bootstraps stay readable by older sandbox runtimes. No driver sets it yet. Signed-off-by: Evan Lezar --- crates/openshell-core/src/cdi.rs | 53 +++++++++++++++++++ crates/openshell-core/src/lib.rs | 1 + .../openshell-driver-docker/src/isolation.rs | 1 + .../src/isolation.rs | 1 + .../openshell-driver-podman/src/isolation.rs | 1 + .../openshell-driver-vm/src/isolation/mod.rs | 1 + .../src/boundary_protocol.rs | 9 ++++ .../openshell-sandbox/src/boundary_server.rs | 9 ++++ 8 files changed, 76 insertions(+) create mode 100644 crates/openshell-core/src/cdi.rs diff --git a/crates/openshell-core/src/cdi.rs b/crates/openshell-core/src/cdi.rs new file mode 100644 index 0000000000..12a45a2062 --- /dev/null +++ b/crates/openshell-core/src/cdi.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared CDI context schema. + +use serde::{Deserialize, Serialize}; + +pub const CDI_CONTEXT_VERSION: u32 = 1; + +/// Base workload-runtime path under which compute drivers mount CDI specification directories. +pub const CDI_SPEC_DIR_BASE: &str = "/.openshell/cdi-specs"; + +/// Return the workload-runtime path used for a CDI specification directory. +#[must_use] +pub fn cdi_spec_mount_path(index: usize) -> String { + format!("{CDI_SPEC_DIR_BASE}/{index}") +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CdiContext { + pub version: u32, + pub selected_devices: Vec, + pub spec_dirs: Vec, +} + +impl CdiContext { + #[must_use] + pub fn new(selected_devices: Vec, spec_dirs: Vec) -> Self { + Self { + version: CDI_CONTEXT_VERSION, + selected_devices, + spec_dirs, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CdiSpecDirectory { + pub path: String, + pub source: String, +} + +impl CdiSpecDirectory { + #[must_use] + pub fn new(path: impl Into, source: impl Into) -> Self { + Self { + path: path.into(), + source: source.into(), + } + } +} diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index e4e41f5531..eef462d50f 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod activity; pub mod auth; +pub mod cdi; #[cfg(target_os = "linux")] pub mod cdi_name { pub use container_device_interface::parser::is_qualified_name; diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs index d921b8c348..b5d4544e00 100644 --- a/crates/openshell-driver-docker/src/isolation.rs +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -118,6 +118,7 @@ impl DockerBoundarySpec { }, resource_claims: resource_claims.clone(), resource_claim_files: BTreeMap::new(), + cdi_context: None, workload_identity: self.workload_identity.clone(), outer_fence: outer_fence.clone(), child_env: self.child_env, diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs index 6a94e6abaa..94ff87d21d 100644 --- a/crates/openshell-driver-kubernetes/src/isolation.rs +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -274,6 +274,7 @@ impl KubernetesSandboxRuntimeBoundarySpec { "kubernetes.workload_pod_uid".to_string(), self.workload_pod_uid_path, )]), + cdi_context: None, workload_identity: self.workload_identity.clone(), outer_fence: outer_fence.clone(), child_env: self.child_env, diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index cffbeb3599..c866e1eb8b 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -277,6 +277,7 @@ pub fn bootstrap_archives( }, resource_claims: resource_claims.clone(), resource_claim_files: BTreeMap::new(), + cdi_context: None, workload_identity: identity.clone(), outer_fence: outer_fence.clone(), child_env: child_env.clone(), diff --git a/crates/openshell-driver-vm/src/isolation/mod.rs b/crates/openshell-driver-vm/src/isolation/mod.rs index a660ad98c7..2316df5494 100644 --- a/crates/openshell-driver-vm/src/isolation/mod.rs +++ b/crates/openshell-driver-vm/src/isolation/mod.rs @@ -118,6 +118,7 @@ impl VmBoundarySpec { }, resource_claims: resource_claims.clone(), resource_claim_files: BTreeMap::new(), + cdi_context: None, workload_identity: workload_identity.clone(), outer_fence: outer_fence.clone(), child_env: self.child_env, diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index 70d491d318..9baee6e0f0 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -510,6 +510,14 @@ pub struct BoundaryConfig { /// Downward API. Other drivers may leave the map empty. #[serde(default)] pub resource_claim_files: std::collections::BTreeMap, + /// Driver-protected CDI selection and workload-local specification projections. + /// + /// The sandbox runtime resolves this context inside the workload mount + /// namespace before applying launch-time filesystem controls. Omitted + /// when absent so non-GPU bootstraps stay readable by runtimes that + /// predate this field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cdi_context: Option, /// Exact identity already applied by the runtime to the sandbox process. pub workload_identity: openshell_isolation_interface::contract::ResolvedWorkloadIdentity, /// Backend-neutral projection of the validated outer network fence. @@ -540,6 +548,7 @@ impl fmt::Debug for BoundaryConfig { .field("listener", &self.listener) .field("resource_claims", &self.resource_claims) .field("resource_claim_files", &self.resource_claim_files) + .field("has_cdi_context", &self.cdi_context.is_some()) .field("workload_identity", &self.workload_identity) .field("outer_fence", &self.outer_fence) .field("child_env_keys", &self.child_env.keys().collect::>()) diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 16299730c9..5f5a6aa2a6 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -3995,6 +3995,7 @@ mod linux { }, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -4109,6 +4110,7 @@ mod linux { "true".to_string(), )]), resource_claim_files: std::collections::BTreeMap::new(), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -4180,6 +4182,7 @@ mod linux { }, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -4774,6 +4777,7 @@ mod linux { }, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -4996,6 +5000,7 @@ mod linux { }, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -5043,6 +5048,7 @@ mod linux { "kubernetes.pod_uid".to_string(), pod_uid_path, )]), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -5082,6 +5088,7 @@ mod linux { }, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -5385,6 +5392,7 @@ mod linux { }, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -5720,6 +5728,7 @@ mod linux { }, resource_claims: std::collections::BTreeMap::new(), resource_claim_files: std::collections::BTreeMap::new(), + cdi_context: None, workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), From d154dd7b99f4a1d0cf9b7de4120708511736d952 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 14 Oct 2026 21:34:53 +0200 Subject: [PATCH 2/6] feat(sandbox): resolve CDI requirements from selected device specs Resolve the selected CDI devices against the spec directories projected into the workload and derive the filesystem requirements their edits imply: - device nodes, which need read-write access; - read-only mounts, expanded to the parent directory for shared libraries; - Unix sockets, kept as exact targets without filesystem grants; - the NVIDIA MPS pipe directory, accepted as a writable IPC directory only when mounted nosuid, nodev and noexec; - additional GIDs, excluding root. Effective mount access comes from the workload's mount table rather than spec options. Ordinary writable mounts, broad roots and conflicting uses of one path are rejected. Captured Orin, Spark and multi-GPU specs drive the production resolver against reviewed baselines. The resolver is wired into launch in the next commit. Signed-off-by: Evan Lezar --- Cargo.lock | 2 + crates/openshell-sandbox/Cargo.toml | 2 + crates/openshell-sandbox/src/cdi.rs | 114 + crates/openshell-sandbox/src/cdi/linux.rs | 1392 +++++++++++ crates/openshell-sandbox/src/launch.rs | 30 + crates/openshell-sandbox/src/lib.rs | 3 + .../cdi/multi-gpu-nvidia-policy-baseline.json | 42 + .../testdata/cdi/multi-gpu-nvidia.yaml | 817 ++++++ .../cdi/orin-nvidia-policy-baseline.json | 99 + .../testdata/cdi/orin-nvidia.yaml | 2218 +++++++++++++++++ .../cdi/spark-nvidia-policy-baseline.json | 39 + .../testdata/cdi/spark-nvidia.yaml | 600 +++++ 12 files changed, 5358 insertions(+) create mode 100644 crates/openshell-sandbox/src/cdi.rs create mode 100644 crates/openshell-sandbox/src/cdi/linux.rs create mode 100644 crates/openshell-sandbox/src/launch.rs create mode 100644 crates/openshell-sandbox/testdata/cdi/multi-gpu-nvidia-policy-baseline.json create mode 100644 crates/openshell-sandbox/testdata/cdi/multi-gpu-nvidia.yaml create mode 100644 crates/openshell-sandbox/testdata/cdi/orin-nvidia-policy-baseline.json create mode 100644 crates/openshell-sandbox/testdata/cdi/orin-nvidia.yaml create mode 100644 crates/openshell-sandbox/testdata/cdi/spark-nvidia-policy-baseline.json create mode 100644 crates/openshell-sandbox/testdata/cdi/spark-nvidia.yaml diff --git a/Cargo.lock b/Cargo.lock index 670089d9e5..1c9853b31a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4837,6 +4837,7 @@ dependencies = [ "base64 0.22.1", "bytes", "clap", + "container-device-interface", "hex", "ipnet", "landlock", @@ -4861,6 +4862,7 @@ dependencies = [ "sha2 0.10.9", "socket2", "tempfile", + "thiserror 2.0.20", "tokio", "tokio-rustls", "tokio-stream", diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 88bce43fab..d6c6ca7fab 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -48,6 +48,7 @@ clap = { workspace = true } # Error handling miette = { workspace = true } +thiserror = { workspace = true } # Unix identity and bootstrap ownership nix = { workspace = true, features = ["resource", "socket", "feature", "poll"] } @@ -73,6 +74,7 @@ russh-sftp = "3.0" uuid = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] +container-device-interface = { workspace = true } landlock = "0.4" seccompiler = "0.5" socket2 = { workspace = true, features = ["all"] } diff --git a/crates/openshell-sandbox/src/cdi.rs b/crates/openshell-sandbox/src/cdi.rs new file mode 100644 index 0000000000..ca8b37e68a --- /dev/null +++ b/crates/openshell-sandbox/src/cdi.rs @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workload-local CDI resolution and resource validation. + +use std::path::Path; + +use openshell_core::cdi::{CDI_CONTEXT_VERSION, CdiContext, CdiSpecDirectory}; + +use crate::launch::{CdiDerivedRequirements, CdiMountRequirement}; + +/// Roots that CDI destinations and expanded library parents must not grant. +pub const FORBIDDEN_CDI_ROOTS: &[&str] = &["/", "/dev", "/proc", "/sys", "/run", "/usr"]; + +/// Writable IPC directories that CDI may mount for GPU daemons. Clients must +/// open the named pipes inside for writing, so these are the only writable +/// directory mounts granted. The NVIDIA MPS pipe directory is the only entry. +pub const CDI_IPC_DIRECTORIES: &[&str] = &["/tmp/nvidia-mps"]; + +/// Mount options a CDI IPC directory must carry, in the spec and in effect. +pub const CDI_IPC_DIRECTORY_MOUNT_OPTIONS: &[&str] = &["nosuid", "nodev", "noexec"]; + +#[derive(Debug, thiserror::Error)] +pub enum CdiError { + #[error("failed to inspect effective CDI mount '{path}': {error}")] + MountInspection { path: String, error: String }, + #[error("unsupported CDI context version {0}")] + UnsupportedContextVersion(u32), + #[error("CDI spec dir '{path}' from source '{diagnostic_source}' is unsafe: {reason}")] + UnsafeSpecDir { + path: String, + diagnostic_source: String, + reason: &'static str, + }, + #[error("selected CDI device '{0}' was not found in mounted CDI specs")] + MissingDevice(String), + #[error( + "selected CDI device '{device}' was not found in mounted CDI specs after CDI spec refresh reported: {refresh_error}" + )] + MissingDeviceAfterRefresh { + device: String, + refresh_error: String, + }, + #[error("failed to merge CDI edits for '{device}': {error}")] + EditMerge { device: String, error: String }, + #[error("failed to encode resolved CDI edits: {source}")] + EditEncode { source: serde_json::Error }, + #[error("failed to decode resolved CDI edits: {source}")] + EditDecode { source: serde_json::Error }, + #[error("CDI-derived path '{path}' is unsafe: {reason}")] + UnsafePolicyPath { path: String, reason: &'static str }, + #[error("CDI path '{path}' requested conflicting access modes")] + ConflictingAccess { path: String }, + #[error( + "CDI writable mount '{path}' is unsupported: ordinary host-backed file and directory mounts must be read-only; filesystem read_write grants do not authorize CDI host writes" + )] + WritableMountNotAllowed { path: String }, + #[error("CDI additionalGids must not contain root GID 0")] + RootAdditionalGid, + #[error("CDI mount '{path}' has conflicting ro/rw options")] + ConflictingMountOptions { path: String }, + #[error("CDI IPC directory '{path}' must be mounted with nosuid, nodev and noexec")] + UnsafeIpcDirectory { path: String }, +} + +#[cfg(any(target_os = "linux", test))] +fn shared_library_parent(path: &str) -> Option { + let path = Path::new(path); + let name = path.file_name()?.to_str()?; + let versioned_library = name.rsplit_once(".so.").is_some_and(|(_, version)| { + version.split('.').all(|component| { + !component.is_empty() && component.bytes().all(|byte| byte.is_ascii_digit()) + }) + }); + let unversioned_library = name + .rsplit_once('.') + .is_some_and(|(_, extension)| extension == "so"); + if !unversioned_library && !versioned_library { + return None; + } + path.parent()?.to_str().map(ToOwned::to_owned) +} + +#[cfg(test)] +mod tests { + use super::shared_library_parent; + + #[test] + fn identifies_shared_library_parent_directories() { + for path in [ + "/opt/nvidia/lib/libcuda.so", + "/opt/nvidia/lib/libnvidia-ml.so.1", + "/opt/nvidia/lib/libnvidia-eglcore.so.540.5.0", + ] { + assert_eq!( + shared_library_parent(path).as_deref(), + Some("/opt/nvidia/lib") + ); + } + for path in [ + "/etc/ld.so.conf", + "/opt/nvidia/lib/libcuda.so.debug", + "/opt/nvidia/lib/libcuda.so.1beta", + "/usr/bin/nvidia-smi", + ] { + assert_eq!(shared_library_parent(path), None); + } + } +} + +#[cfg(target_os = "linux")] +mod linux; +#[cfg(target_os = "linux")] +pub use linux::{CdiMountTable, resolve_cdi_context, validate_cdi_requirements}; diff --git a/crates/openshell-sandbox/src/cdi/linux.rs b/crates/openshell-sandbox/src/cdi/linux.rs new file mode 100644 index 0000000000..f5c1ec611a --- /dev/null +++ b/crates/openshell-sandbox/src/cdi/linux.rs @@ -0,0 +1,1392 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Linux implementation of CDI policy resolution. + +use super::{ + CDI_CONTEXT_VERSION, CdiContext, CdiDerivedRequirements, CdiError, CdiMountRequirement, + CdiSpecDirectory, shared_library_parent, +}; +use container_device_interface::{ + cache::{Cache, with_auto_refresh}, + container_edits::ContainerEdits as UpstreamContainerEdits, + spec_dirs::with_spec_dirs, +}; +use openshell_core::paths::normalize_path; +use serde::Deserialize; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::fmt; +use std::path::{Component, Path}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CdiPathKind { + File, + Directory, + CharacterDevice, + BlockDevice, + Socket, + Other, +} + +// Temporary view over upstream-resolved CDI edits. The Rust CDI crate currently +// keeps some spec-model fields crate-private even though they are serialized and +// public in specs-go, so OpenShell serializes the merged upstream model and +// decodes only the policy-relevant fields here. +#[derive(Debug, Default, Deserialize)] +struct CdiContainerEdits { + #[serde(default, rename = "deviceNodes")] + device_nodes: Vec, + #[serde(default)] + mounts: Vec, + #[serde(default, rename = "additionalGids")] + additional_gids: Vec, +} + +#[derive(Debug, Deserialize)] +struct CdiDeviceNode { + path: String, +} + +#[derive(Debug, Deserialize)] +struct CdiMount { + #[serde(rename = "containerPath")] + container_path: String, + #[serde(default)] + options: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CdiAccess { + ReadOnly, + ReadWrite, +} + +#[derive(Debug, Default)] +struct RequirementAccumulator { + device_node_paths: BTreeSet, + mount_paths: BTreeMap, + socket_paths: BTreeSet, + ipc_directory_paths: BTreeSet, + additional_gids: BTreeSet, +} + +impl RequirementAccumulator { + fn add_device_node(&mut self, path: String) -> Result<(), CdiError> { + if self.socket_paths.contains(&path) || self.ipc_directory_paths.contains(&path) { + return Err(CdiError::ConflictingAccess { path }); + } + match self.mount_paths.get(&path).copied() { + Some(CdiAccess::ReadOnly) => Err(CdiError::ConflictingAccess { path }), + Some(CdiAccess::ReadWrite) | None => { + self.device_node_paths.insert(path); + Ok(()) + } + } + } + + fn add_mount(&mut self, path: String, access: CdiAccess) -> Result<(), CdiError> { + if self.socket_paths.contains(&path) + || self.ipc_directory_paths.contains(&path) + || (access == CdiAccess::ReadOnly && self.device_node_paths.contains(&path)) + { + return Err(CdiError::ConflictingAccess { path }); + } + match self.mount_paths.get(&path).copied() { + Some(existing) if existing != access => Err(CdiError::ConflictingAccess { path }), + Some(_) => Ok(()), + None => { + self.mount_paths.insert(path, access); + Ok(()) + } + } + } + + fn add_socket(&mut self, path: String) -> Result<(), CdiError> { + if self.device_node_paths.contains(&path) + || self.mount_paths.contains_key(&path) + || self.ipc_directory_paths.contains(&path) + { + return Err(CdiError::ConflictingAccess { path }); + } + self.socket_paths.insert(path); + Ok(()) + } + + fn add_ipc_directory(&mut self, path: String) -> Result<(), CdiError> { + if self.device_node_paths.contains(&path) + || self.mount_paths.contains_key(&path) + || self.socket_paths.contains(&path) + { + return Err(CdiError::ConflictingAccess { path }); + } + self.ipc_directory_paths.insert(path); + Ok(()) + } + + fn add_gid(&mut self, gid: u32) -> Result<(), CdiError> { + if gid == 0 { + return Err(CdiError::RootAdditionalGid); + } + self.additional_gids.insert(gid); + Ok(()) + } + + fn build(self) -> Result { + let mut read_only_paths = BTreeSet::new(); + let mut read_write_mount_paths = Vec::new(); + let mut mounts = Vec::new(); + for (path, access) in self.mount_paths { + mounts.push(CdiMountRequirement { + path: path.clone(), + read_only: access == CdiAccess::ReadOnly, + }); + match access { + CdiAccess::ReadOnly => { + let expanded = shared_library_parent(&path).unwrap_or(path); + read_only_paths.insert(normalize_policy_path(&expanded)?); + } + CdiAccess::ReadWrite => read_write_mount_paths.push(path), + } + } + Ok(CdiDerivedRequirements { + device_node_paths: self.device_node_paths.into_iter().collect(), + socket_paths: self.socket_paths.into_iter().collect(), + read_only_paths: read_only_paths.into_iter().collect(), + read_write_mount_paths, + ipc_directory_paths: self.ipc_directory_paths.into_iter().collect(), + additional_gids: self.additional_gids.into_iter().collect(), + mounts, + }) + } +} + +/// Facts observed in the workload namespace, independent of spec mount options. +#[derive(Debug, Clone, Copy)] +struct CdiMountState { + kind: CdiPathKind, + access: CdiAccess, +} + +pub fn resolve_cdi_context(context: &CdiContext) -> Result { + let mut table = None; + resolve_cdi_context_with_mount_inspector(context, |path| inspect_cdi_mount(path, &mut table)) +} + +/// Use the production resolver with injectable workload mount observations. +/// Only the OS inspection is replaced in hardware-fixture tests. +fn resolve_cdi_context_with_mount_inspector( + context: &CdiContext, + mut inspect_mount: impl FnMut(&str) -> Result, +) -> Result { + validate_context(context)?; + let selected_devices = selected_cdi_devices(&context.selected_devices); + if selected_devices.is_empty() || context.spec_dirs.is_empty() { + return Ok(CdiDerivedRequirements::default()); + } + let edits = resolve_container_edits(context, &selected_devices)?; + let mut accumulator = RequirementAccumulator::default(); + for node in &edits.device_nodes { + accumulator.add_device_node(normalize_policy_path(&node.path)?)?; + } + for mount in &edits.mounts { + let path = normalize_policy_path(&mount.container_path)?; + validate_mount_options(&path, &mount.options)?; + let state = inspect_mount(&path)?; + if state.kind == CdiPathKind::Socket { + accumulator.add_socket(path)?; + } else if is_ipc_directory(&path, state) { + validate_ipc_directory_options(&path, &mount.options)?; + accumulator.add_ipc_directory(path)?; + } else { + accumulator.add_mount(path, state.access)?; + } + } + for gid in &edits.additional_gids { + accumulator.add_gid(*gid)?; + } + accumulator.build() +} + +/// A workload-local mount snapshot, reused for all original directory mounts. +/// Expanded shared-library parents are not inspected as original mounts. +pub struct CdiMountTable { + mounts: Vec<(std::path::PathBuf, bool)>, +} + +/// Decode the octal escapes mountinfo uses for space, tab, newline and +/// backslash in a single pass, so an escaped backslash is never re-decoded. +fn decode_mountinfo_path(field: &str) -> String { + let mut decoded = String::with_capacity(field.len()); + let mut rest = field; + while let Some(index) = rest.find('\\') { + decoded.push_str(&rest[..index]); + let escape = rest.get(index + 1..index + 4); + match escape { + Some("040") => decoded.push(' '), + Some("011") => decoded.push('\t'), + Some("012") => decoded.push('\n'), + Some("134") => decoded.push('\\'), + _ => { + decoded.push('\\'); + rest = &rest[index + 1..]; + continue; + } + } + rest = &rest[index + 4..]; + } + decoded.push_str(rest); + decoded +} + +impl CdiMountTable { + pub fn read() -> Result { + let text = std::fs::read_to_string("/proc/self/mountinfo") + .map_err(|error| mount_inspection("/proc/self/mountinfo", error))?; + Self::parse(&text) + } + + fn parse(text: &str) -> Result { + let mut mounts = Vec::new(); + for line in text.lines() { + // The mount source may be empty, so split on the separator rather + // than counting whitespace-delimited fields. Paths are escaped and + // never contain a literal " - ". + let malformed = || mount_inspection("/proc/self/mountinfo", "malformed mount record"); + let (head, tail) = line.split_once(" - ").ok_or_else(malformed)?; + let fields = head.split(' ').collect::>(); + let tail = tail.split(' ').collect::>(); + if fields.len() < 6 || tail.len() != 3 || fields[..6].iter().any(|f| f.is_empty()) { + return Err(malformed()); + } + let path = decode_mountinfo_path(fields[4]); + if !Path::new(&path).is_absolute() { + return Err(mount_inspection( + "/proc/self/mountinfo", + "relative mount point", + )); + } + let read_only = fields[5].split(',').any(|option| option == "ro") + || tail[2].split(',').any(|option| option == "ro"); + if !read_only && !fields[5].split(',').any(|option| option == "rw") { + return Err(mount_inspection( + "/proc/self/mountinfo", + "missing mount access flags", + )); + } + mounts.push((path.into(), read_only)); + } + if mounts.is_empty() { + return Err(mount_inspection( + "/proc/self/mountinfo", + "empty mount table", + )); + } + Ok(Self { mounts }) + } + + /// Inspect the same object descriptor that will anchor a Landlock rule. + pub fn read_only(&self, path: &str, fd: &F) -> Result { + use rustix::fs::{FileType, StatVfsMountFlags, fstat, fstatvfs}; + use std::os::fd::AsRawFd; + let flags = fstatvfs(fd).map_err(|error| mount_inspection(path, error))?; + if !flags.f_flag.contains(StatVfsMountFlags::RDONLY) { + return Ok(false); + } + let stat = fstat(fd).map_err(|error| mount_inspection(path, error))?; + if FileType::from_raw_mode(stat.st_mode) != FileType::Directory { + return Ok(true); + } + let resolved = std::fs::read_link(format!("/proc/self/fd/{}", fd.as_fd().as_raw_fd())) + .map_err(|error| mount_inspection(path, error))?; + if resolved + .as_os_str() + .as_encoded_bytes() + .ends_with(b" (deleted)") + { + return Err(mount_inspection(path, "directory was removed")); + } + Ok(!self.mounts.iter().any(|(mount, read_only)| { + mount != &resolved && mount.starts_with(&resolved) && !read_only + })) + } +} + +fn mount_inspection(path: &str, error: impl fmt::Display) -> CdiError { + CdiError::MountInspection { + path: path.into(), + error: error.to_string(), + } +} + +fn inspect_cdi_mount( + path: &str, + table: &mut Option, +) -> Result { + let fd = rustix::fs::open( + path, + rustix::fs::OFlags::PATH | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .map_err(|error| mount_inspection(path, error))?; + let stat = rustix::fs::fstat(&fd).map_err(|error| mount_inspection(path, error))?; + let kind = match rustix::fs::FileType::from_raw_mode(stat.st_mode) { + rustix::fs::FileType::RegularFile => CdiPathKind::File, + rustix::fs::FileType::Directory => CdiPathKind::Directory, + rustix::fs::FileType::CharacterDevice => CdiPathKind::CharacterDevice, + rustix::fs::FileType::BlockDevice => CdiPathKind::BlockDevice, + rustix::fs::FileType::Socket => CdiPathKind::Socket, + _ => CdiPathKind::Other, + }; + // Sockets are exact validation targets, without filesystem-access grants. + let access = if kind == CdiPathKind::Socket { + CdiAccess::ReadOnly + } else { + if table.is_none() { + *table = Some(CdiMountTable::read()?); + } + if table + .as_ref() + .expect("mount table initialized") + .read_only(path, &fd)? + { + CdiAccess::ReadOnly + } else { + CdiAccess::ReadWrite + } + }; + Ok(CdiMountState { kind, access }) +} + +/// Ordinary writable CDI mounts need dedicated host-write authorization, which +/// is not currently supported. Authored filesystem grants cannot authorize them. +pub fn validate_cdi_requirements(requirements: &CdiDerivedRequirements) -> Result<(), CdiError> { + if let Some(path) = requirements.read_write_mount_paths.first() { + return Err(CdiError::WritableMountNotAllowed { path: path.clone() }); + } + Ok(()) +} + +fn validate_context(context: &CdiContext) -> Result<(), CdiError> { + if context.version != CDI_CONTEXT_VERSION { + return Err(CdiError::UnsupportedContextVersion(context.version)); + } + for spec_dir in &context.spec_dirs { + validate_absolute_no_parent(&spec_dir.path).map_err(|reason| CdiError::UnsafeSpecDir { + path: spec_dir.path.clone(), + diagnostic_source: spec_dir.source.clone(), + reason, + })?; + } + Ok(()) +} + +fn selected_cdi_devices(device_ids: &[String]) -> Vec { + let mut seen = HashSet::new(); + let mut parsed = Vec::new(); + for raw in device_ids { + let raw = raw.trim(); + if raw.is_empty() { + continue; + } + if seen.insert(raw.to_string()) { + parsed.push(raw.to_string()); + } + } + parsed +} + +fn resolve_container_edits( + context: &CdiContext, + selected_devices: &[String], +) -> Result { + let (mut cache, refresh_error) = build_cache(&context.spec_dirs); + let mut merged = UpstreamContainerEdits::new(); + let mut applied_specs = BTreeSet::new(); + + for device_id in selected_devices { + let device = cache + .get_device(device_id) + .cloned() + .ok_or_else(|| missing_device_error(device_id, refresh_error.as_deref()))?; + let mut spec = device.get_spec(); + let spec_path = spec.get_path(); + if applied_specs.insert(spec_path) + && let Some(spec_edits) = spec.edits() + { + merged + .append(spec_edits) + .map_err(|err| CdiError::EditMerge { + device: device_id.clone(), + error: err.to_string(), + })?; + } + merged + .append(device.edits()) + .map_err(|err| CdiError::EditMerge { + device: device_id.clone(), + error: err.to_string(), + })?; + } + + let value = serde_json::to_value(&merged.container_edits) + .map_err(|source| CdiError::EditEncode { source })?; + serde_json::from_value(value).map_err(|source| CdiError::EditDecode { source }) +} + +fn missing_device_error(device: &str, refresh_error: Option<&str>) -> CdiError { + refresh_error.map_or_else( + || CdiError::MissingDevice(device.to_string()), + |refresh_error| CdiError::MissingDeviceAfterRefresh { + device: device.to_string(), + refresh_error: refresh_error.to_string(), + }, + ) +} + +fn build_cache(spec_dirs: &[CdiSpecDirectory]) -> (Cache, Option) { + let spec_dir_paths = spec_dirs + .iter() + .map(|spec_dir| spec_dir.path.as_str()) + .collect::>(); + let mut cache = Cache::default(); + cache.configure(vec![ + with_spec_dirs(&spec_dir_paths), + with_auto_refresh(false), + ]); + let refresh_error = cache.refresh().err().map(|err| { + tracing::debug!( + error = %err, + "Ignoring CDI cache refresh error; requested device lookup will determine availability" + ); + err.to_string() + }); + (cache, refresh_error) +} + +/// A writable directory at a known IPC destination is an IPC directory rather +/// than an ordinary writable mount. Read-only mounts there stay ordinary. +fn is_ipc_directory(path: &str, state: CdiMountState) -> bool { + state.kind == CdiPathKind::Directory + && state.access == CdiAccess::ReadWrite + && super::CDI_IPC_DIRECTORIES.contains(&path) +} + +fn validate_ipc_directory_options(path: &str, options: &[String]) -> Result<(), CdiError> { + let complete = super::CDI_IPC_DIRECTORY_MOUNT_OPTIONS + .iter() + .all(|required| { + options + .iter() + .any(|option| option.eq_ignore_ascii_case(required)) + }); + if complete { + Ok(()) + } else { + Err(CdiError::UnsafeIpcDirectory { + path: path.to_string(), + }) + } +} + +fn validate_mount_options(path: &str, options: &[String]) -> Result<(), CdiError> { + let read_only_requested = options + .iter() + .any(|option| option.eq_ignore_ascii_case("ro")); + let read_write_requested = options + .iter() + .any(|option| option.eq_ignore_ascii_case("rw")); + // CDI mount options are stringly typed and runtime-specific normalization + // can differ. Treat simultaneous ro/rw as malformed instead of guessing + // which option a later mount syscall would effectively apply. + if read_only_requested && read_write_requested { + return Err(CdiError::ConflictingMountOptions { + path: path.to_string(), + }); + } + Ok(()) +} + +fn normalize_policy_path(path: &str) -> Result { + validate_absolute_no_parent(path).map_err(|reason| CdiError::UnsafePolicyPath { + path: path.to_string(), + reason, + })?; + let normalized = normalize_path(path); + if super::FORBIDDEN_CDI_ROOTS.contains(&normalized.as_str()) { + return Err(CdiError::UnsafePolicyPath { + path: path.to_string(), + reason: "broad root path is not allowed", + }); + } + Ok(normalized) +} + +fn validate_absolute_no_parent(path: &str) -> Result<(), &'static str> { + let path = Path::new(path); + if !path.is_absolute() { + return Err("path must be absolute"); + } + for component in path.components() { + match component { + Component::ParentDir => return Err("path must not contain '..'"), + Component::Prefix(_) => return Err("path must be a Unix-style absolute path"), + Component::CurDir => return Err("path must be normalized"), + Component::RootDir | Component::Normal(_) => {} + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn runtime_resolution_keeps_socket_destinations_exact_without_file_grants() { + let directory = tempfile::tempdir().unwrap(); + let specs = directory.path().join("specs"); + std::fs::create_dir(&specs).unwrap(); + for name in [ + "persistenced/socket", + "fabricmanager/socket", + "libdaemon.so.1", + ] { + let target = directory.path().join(name); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + let _listener = std::os::unix::net::UnixListener::bind(&target).unwrap(); + for options in ["[]", "[ro, bind]", "[rw, rbind]"] { + write_spec( + &specs, + "socket.yaml", + &format!( + "cdiVersion: 0.7.0\nkind: example.com/device\ndevices:\n - name: test\n containerEdits:\n mounts:\n - hostPath: /host/socket\n containerPath: {}\n options: {}\n", + target.display(), + options, + ), + ); + let requirements = + resolve_cdi_context(&context(&specs, &["example.com/device=test"])).unwrap(); + assert_eq!(requirements.socket_paths, vec![target.to_string_lossy()]); + assert!(requirements.read_only_paths.is_empty()); + assert!(requirements.read_write_mount_paths.is_empty()); + assert!(requirements.mounts.is_empty()); + validate_cdi_requirements(&requirements).unwrap(); + } + } + } + + #[test] + fn runtime_resolution_classifies_a_mount_without_access_options() { + let directory = tempfile::tempdir().unwrap(); + let target = directory.path().join("runtime.json"); + std::fs::write(&target, "{}").unwrap(); + let specs = directory.path().join("specs"); + std::fs::create_dir(&specs).unwrap(); + write_spec( + &specs, + "runtime.yaml", + &format!( + "cdiVersion: 0.7.0\nkind: example.com/device\ndevices:\n - name: test\n containerEdits:\n mounts:\n - hostPath: /host/runtime.json\n containerPath: {}\n", + target.display() + ), + ); + let requirements = + resolve_cdi_context(&context(&specs, &["example.com/device=test"])).unwrap(); + assert_eq!( + requirements.read_write_mount_paths, + vec![target.to_string_lossy()] + ); + assert!(requirements.read_only_paths.is_empty()); + } + + #[test] + fn effective_mount_flags_override_omitted_bind_and_ro_options() { + let directory = tempfile::tempdir().unwrap(); + let target = directory.path().join("runtime.json"); + std::fs::write(&target, "{}").unwrap(); + for options in [vec![], vec!["bind"], vec!["rbind"], vec!["ro"]] { + let path = target.to_string_lossy().into_owned(); + let requirements = + resolve_cdi_context(&runtime_mount_context(directory.path(), &path, &options)) + .unwrap(); + assert_eq!(requirements.read_write_mount_paths, vec![path.clone()]); + assert!(requirements.read_only_paths.is_empty()); + assert!(matches!( + validate_cdi_requirements(&requirements), + Err(CdiError::WritableMountNotAllowed { .. }) + )); + } + } + + #[test] + #[ignore = "requires a Linux container with read-only sysfs"] + fn effective_read_only_mount_needs_no_explicit_ro_option() { + let path = "/sys/kernel/uevent_seqnum"; + let directory = tempfile::tempdir().unwrap(); + let requirements = + resolve_cdi_context(&runtime_mount_context(directory.path(), path, &[])).unwrap(); + assert_eq!(requirements.read_only_paths, vec![path]); + assert!(requirements.read_write_mount_paths.is_empty()); + } + + #[test] + fn mount_table_accepts_empty_sources_and_optional_fields() { + let table = CdiMountTable::parse(concat!( + "1 0 0:1 / / rw shared:1 master:2 - overlay overlay rw,lowerdir=/l\n", + "2 1 0:2 / /usr/lib/x86_64-linux-gnu/libcuda.so.1 ro,nosuid - ext4 rw\n", + "3 1 0:3 / /run/nvidia\\040dir rw - tmpfs tmpfs ro\n", + "4 1 0:4 / /odd\\134040 rw - tmpfs tmpfs rw\n", + )) + .unwrap(); + assert_eq!( + table.mounts, + vec![ + ("/".into(), false), + ("/usr/lib/x86_64-linux-gnu/libcuda.so.1".into(), true), + ("/run/nvidia dir".into(), true), + ("/odd\\040".into(), false), + ] + ); + } + + #[test] + fn mount_table_rejects_malformed_records() { + for text in [ + "", + "malformed", + "1 0 0:1 / / rw overlay overlay rw", + "1 0 0:1 / / rw - overlay overlay", + "1 0 0:1 / relative rw - tmpfs tmpfs rw", + "1 0 0:1 / / nosuid - tmpfs tmpfs nosuid", + ] { + assert!(CdiMountTable::parse(text).is_err(), "{text:?}"); + } + } + + #[test] + #[ignore = "requires a Linux container with read-only sysfs"] + fn original_directory_mounts_reject_writable_descendants_and_resolve_aliases() { + let directory = tempfile::tempdir().unwrap(); + let alias = directory.path().join("alias"); + std::os::unix::fs::symlink("/sys/kernel", &alias).unwrap(); + let fd = + rustix::fs::open(&alias, rustix::fs::OFlags::PATH, rustix::fs::Mode::empty()).unwrap(); + let table = CdiMountTable::parse( + "1 0 0:1 / /sys ro - sysfs sysfs ro\n2 1 0:2 / /sys/kernel/debug rw - tmpfs tmpfs rw\n", + ) + .unwrap(); + assert!(!table.read_only(alias.to_str().unwrap(), &fd).unwrap()); + let table = CdiMountTable::parse("1 0 0:1 / /sys ro - sysfs sysfs ro\n2 1 0:2 / /sys/kernel/debug ro - tmpfs tmpfs ro\n3 0 0:3 / /sys/kernel-other rw - tmpfs tmpfs rw\n").unwrap(); + assert!(table.read_only(alias.to_str().unwrap(), &fd).unwrap()); + assert!(CdiMountTable::parse("malformed").is_err()); + assert!(CdiMountTable::parse("").is_err()); + } + + #[test] + fn effective_writable_directory_is_rejected_and_missing_mount_fails() { + let directory = tempfile::tempdir().unwrap(); + for path in [ + directory.path().to_string_lossy().into_owned(), + directory + .path() + .join("missing") + .to_string_lossy() + .into_owned(), + ] { + let context = runtime_mount_context(directory.path(), &path, &["ro"]); + if Path::new(&path).exists() { + let requirements = resolve_cdi_context(&context).unwrap(); + assert!(matches!( + validate_cdi_requirements(&requirements), + Err(CdiError::WritableMountNotAllowed { .. }) + )); + } else { + assert!(matches!( + resolve_cdi_context(&context), + Err(CdiError::MountInspection { .. }) + )); + } + } + } + + /// Options the NVIDIA Container Toolkit applies to IPC mounts. + const TOOLKIT_IPC_OPTIONS: &[&str] = &["nosuid", "nodev", "rbind", "rprivate", "noexec"]; + + #[test] + fn writable_mps_pipe_directory_is_an_ipc_directory() { + let directory = tempfile::tempdir().unwrap(); + let context = + runtime_mount_context(directory.path(), "/tmp/nvidia-mps", TOOLKIT_IPC_OPTIONS); + let requirements = resolve_with_kind_and_access(&context, CdiAccess::ReadWrite, |_| { + Some(CdiPathKind::Directory) + }) + .unwrap(); + assert_eq!(requirements.ipc_directory_paths, vec!["/tmp/nvidia-mps"]); + assert!(requirements.read_write_mount_paths.is_empty()); + assert!(requirements.read_only_paths.is_empty()); + assert!(requirements.mounts.is_empty()); + } + + #[test] + fn mps_pipe_directory_requires_restrictive_spec_options() { + let directory = tempfile::tempdir().unwrap(); + for missing in ["nosuid", "nodev", "noexec"] { + let options = TOOLKIT_IPC_OPTIONS + .iter() + .copied() + .filter(|option| *option != missing) + .collect::>(); + let context = runtime_mount_context(directory.path(), "/tmp/nvidia-mps", &options); + assert!( + matches!( + resolve_with_kind_and_access(&context, CdiAccess::ReadWrite, |_| { + Some(CdiPathKind::Directory) + }), + Err(CdiError::UnsafeIpcDirectory { .. }) + ), + "missing {missing}" + ); + } + } + + #[test] + fn ipc_directory_exception_is_limited_to_writable_mps_directories() { + let directory = tempfile::tempdir().unwrap(); + // A read-only MPS mount stays an ordinary read-only mount. + let context = + runtime_mount_context(directory.path(), "/tmp/nvidia-mps", TOOLKIT_IPC_OPTIONS); + let requirements = resolve_with_kind_and_access(&context, CdiAccess::ReadOnly, |_| { + Some(CdiPathKind::Directory) + }) + .unwrap(); + assert!(requirements.ipc_directory_paths.is_empty()); + assert_eq!(requirements.read_only_paths, vec!["/tmp/nvidia-mps"]); + // A writable file at the MPS path, or a writable directory elsewhere, + // remains an unsupported writable mount. + for (path, kind) in [ + ("/tmp/nvidia-mps", CdiPathKind::File), + ("/tmp/nvidia-mps-other", CdiPathKind::Directory), + ("/var/run/nvidia-mps", CdiPathKind::Directory), + ] { + let context = runtime_mount_context(directory.path(), path, TOOLKIT_IPC_OPTIONS); + assert!( + matches!( + resolve_with_kind_and_access(&context, CdiAccess::ReadWrite, |_| Some(kind)), + Err(CdiError::WritableMountNotAllowed { .. }) + ), + "{path}" + ); + } + } + + #[test] + fn ipc_directory_conflicts_with_other_cdi_uses_of_the_same_path() { + let mut accumulator = RequirementAccumulator::default(); + accumulator + .add_ipc_directory("/tmp/nvidia-mps".into()) + .unwrap(); + assert!( + accumulator + .add_device_node("/tmp/nvidia-mps".into()) + .is_err() + ); + assert!(accumulator.add_socket("/tmp/nvidia-mps".into()).is_err()); + assert!( + accumulator + .add_mount("/tmp/nvidia-mps".into(), CdiAccess::ReadOnly) + .is_err() + ); + let mut accumulator = RequirementAccumulator::default(); + accumulator + .add_mount("/tmp/nvidia-mps".into(), CdiAccess::ReadOnly) + .unwrap(); + assert!( + accumulator + .add_ipc_directory("/tmp/nvidia-mps".into()) + .is_err() + ); + } + + #[test] + fn writable_cdi_mounts_are_rejected() { + // Authored write grants cannot authorize them; enforcement covers that + // in the Landlock tests. + for path in ["/opt/nvidia/cache.db", "/opt/nvidia/cache"] { + let requirements = CdiDerivedRequirements { + read_write_mount_paths: vec![path.into()], + ..Default::default() + }; + assert!(matches!( + validate_cdi_requirements(&requirements), + Err(CdiError::WritableMountNotAllowed { .. }) + )); + } + } + + #[test] + fn rejects_broad_shared_library_parents() { + for parent in ["", "/dev", "/proc", "/sys", "/run", "/usr"] { + let mut accumulator = RequirementAccumulator::default(); + accumulator + .add_mount(format!("{parent}/libexample.so.1"), CdiAccess::ReadOnly) + .unwrap(); + assert!(matches!( + accumulator.build(), + Err(CdiError::UnsafePolicyPath { .. }) + )); + } + } + + fn write_spec(dir: &Path, name: &str, yaml: &str) { + std::fs::write(dir.join(name), yaml).unwrap(); + } + + fn context(dir: &Path, selected_devices: &[&str]) -> CdiContext { + CdiContext::new( + selected_devices + .iter() + .map(|device| (*device).to_string()) + .collect(), + vec![CdiSpecDirectory::new( + dir.to_string_lossy().into_owned(), + "/host/cdi", + )], + ) + } + + fn resolve_with_kind( + context: &CdiContext, + kind: impl Fn(&str) -> Option, + ) -> Result { + resolve_with_kind_and_access(context, CdiAccess::ReadOnly, kind) + } + + fn resolve_with_kind_and_access( + context: &CdiContext, + access: CdiAccess, + kind: impl Fn(&str) -> Option, + ) -> Result { + let requirements = resolve_cdi_context_with_mount_inspector(context, |path| { + let kind = kind(path).ok_or_else(|| mount_inspection(path, "missing fixture mount"))?; + Ok(CdiMountState { kind, access }) + })?; + validate_cdi_requirements(&requirements)?; + Ok(requirements) + } + + fn fixture_mount_state(path: &str) -> CdiMountState { + // These captured NVIDIA layouts model ordinary mounts as effectively + // read-only and the persistenced endpoint as a Unix socket. Neither + // observation is inferred from ro/rw strings in the specs. + CdiMountState { + kind: if path == "/run/nvidia-persistenced/socket" { + CdiPathKind::Socket + } else { + CdiPathKind::File + }, + access: CdiAccess::ReadOnly, + } + } + + fn resolve_fixture(context: &CdiContext) -> Result { + resolve_cdi_context_with_mount_inspector(context, |path| Ok(fixture_mount_state(path))) + } + + fn runtime_mount_context(dir: &Path, path: &str, options: &[&str]) -> CdiContext { + let specs = dir.join("specs"); + std::fs::create_dir_all(&specs).unwrap(); + let spec = serde_json::json!({ + "cdiVersion": "1.1.0", "kind": "example.com/device", + "devices": [{"name": "test", "containerEdits": {"mounts": [{ + "hostPath": "/host/mount", "containerPath": path, "options": options + }]}}] + }); + write_spec(&specs, "runtime.json", &spec.to_string()); + context(&specs, &["example.com/device=test"]) + } + + fn always_missing(_: &str) -> Option { + None + } + + fn fake_device_node(path: &str) -> Option { + path.starts_with("/dev/") + .then_some(CdiPathKind::CharacterDevice) + } + + #[derive(serde::Serialize)] + struct CdiRequirementsBaseline<'a> { + device_node_paths: &'a [String], + socket_paths: &'a [String], + read_only_paths: &'a [String], + read_write_mount_paths: &'a [String], + ipc_directory_paths: &'a [String], + additional_gids: &'a [u32], + } + + fn assert_cdi_spec_matches_derived_requirements_baseline(spec: &str, expected: &str) { + let dir = tempfile::tempdir().unwrap(); + write_spec(dir.path(), "nvidia.yaml", spec); + + let requirements = resolve_fixture(&context(dir.path(), &["nvidia.com/gpu=all"])).unwrap(); + let baseline = CdiRequirementsBaseline { + device_node_paths: &requirements.device_node_paths, + socket_paths: &requirements.socket_paths, + read_only_paths: &requirements.read_only_paths, + read_write_mount_paths: &requirements.read_write_mount_paths, + ipc_directory_paths: &requirements.ipc_directory_paths, + additional_gids: &requirements.additional_gids, + }; + let generated = serde_json::to_string_pretty(&baseline).unwrap() + "\n"; + + assert_eq!(generated, expected); + } + + #[test] + fn orin_cdi_spec_matches_derived_requirements_baseline() { + assert_cdi_spec_matches_derived_requirements_baseline( + include_str!("../../testdata/cdi/orin-nvidia.yaml"), + include_str!("../../testdata/cdi/orin-nvidia-policy-baseline.json"), + ); + } + + #[test] + fn spark_cdi_spec_matches_derived_requirements_baseline() { + assert_cdi_spec_matches_derived_requirements_baseline( + include_str!("../../testdata/cdi/spark-nvidia.yaml"), + include_str!("../../testdata/cdi/spark-nvidia-policy-baseline.json"), + ); + } + + #[test] + fn multi_gpu_cdi_spec_matches_derived_requirements_baseline() { + assert_cdi_spec_matches_derived_requirements_baseline( + include_str!("../../testdata/cdi/multi-gpu-nvidia.yaml"), + include_str!("../../testdata/cdi/multi-gpu-nvidia-policy-baseline.json"), + ); + } + + #[test] + fn fixture_resolution_uses_mount_state_instead_of_spec_options() { + let directory = tempfile::tempdir().unwrap(); + for (options, access) in [ + (vec![], CdiAccess::ReadOnly), + (vec!["rw"], CdiAccess::ReadOnly), + (vec!["ro"], CdiAccess::ReadWrite), + ] { + let context = + runtime_mount_context(directory.path(), "/opt/nvidia/runtime.json", &options); + let requirements = resolve_cdi_context_with_mount_inspector(&context, |_| { + Ok(CdiMountState { + kind: CdiPathKind::File, + access, + }) + }) + .unwrap(); + if access == CdiAccess::ReadOnly { + assert_eq!( + requirements.read_only_paths, + vec!["/opt/nvidia/runtime.json"] + ); + assert!(requirements.read_write_mount_paths.is_empty()); + } else { + assert_eq!( + requirements.read_write_mount_paths, + vec!["/opt/nvidia/runtime.json"] + ); + assert!(requirements.read_only_paths.is_empty()); + } + } + } + + #[test] + fn resolves_native_single_device_requirements() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "nvidia.yaml", + r#" +cdiVersion: 0.6.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + deviceNodes: + - path: /dev/nvidiactl + - path: /dev/nvidia0 + mounts: + - hostPath: /host/libcuda.so.1 + containerPath: /usr/local/cuda/lib64/libcuda.so.1 +"#, + ); + + let requirements = resolve_fixture(&context(dir.path(), &["nvidia.com/gpu=0"])).unwrap(); + + assert_eq!( + requirements.device_node_paths, + vec!["/dev/nvidia0", "/dev/nvidiactl"] + ); + assert_eq!(requirements.read_only_paths, vec!["/usr/local/cuda/lib64"]); + } + + #[test] + fn resolves_native_all_device_requirements() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "nvidia.yaml", + r" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: all + containerEdits: + deviceNodes: + - path: /dev/nvidiactl + - path: /dev/nvidia0 + - path: /dev/nvidia1 +", + ); + + let requirements = resolve_fixture(&context(dir.path(), &["nvidia.com/gpu=all"])).unwrap(); + + assert_eq!( + requirements.device_node_paths, + vec!["/dev/nvidia0", "/dev/nvidia1", "/dev/nvidiactl"] + ); + } + + #[test] + fn resolves_wsl_shape_requirements() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "wsl.yaml", + r" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: all + containerEdits: + deviceNodes: + - path: /dev/dxg + mounts: + - hostPath: /host/wsl/lib/libcuda.so.1 + containerPath: /usr/lib/wsl/lib/libcuda.so.1 +", + ); + + let requirements = resolve_fixture(&context(dir.path(), &["nvidia.com/gpu=all"])).unwrap(); + + assert_eq!(requirements.device_node_paths, vec!["/dev/dxg"]); + assert_eq!(requirements.read_only_paths, vec!["/usr/lib/wsl/lib"]); + } + + #[test] + fn resolves_supported_shared_library_parent_directories() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "libraries.yaml", + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + mounts: + - hostPath: /host/lib/libcuda.so + containerPath: /opt/nvidia/lib/libcuda.so + - hostPath: /host/lib/libnvidia-ml.so.1 + containerPath: /opt/nvidia/lib/libnvidia-ml.so.1 + - hostPath: /host/lib/libnvidia-encode.so.9 + containerPath: /opt/nvidia/encode/libnvidia-encode.so.9 + - hostPath: /host/lib/libnvidia-eglcore.so.540.5.0 + containerPath: /opt/nvidia/versioned/libnvidia-eglcore.so.540.5.0 + - hostPath: /host/lib/libunsupported.so.debug + containerPath: /opt/nvidia/debug/libunsupported.so.debug + - hostPath: /host/lib/libunsupported.so.1beta + containerPath: /opt/nvidia/debug/libunsupported.so.1beta + - hostPath: /host/config/ld.so.conf + containerPath: /etc/nvidia/ld.so.conf + - hostPath: /host/bin/nvidia-smi + containerPath: /usr/bin/nvidia-smi + - hostPath: /host/config/runtime.json + containerPath: /etc/nvidia/runtime.json +"#, + ); + + let requirements = resolve_fixture(&context(dir.path(), &["nvidia.com/gpu=0"])).unwrap(); + + assert_eq!( + requirements.read_only_paths, + vec![ + "/etc/nvidia/ld.so.conf", + "/etc/nvidia/runtime.json", + "/opt/nvidia/debug/libunsupported.so.1beta", + "/opt/nvidia/debug/libunsupported.so.debug", + "/opt/nvidia/encode", + "/opt/nvidia/lib", + "/opt/nvidia/versioned", + "/usr/bin/nvidia-smi", + ] + ); + } + + #[test] + fn resolves_tegra_shape_requirements_and_gids() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "tegra.yaml", + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +containerEdits: + additionalGids: [44] +devices: + - name: "0" + containerEdits: + deviceNodes: + - path: /dev/nvmap + - path: /dev/nvhost-gpu +"#, + ); + + let requirements = resolve_fixture(&context(dir.path(), &["nvidia.com/gpu=0"])).unwrap(); + + assert_eq!(requirements.additional_gids, vec![44]); + assert_eq!( + requirements.device_node_paths, + vec!["/dev/nvhost-gpu", "/dev/nvmap"] + ); + } + + #[test] + fn rejects_writable_single_file_mount_with_exact_filesystem_grant() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "rw-file.yaml", + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + mounts: + - hostPath: /host/nvidia/cache.db + containerPath: /opt/nvidia/cache.db + options: [rw] +"#, + ); + + let result = resolve_with_kind_and_access( + &context(dir.path(), &["nvidia.com/gpu=0"]), + CdiAccess::ReadWrite, + |path| (path == "/opt/nvidia/cache.db").then_some(CdiPathKind::File), + ); + assert!(matches!( + result, + Err(CdiError::WritableMountNotAllowed { .. }) + )); + } + + #[test] + fn rejects_missing_selected_device() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "nvidia.yaml", + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + env: + - OPEN_SHELL_TEST=1 +"#, + ); + + let err = resolve_with_kind(&context(dir.path(), &["nvidia.com/gpu=1"]), always_missing) + .unwrap_err(); + + assert!(matches!(err, CdiError::MissingDevice(device) if device == "nvidia.com/gpu=1")); + } + + #[test] + fn reports_refresh_errors_when_requested_device_is_missing() { + let dir = tempfile::tempdir().unwrap(); + write_spec(dir.path(), "broken.yaml", "kind: ["); + + let err = resolve_with_kind( + &context(dir.path(), &["nvidia.com/gpu=0"]), + fake_device_node, + ) + .unwrap_err(); + + assert!( + matches!(err, CdiError::MissingDeviceAfterRefresh { device, refresh_error } + if device == "nvidia.com/gpu=0" && !refresh_error.is_empty()) + ); + } + + #[test] + fn rejects_duplicate_cdi_device_names() { + let dir = tempfile::tempdir().unwrap(); + for (name, value) in [("first.yaml", "FIRST"), ("second.yaml", "SECOND")] { + write_spec( + dir.path(), + name, + &format!( + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + env: + - OPEN_SHELL_TEST={value} +"# + ), + ); + } + + let err = resolve_with_kind(&context(dir.path(), &["nvidia.com/gpu=0"]), always_missing) + .unwrap_err(); + + assert!( + matches!(err, CdiError::MissingDeviceAfterRefresh { device, refresh_error } + if device == "nvidia.com/gpu=0" + && refresh_error.contains("conflicting device nvidia.com/gpu=0")) + ); + } + + #[test] + fn empty_selection_is_noop() { + let dir = tempfile::tempdir().unwrap(); + let requirements = resolve_with_kind(&context(dir.path(), &[]), always_missing) + .expect("empty selection should resolve to empty requirements"); + + assert_eq!(requirements, CdiDerivedRequirements::default()); + } + + #[test] + fn empty_spec_dirs_are_noop() { + let context = CdiContext::new(vec!["nvidia.com/gpu=0".to_string()], Vec::new()); + + let requirements = resolve_with_kind(&context, always_missing) + .expect("empty spec dirs should resolve to empty requirements"); + + assert_eq!(requirements, CdiDerivedRequirements::default()); + } + + #[test] + fn defers_device_id_shape_to_upstream_resolution() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "nvidia.yaml", + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + env: + - OPEN_SHELL_TEST=1 +"#, + ); + + let err = + resolve_with_kind(&context(dir.path(), &["not-a-cdi-id"]), always_missing).unwrap_err(); + + assert!(matches!(err, CdiError::MissingDevice(device) if device == "not-a-cdi-id")); + } + + #[test] + fn rejects_unsafe_policy_paths() { + for path in [ + "relative", "/dev", "/proc", "/sys", "/run", "/usr", "/a/../b", + ] { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "unsafe.yaml", + &format!( + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + deviceNodes: + - path: {path} +"# + ), + ); + + let err = + resolve_with_kind(&context(dir.path(), &["nvidia.com/gpu=0"]), always_missing) + .unwrap_err(); + + assert!( + matches!(err, CdiError::UnsafePolicyPath { .. }), + "expected unsafe path error for {path}, got {err:?}" + ); + } + } + + #[test] + fn rejects_duplicate_access_conflicts() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "conflict.yaml", + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + mounts: + - hostPath: /host/dev/nvidia0 + containerPath: /dev/nvidia0 + options: [ro] +"#, + ); + + let err = resolve_with_kind( + &context(dir.path(), &["nvidia.com/gpu=0"]), + fake_device_node, + ) + .unwrap_err(); + + assert!(matches!(err, CdiError::ConflictingAccess { .. })); + } + + #[test] + fn rejects_root_additional_gid() { + let dir = tempfile::tempdir().unwrap(); + write_spec( + dir.path(), + "root-gid.yaml", + r#" +cdiVersion: 1.1.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + additionalGids: [0, 44] +"#, + ); + + let err = resolve_with_kind(&context(dir.path(), &["nvidia.com/gpu=0"]), always_missing) + .unwrap_err(); + + assert!(matches!(err, CdiError::RootAdditionalGid)); + } +} diff --git a/crates/openshell-sandbox/src/launch.rs b/crates/openshell-sandbox/src/launch.rs new file mode 100644 index 0000000000..bddf14bc7a --- /dev/null +++ b/crates/openshell-sandbox/src/launch.rs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workload-local requirements derived for launch-time enforcement. + +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct CdiDerivedRequirements { + pub device_node_paths: Vec, + /// Exact Unix socket destinations. Never expanded into filesystem grants. + pub socket_paths: Vec, + /// Derived read-only policy paths. Shared-library mounts contribute their + /// parent directory; other read-only mounts contribute their exact path. + pub read_only_paths: Vec, + pub read_write_mount_paths: Vec, + /// Known writable IPC directories, such as the NVIDIA MPS pipe directory. + /// Granted read-write on the exact directory and revalidated at each launch. + pub ipc_directory_paths: Vec, + pub additional_gids: Vec, + /// Original ordinary mount destinations, separate from sockets and library parents. + /// Access is confirmed in the workload namespace before enrichment. + pub mounts: Vec, +} + +#[cfg(target_os = "linux")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CdiMountRequirement { + pub path: String, + pub read_only: bool, +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index da99518833..f12041a714 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -9,10 +9,13 @@ pub mod boundary_exec; pub mod boundary_io; mod boundary_server; #[cfg(target_os = "linux")] +mod cdi; +#[cfg(target_os = "linux")] pub mod child_env; pub mod container_log; #[cfg(target_os = "linux")] pub(crate) mod delegated; +pub mod launch; #[cfg(target_os = "linux")] pub mod linux; #[cfg(target_os = "linux")] diff --git a/crates/openshell-sandbox/testdata/cdi/multi-gpu-nvidia-policy-baseline.json b/crates/openshell-sandbox/testdata/cdi/multi-gpu-nvidia-policy-baseline.json new file mode 100644 index 0000000000..012c7fc6ac --- /dev/null +++ b/crates/openshell-sandbox/testdata/cdi/multi-gpu-nvidia-policy-baseline.json @@ -0,0 +1,42 @@ +{ + "device_node_paths": [ + "/dev/nvidia-modeset", + "/dev/nvidia-uvm", + "/dev/nvidia-uvm-tools", + "/dev/nvidia0", + "/dev/nvidia1", + "/dev/nvidiactl" + ], + "socket_paths": [ + "/run/nvidia-persistenced/socket" + ], + "read_only_paths": [ + "/etc/OpenCL/vendors/nvidia.icd", + "/etc/vulkan/icd.d/nvidia_icd.json", + "/etc/vulkan/implicit_layer.d/nvidia_layers.json", + "/lib/firmware/nvidia/615.71.09/gsp_ga10x.bin", + "/lib/firmware/nvidia/615.71.09/gsp_tu10x.bin", + "/run/nvidia/driver/lib/firmware/nvidia/615.71.09/gsp_ga10x.bin", + "/run/nvidia/driver/lib/firmware/nvidia/615.71.09/gsp_tu10x.bin", + "/usr/bin/nvidia-cuda-mps-control", + "/usr/bin/nvidia-cuda-mps-server", + "/usr/bin/nvidia-debugdump", + "/usr/bin/nvidia-imex", + "/usr/bin/nvidia-imex-ctl", + "/usr/bin/nvidia-persistenced", + "/usr/bin/nvidia-smi", + "/usr/bin/nvidia-xconfig", + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/x86_64-linux-gnu/vdpau", + "/usr/share/egl/egl_external_platform.d/09_nvidia_wayland2.json", + "/usr/share/egl/egl_external_platform.d/10_nvidia_wayland.json", + "/usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json", + "/usr/share/egl/egl_external_platform.d/20_nvidia_xcb.json", + "/usr/share/egl/egl_external_platform.d/20_nvidia_xlib.json", + "/usr/share/glvnd/egl_vendor.d/10_nvidia.json", + "/usr/share/nvidia/nvoptix.bin" + ], + "read_write_mount_paths": [], + "ipc_directory_paths": [], + "additional_gids": [] +} diff --git a/crates/openshell-sandbox/testdata/cdi/multi-gpu-nvidia.yaml b/crates/openshell-sandbox/testdata/cdi/multi-gpu-nvidia.yaml new file mode 100644 index 0000000000..e6576987e1 --- /dev/null +++ b/crates/openshell-sandbox/testdata/cdi/multi-gpu-nvidia.yaml @@ -0,0 +1,817 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Copied from ~/Downloads/nvidia.yaml on 2026-09-17 as a multi-GPU fixture. +# SHA-256: 51e2c048894654bf1dba558b9cdaff8586ffb2bbb472f75855c1eb5f8bc5ad71 +--- +cdiVersion: 0.5.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + hostPath: /run/nvidia/driver/dev/nvidia0 + major: 195 + fileMode: 438 + permissions: rwm + hooks: + - hookName: createRuntime + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - apply-cuda-memory-limits + - --driver-root + - /run/nvidia/driver + - --gpu-id + - GPU-ebd34bdf-1083-eaac-2aff-4b71a022f9bd + env: + - NVIDIA_CTK_DEBUG=false + - name: "1" + containerEdits: + deviceNodes: + - path: /dev/nvidia1 + hostPath: /run/nvidia/driver/dev/nvidia1 + major: 195 + minor: 1 + fileMode: 438 + permissions: rwm + hooks: + - hookName: createRuntime + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - apply-cuda-memory-limits + - --driver-root + - /run/nvidia/driver + - --gpu-id + - GPU-1795e88b-3395-b27b-dad8-0488474eec0c + env: + - NVIDIA_CTK_DEBUG=false + - name: GPU-1795e88b-3395-b27b-dad8-0488474eec0c + containerEdits: + deviceNodes: + - path: /dev/nvidia1 + hostPath: /run/nvidia/driver/dev/nvidia1 + major: 195 + minor: 1 + fileMode: 438 + permissions: rwm + hooks: + - hookName: createRuntime + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - apply-cuda-memory-limits + - --driver-root + - /run/nvidia/driver + - --gpu-id + - GPU-1795e88b-3395-b27b-dad8-0488474eec0c + env: + - NVIDIA_CTK_DEBUG=false + - name: GPU-ebd34bdf-1083-eaac-2aff-4b71a022f9bd + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + hostPath: /run/nvidia/driver/dev/nvidia0 + major: 195 + fileMode: 438 + permissions: rwm + hooks: + - hookName: createRuntime + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - apply-cuda-memory-limits + - --driver-root + - /run/nvidia/driver + - --gpu-id + - GPU-ebd34bdf-1083-eaac-2aff-4b71a022f9bd + env: + - NVIDIA_CTK_DEBUG=false + - name: all + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + hostPath: /run/nvidia/driver/dev/nvidia0 + major: 195 + fileMode: 438 + permissions: rwm + - path: /dev/nvidia1 + hostPath: /run/nvidia/driver/dev/nvidia1 + major: 195 + minor: 1 + fileMode: 438 + permissions: rwm + hooks: + - hookName: createRuntime + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - apply-cuda-memory-limits + - --driver-root + - /run/nvidia/driver + - --gpu-id + - GPU-ebd34bdf-1083-eaac-2aff-4b71a022f9bd + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createRuntime + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - apply-cuda-memory-limits + - --driver-root + - /run/nvidia/driver + - --gpu-id + - GPU-1795e88b-3395-b27b-dad8-0488474eec0c + env: + - NVIDIA_CTK_DEBUG=false +containerEdits: + env: + - NVIDIA_CTK_LIBCUDA_DIR=/usr/lib/x86_64-linux-gnu + - NVIDIA_VISIBLE_DEVICES=void + deviceNodes: + - path: /dev/nvidia-modeset + hostPath: /run/nvidia/driver/dev/nvidia-modeset + major: 195 + minor: 254 + fileMode: 438 + permissions: rwm + - path: /dev/nvidia-uvm + hostPath: /run/nvidia/driver/dev/nvidia-uvm + major: 511 + fileMode: 438 + permissions: rwm + - path: /dev/nvidia-uvm-tools + hostPath: /run/nvidia/driver/dev/nvidia-uvm-tools + major: 511 + minor: 1 + fileMode: 438 + permissions: rwm + - path: /dev/nvidiactl + hostPath: /run/nvidia/driver/dev/nvidiactl + major: 195 + minor: 255 + fileMode: 438 + permissions: rwm + hooks: + - hookName: createContainer + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - ../libnvidia-allocator.so.1::/usr/lib/x86_64-linux-gnu/gbm/nvidia-drm_gbm.so + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - libEGL_nvidia.so.615.71.09::/usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.0 + - --link + - libGLESv1_CM_nvidia.so.615.71.09::/usr/lib/x86_64-linux-gnu/libGLESv1_CM_nvidia.so.1 + - --link + - libGLESv2_nvidia.so.615.71.09::/usr/lib/x86_64-linux-gnu/libGLESv2_nvidia.so.2 + - --link + - libGLX_nvidia.so.615.71.09::/usr/lib/x86_64-linux-gnu/libGLX_indirect.so.0 + - --link + - libGLX_nvidia.so.615.71.09::/usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.0 + - --link + - libcuda.so.1::/usr/lib/x86_64-linux-gnu/libcuda.so + - --link + - libcuda.so.615.71.09::/usr/lib/x86_64-linux-gnu/libcuda.so.1 + - --link + - libcudadebugger.so.615.71.09::/usr/lib/x86_64-linux-gnu/libcudadebugger.so.1 + - --link + - libnvcuvid.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvcuvid.so.1 + - --link + - libnvcuvid.so.1::/usr/lib/x86_64-linux-gnu/libnvcuvid.so + - --link + - libnvidia-allocator.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.1 + - --link + - libnvidia-allocator.so.1::/usr/lib/x86_64-linux-gnu/libnvidia-allocator.so + - --link + - libnvidia-cfg.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.1 + - --link + - libnvidia-cfg.so.1::/usr/lib/x86_64-linux-gnu/libnvidia-cfg.so + - --link + - libnvidia-encode.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-encode.so.1 + - --link + - libnvidia-encode.so.1::/usr/lib/x86_64-linux-gnu/libnvidia-encode.so + - --link + - libnvidia-fbc.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-fbc.so.1 + - --link + - libnvidia-fbc.so.1::/usr/lib/x86_64-linux-gnu/libnvidia-fbc.so + - --link + - libnvidia-fmdrv.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-fmdrv.so.1 + - --link + - libnvidia-imex.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-imex.so.1 + - --link + - libnvidia-ml.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1 + - --link + - libnvidia-ml.so.1::/usr/lib/x86_64-linux-gnu/libnvidia-ml.so + - --link + - libnvidia-ngx.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-ngx.so.1 + - --link + - libnvidia-nscq.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-nscq.so.2 + - --link + - libnvidia-nscq.so.2::/usr/lib/x86_64-linux-gnu/libnvidia-nscq.so + - --link + - libnvidia-nvvm.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-nvvm.so.4 + - --link + - libnvidia-nvvm.so.4::/usr/lib/x86_64-linux-gnu/libnvidia-nvvm.so + - --link + - libnvidia-opencl.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.1 + - --link + - libnvidia-opticalflow.so.1::/usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so + - --link + - libnvidia-opticalflow.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.1 + - --link + - libnvidia-ptxjitcompiler.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.1 + - --link + - libnvidia-ptxjitcompiler.so.1::/usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so + - --link + - libnvidia-sandboxutils.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-sandboxutils.so.1 + - --link + - libnvidia-sandboxutils.so.1::/usr/lib/x86_64-linux-gnu/libnvidia-sandboxutils.so + - --link + - libnvidia-vksc-core.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvidia-vksc-core.so.1 + - --link + - libnvoptix.so.615.71.09::/usr/lib/x86_64-linux-gnu/libnvoptix.so.1 + - --link + - libvdpau_nvidia.so.615.71.09::/usr/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.1 + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - enable-cuda-compat + - --host-driver-version=615.71.09 + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-ldcache + - --folder + - /usr/lib/x86_64-linux-gnu + - --folder + - /usr/lib/x86_64-linux-gnu/vdpau + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - disable-device-node-modification + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/local/nvidia/toolkit/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-application-profile + env: + - NVIDIA_CTK_DEBUG=false + mounts: + - hostPath: /run/nvidia/driver/run/nvidia-persistenced/socket + containerPath: /run/nvidia-persistenced/socket + options: + - nosuid + - nodev + - rbind + - rprivate + - noexec + - hostPath: /run/nvidia/driver/usr/bin/nvidia-cuda-mps-control + containerPath: /usr/bin/nvidia-cuda-mps-control + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/bin/nvidia-cuda-mps-server + containerPath: /usr/bin/nvidia-cuda-mps-server + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/bin/nvidia-debugdump + containerPath: /usr/bin/nvidia-debugdump + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/bin/nvidia-imex + containerPath: /usr/bin/nvidia-imex + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/bin/nvidia-imex-ctl + containerPath: /usr/bin/nvidia-imex-ctl + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/bin/nvidia-persistenced + containerPath: /usr/bin/nvidia-persistenced + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/bin/nvidia-smi + containerPath: /usr/bin/nvidia-smi + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/bin/nvidia-xconfig + containerPath: /usr/bin/nvidia-xconfig + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/etc/OpenCL/vendors/nvidia.icd + containerPath: /etc/OpenCL/vendors/nvidia.icd + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/etc/vulkan/icd.d/nvidia_icd.json + containerPath: /etc/vulkan/icd.d/nvidia_icd.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/etc/vulkan/implicit_layer.d/nvidia_layers.json + containerPath: /etc/vulkan/implicit_layer.d/nvidia_layers.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libGLESv1_CM_nvidia.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libGLESv1_CM_nvidia.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libGLESv2_nvidia.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libGLESv2_nvidia.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libcuda.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libcuda.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libcudadebugger.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libcudadebugger.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvcuvid.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvcuvid.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-egl-gbm.so.1.1.3 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-egl-gbm.so.1.1.3 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-egl-wayland.so.1.1.20 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-egl-wayland.so.1.1.20 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-egl-wayland2.so.1.0.2 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-egl-wayland2.so.1.0.2 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-egl-xcb.so.1.0.6 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-egl-xcb.so.1.0.6 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-egl-xlib.so.1.0.6 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-egl-xlib.so.1.0.6 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-eglcore.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-eglcore.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-encode.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-encode.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-fbc.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-fbc.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-fmdrv.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-fmdrv.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-glcore.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-glcore.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-glsi.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-glsi.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-glvkspirv.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-glvkspirv.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-gpucomp.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-gpucomp.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-gtk2.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-gtk2.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-gtk3.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-gtk3.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-imex.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-imex.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ngx.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-ngx.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-nscq.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-nscq.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-nvvm.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-nvvm.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-nvvm70.so.4 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-nvvm70.so.4 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-pkcs11-openssl3.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-pkcs11-openssl3.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-pkcs11.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-pkcs11.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-present.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-present.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-rtcore.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-rtcore.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-sandboxutils.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-sandboxutils.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-tileiras.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-tileiras.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-tls.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-tls.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-vksc-core.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-vksc-core.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-wayland-client.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvidia-wayland-client.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvoptix.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/libnvoptix.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/share/nvidia/nvoptix.bin + containerPath: /usr/share/nvidia/nvoptix.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/lib/firmware/nvidia/615.71.09/gsp_ga10x.bin + containerPath: /lib/firmware/nvidia/615.71.09/gsp_ga10x.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/lib/firmware/nvidia/615.71.09/gsp_tu10x.bin + containerPath: /lib/firmware/nvidia/615.71.09/gsp_tu10x.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.615.71.09 + containerPath: /usr/lib/x86_64-linux-gnu/vdpau/libvdpau_nvidia.so.615.71.09 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/share/egl/egl_external_platform.d/09_nvidia_wayland2.json + containerPath: /usr/share/egl/egl_external_platform.d/09_nvidia_wayland2.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/share/egl/egl_external_platform.d/10_nvidia_wayland.json + containerPath: /usr/share/egl/egl_external_platform.d/10_nvidia_wayland.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json + containerPath: /usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/share/egl/egl_external_platform.d/20_nvidia_xcb.json + containerPath: /usr/share/egl/egl_external_platform.d/20_nvidia_xcb.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/share/egl/egl_external_platform.d/20_nvidia_xlib.json + containerPath: /usr/share/egl/egl_external_platform.d/20_nvidia_xlib.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/usr/share/glvnd/egl_vendor.d/10_nvidia.json + containerPath: /usr/share/glvnd/egl_vendor.d/10_nvidia.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/run/nvidia/driver/lib/firmware/nvidia/615.71.09/gsp_ga10x.bin + containerPath: /run/nvidia/driver/lib/firmware/nvidia/615.71.09/gsp_ga10x.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /run/nvidia/driver/run/nvidia/driver/lib/firmware/nvidia/615.71.09/gsp_tu10x.bin + containerPath: /run/nvidia/driver/lib/firmware/nvidia/615.71.09/gsp_tu10x.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate diff --git a/crates/openshell-sandbox/testdata/cdi/orin-nvidia-policy-baseline.json b/crates/openshell-sandbox/testdata/cdi/orin-nvidia-policy-baseline.json new file mode 100644 index 0000000000..f3d87ba376 --- /dev/null +++ b/crates/openshell-sandbox/testdata/cdi/orin-nvidia-policy-baseline.json @@ -0,0 +1,99 @@ +{ + "device_node_paths": [ + "/dev/dri/card0", + "/dev/dri/renderD128", + "/dev/host1x-fence", + "/dev/nvgpu/igpu0/as", + "/dev/nvgpu/igpu0/channel", + "/dev/nvgpu/igpu0/ctrl", + "/dev/nvgpu/igpu0/ctxsw", + "/dev/nvgpu/igpu0/dbg", + "/dev/nvgpu/igpu0/nvsched", + "/dev/nvgpu/igpu0/power", + "/dev/nvgpu/igpu0/prof", + "/dev/nvgpu/igpu0/prof-ctx", + "/dev/nvgpu/igpu0/prof-dev", + "/dev/nvgpu/igpu0/sched", + "/dev/nvgpu/igpu0/tsg", + "/dev/nvhost-as-gpu", + "/dev/nvhost-ctrl-gpu", + "/dev/nvhost-ctxsw-gpu", + "/dev/nvhost-dbg-gpu", + "/dev/nvhost-gpu", + "/dev/nvhost-nvsched-gpu", + "/dev/nvhost-power-gpu", + "/dev/nvhost-prof-ctx-gpu", + "/dev/nvhost-prof-dev-gpu", + "/dev/nvhost-prof-gpu", + "/dev/nvhost-sched-gpu", + "/dev/nvhost-tsg-gpu", + "/dev/nvidia0", + "/dev/nvidiactl", + "/dev/nvmap", + "/dev/nvsciipc", + "/dev/v4l2-nvdec" + ], + "socket_paths": [], + "read_only_paths": [ + "/dev/dri/by-path", + "/etc/nv_tegra_release", + "/etc/vulkansc/icd.d/nvidia_icd_vksc.json", + "/usr/bin/nvidia-cuda-mps-control", + "/usr/bin/nvidia-cuda-mps-server", + "/usr/lib/aarch64-linux-gnu/nvidia", + "/usr/lib/aarch64-linux-gnu/nvidia/ld.so.conf", + "/usr/lib/aarch64-linux-gnu/nvidia/nvidia_icd.json", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/EGLWLInputEventExample", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/EGLWLMockNavigation", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/LayerManagerControl", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/simple-weston-client", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/spring-tool", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-calibrator", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-clickdot", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-cliptest", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-content-protection", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-debug", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-desktop-shell", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-dnd", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-eventdemo", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-flower", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-fullscreen", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-image", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-ivi-shell-user-interface", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-keyboard", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-multi-resource", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-presentation-shm", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-resizor", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-scaler", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-screenshooter", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-damage", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-egl", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-egldevice", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-feedback", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-egl", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-im", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-shm", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-touch", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-smoke", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-stacking", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-subsurfaces", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-tablet", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-terminal", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-touch-calibrator", + "/usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-transformed", + "/usr/lib/aarch64-linux-gnu/tegra-egl", + "/usr/lib/aarch64-linux-gnu/tegra-egl/ld.so.conf", + "/usr/lib/aarch64-linux-gnu/tegra-egl/nvidia.json", + "/usr/lib/firmware/tegra23x/nvhost_nvdec050_desc_prod.bin", + "/usr/sbin/nvidia-smi" + ], + "read_write_mount_paths": [], + "ipc_directory_paths": [], + "additional_gids": [ + 44, + 104, + 995 + ] +} diff --git a/crates/openshell-sandbox/testdata/cdi/orin-nvidia.yaml b/crates/openshell-sandbox/testdata/cdi/orin-nvidia.yaml new file mode 100644 index 0000000000..1c0b76bcc1 --- /dev/null +++ b/crates/openshell-sandbox/testdata/cdi/orin-nvidia.yaml @@ -0,0 +1,2218 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Captured from orin-dev:/var/run/cdi/nvidia.yaml on 2026-09-17. +# Original SHA-256: 52d0de5622e816351b256213e21f72d32b3ea4e4dbb2de68ce1afe7e13e80b36 +--- +cdiVersion: 0.7.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + deviceNodes: + - path: /dev/host1x-fence + major: 492 + fileMode: 438 + permissions: rwm + - path: /dev/nvhost-as-gpu + major: 505 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-ctrl-gpu + major: 505 + minor: 2 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-ctxsw-gpu + major: 505 + minor: 3 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-dbg-gpu + major: 505 + minor: 4 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-gpu + major: 505 + minor: 1 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-nvsched-gpu + major: 505 + minor: 9 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-power-gpu + major: 506 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-prof-ctx-gpu + major: 505 + minor: 6 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-prof-dev-gpu + major: 505 + minor: 7 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-prof-gpu + major: 505 + minor: 5 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-sched-gpu + major: 505 + minor: 8 + fileMode: 432 + permissions: rwm + - path: /dev/nvhost-tsg-gpu + major: 505 + minor: 11 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvidia0 + major: 195 + fileMode: 438 + permissions: rwm + - path: /dev/nvidiactl + major: 195 + minor: 255 + fileMode: 438 + permissions: rwm + - path: /dev/nvmap + major: 10 + minor: 124 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvsciipc + major: 507 + fileMode: 438 + permissions: rwm + - path: /dev/v4l2-nvdec + major: 1 + minor: 3 + fileMode: 438 + permissions: rwm + - path: /dev/dri/card0 + major: 226 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/dri/renderD128 + major: 226 + minor: 128 + fileMode: 432 + permissions: rwm + gid: 104 + - path: /dev/nvgpu/igpu0/as + major: 505 + minor: 12 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/channel + major: 505 + minor: 13 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/ctrl + major: 505 + minor: 14 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/ctxsw + major: 505 + minor: 15 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/dbg + major: 505 + minor: 16 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/nvsched + major: 505 + minor: 21 + fileMode: 416 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/power + major: 506 + minor: 1 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/prof + major: 505 + minor: 17 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/prof-ctx + major: 505 + minor: 18 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/prof-dev + major: 505 + minor: 19 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/sched + major: 505 + minor: 20 + fileMode: 432 + permissions: rwm + - path: /dev/nvgpu/igpu0/tsg + major: 505 + minor: 23 + fileMode: 432 + permissions: rwm + gid: 44 + additionalGids: + - 44 + - 104 + - 995 + - name: all + containerEdits: + deviceNodes: + - path: /dev/host1x-fence + major: 492 + fileMode: 438 + permissions: rwm + - path: /dev/nvhost-as-gpu + major: 505 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-ctrl-gpu + major: 505 + minor: 2 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-ctxsw-gpu + major: 505 + minor: 3 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-dbg-gpu + major: 505 + minor: 4 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-gpu + major: 505 + minor: 1 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-nvsched-gpu + major: 505 + minor: 9 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-power-gpu + major: 506 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvhost-prof-ctx-gpu + major: 505 + minor: 6 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-prof-dev-gpu + major: 505 + minor: 7 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-prof-gpu + major: 505 + minor: 5 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvhost-sched-gpu + major: 505 + minor: 8 + fileMode: 432 + permissions: rwm + - path: /dev/nvhost-tsg-gpu + major: 505 + minor: 11 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvidia0 + major: 195 + fileMode: 438 + permissions: rwm + - path: /dev/nvidiactl + major: 195 + minor: 255 + fileMode: 438 + permissions: rwm + - path: /dev/nvmap + major: 10 + minor: 124 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvsciipc + major: 507 + fileMode: 438 + permissions: rwm + - path: /dev/v4l2-nvdec + major: 1 + minor: 3 + fileMode: 438 + permissions: rwm + - path: /dev/dri/card0 + major: 226 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/dri/renderD128 + major: 226 + minor: 128 + fileMode: 432 + permissions: rwm + gid: 104 + - path: /dev/nvgpu/igpu0/as + major: 505 + minor: 12 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/channel + major: 505 + minor: 13 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/ctrl + major: 505 + minor: 14 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/ctxsw + major: 505 + minor: 15 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/dbg + major: 505 + minor: 16 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/nvsched + major: 505 + minor: 21 + fileMode: 416 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/power + major: 506 + minor: 1 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/nvgpu/igpu0/prof + major: 505 + minor: 17 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/prof-ctx + major: 505 + minor: 18 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/prof-dev + major: 505 + minor: 19 + fileMode: 432 + permissions: rwm + gid: 995 + - path: /dev/nvgpu/igpu0/sched + major: 505 + minor: 20 + fileMode: 432 + permissions: rwm + - path: /dev/nvgpu/igpu0/tsg + major: 505 + minor: 23 + fileMode: 432 + permissions: rwm + gid: 44 + additionalGids: + - 44 + - 104 + - 995 +containerEdits: + env: + - NVIDIA_VISIBLE_DEVICES=void + hooks: + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - libcuda.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so + - --link + - libcuda.so.1.1::/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1 + - --link + - libnvbufsurface.so.1.0.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurface.so + - --link + - libnvbufsurftransform.so.1.0.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurftransform.so + - --link + - libnvdsbufferpool.so.1.0.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvdsbufferpool.so + - --link + - libnvidia-allocator.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-allocator.so + - --link + - libnvidia-egl-gbm.so.1.1.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-egl-gbm.so.1 + - --link + - libnvidia-egl-wayland.so.1.1.11::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-egl-wayland.so.1 + - --link + - libnvidia-kms.so.540.5.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-kms.so + - --link + - libnvidia-nvvm.so.540.5.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-nvvm.so.4 + - --link + - libnvidia-ptxjitcompiler.so.540.5.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ptxjitcompiler.so.1 + - --link + - libnvidia-vksc-core.so.540.5.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-vksc-core.so + - --link + - libnvid_mapper.so.1.0.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvid_mapper.so + - --link + - libnvscibuf.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvscibuf.so + - --link + - libnvscicommon.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvscicommon.so + - --link + - libnvscistream.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvscistream.so + - --link + - libnvscisync.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvscisync.so + - --link + - libvulkansc.so.1.0.10::/usr/lib/aarch64-linux-gnu/nvidia/libvulkansc.so.1 + - --link + - libvulkansc.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libvulkansc.so + - --link + - libwayland-client.so.0.22.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-client.so.0 + - --link + - libwayland-client.so.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-client.so + - --link + - libwayland-cursor.so.0.22.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-cursor.so.0 + - --link + - libwayland-cursor.so.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-cursor.so + - --link + - libwayland-egl.so.1.22.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-egl.so.1 + - --link + - libwayland-egl.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-egl.so + - --link + - libwayland-server.so.0.22.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-server.so.0 + - --link + - libwayland-server.so.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-server.so + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - /usr/lib/aarch64-linux-gnu/nvidia/nvidia_icd.json::/etc/vulkan/icd.d/nvidia_icd.json + - --link + - ../nvidia/libnvidia-allocator.so::/usr/lib/aarch64-linux-gnu/gbm/nvidia-drm_gbm.so + - --link + - libnvidia-allocator.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-allocator.so + - --link + - ../nvidia/libnvidia-allocator.so::/usr/lib/aarch64-linux-gnu/gbm/tegra_gbm.so + - --link + - ../nvidia/libnvidia-allocator.so::/usr/lib/aarch64-linux-gnu/gbm/tegra-udrm_gbm.so + - --link + - nvidia/libcuda.so::/usr/lib/aarch64-linux-gnu/libcuda.so + - --link + - libcuda.so.1.1::/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so + - --link + - nvidia/libnvcucompat.so::/usr/lib/aarch64-linux-gnu/libnvcucompat.so + - --link + - nvidia/libnvcudla.so::/usr/lib/aarch64-linux-gnu/libnvcudla.so + - --link + - nvidia/libnvv4l2.so::/usr/lib/aarch64-linux-gnu/libv4l2.so.0.0.999999 + - --link + - nvidia/libnvv4lconvert.so::/usr/lib/aarch64-linux-gnu/libv4lconvert.so.0.0.999999 + - --link + - ../../../nvidia/libv4l2_nvargus.so::/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so + - --link + - ../../../nvidia/libv4l2_nvcuvidvideocodec.so::/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvcuvidvideocodec.so + - --link + - ../../../nvidia/libv4l2_nvvideocodec.so::/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvvideocodec.so + - --link + - libcuda.so.1.1::/usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1 + - --link + - libnvbufsurface.so.1.0.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurface.so + - --link + - libnvbufsurftransform.so.1.0.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurftransform.so + - --link + - libnvdsbufferpool.so.1.0.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvdsbufferpool.so + - --link + - libnvidia-egl-gbm.so.1.1.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-egl-gbm.so.1 + - --link + - libnvidia-egl-wayland.so.1.1.11::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-egl-wayland.so.1 + - --link + - libnvidia-kms.so.540.5.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-kms.so + - --link + - libnvidia-nvvm.so.540.5.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-nvvm.so.4 + - --link + - libnvidia-ptxjitcompiler.so.540.5.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ptxjitcompiler.so.1 + - --link + - libnvidia-vksc-core.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-vksc-core.so + - --link + - libnvidia-vksc-core.so.540.5.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvidia-vksc-core.so.1 + - --link + - libnvid_mapper.so.1.0.0::/usr/lib/aarch64-linux-gnu/nvidia/libnvid_mapper.so + - --link + - libnvscibuf.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvscibuf.so + - --link + - libnvscicommon.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvscicommon.so + - --link + - libnvscistream.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvscistream.so + - --link + - libnvscisync.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libnvscisync.so + - --link + - libnvv4l2.so::/usr/lib/aarch64-linux-gnu/nvidia/libv4l2.so.0 + - --link + - libnvv4lconvert.so::/usr/lib/aarch64-linux-gnu/nvidia/libv4lconvert.so.0 + - --link + - libvulkansc.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libvulkansc.so + - --link + - libvulkansc.so.1.0.10::/usr/lib/aarch64-linux-gnu/nvidia/libvulkansc.so.1 + - --link + - libwayland-client.so.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-client.so + - --link + - libwayland-client.so.0.22.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-client.so.0 + - --link + - libwayland-cursor.so.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-cursor.so + - --link + - libwayland-cursor.so.0.22.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-cursor.so.0 + - --link + - libwayland-egl.so.1::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-egl.so + - --link + - libwayland-egl.so.1.22.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-egl.so.1 + - --link + - libwayland-server.so.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-server.so + - --link + - libwayland-server.so.0.22.0::/usr/lib/aarch64-linux-gnu/nvidia/libwayland-server.so.0 + - --link + - ../../../lib/aarch64-linux-gnu/tegra-egl/nvidia.json::/usr/share/glvnd/egl_vendor.d/10_nvidia.json + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - enable-cuda-compat + - --host-cuda-version=12.6 + - --cuda-compat-container-root=/dev/null + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-ldcache + - --folder + - /usr/lib/aarch64-linux-gnu/nvidia + - --folder + - /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0 + - --folder + - /usr/lib/aarch64-linux-gnu/tegra-egl + env: + - NVIDIA_CTK_DEBUG=false + mounts: + - hostPath: /etc/nv_tegra_release + containerPath: /etc/nv_tegra_release + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /dev/dri/by-path + containerPath: /dev/dri/by-path + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/bin/nvidia-cuda-mps-control + containerPath: /usr/bin/nvidia-cuda-mps-control + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/bin/nvidia-cuda-mps-server + containerPath: /usr/bin/nvidia-cuda-mps-server + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/sbin/nvidia-smi + containerPath: /usr/sbin/nvidia-smi + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /etc/vulkansc/icd.d/nvidia_icd_vksc.json + containerPath: /etc/vulkansc/icd.d/nvidia_icd_vksc.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/ld.so.conf + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/ld.so.conf + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libGLX_nvidia.so.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libGLX_nvidia.so.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libVkLayer_json_gen.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libVkLayer_json_gen.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libVkSCLayer_khronos_validation.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libVkSCLayer_khronos_validation.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libcuda.so.1.1 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libjetsonpower.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libjetsonpower.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvargus.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvargus.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvargus_socketclient.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvargus_socketclient.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvargus_socketserver.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvargus_socketserver.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvbuf_fdmap.so.1.0.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvbuf_fdmap.so.1.0.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurface.so.1.0.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurface.so.1.0.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurftransform.so.1.0.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvbufsurftransform.so.1.0.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcam_imageencoder.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcam_imageencoder.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcameratools.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcameratools.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcamerautils.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcamerautils.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcamlog.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcamlog.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcamv4l2.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcamv4l2.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcapture.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcapture.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcolorutil.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcolorutil.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcucompat.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcucompat.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcudla.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcudla.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcuvidv4l2.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvcuvidv4l2.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvdc.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvdc.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvddk_2d_v2.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvddk_2d_v2.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvddk_vic.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvddk_vic.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvdecode2eglimage.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvdecode2eglimage.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvdla_runtime.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvdla_runtime.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvdsbufferpool.so.1.0.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvdsbufferpool.so.1.0.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnveventlib.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnveventlib.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvexif.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvexif.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfnet.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfnet.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfnetstoredefog.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfnetstoredefog.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfnetstorehdfx.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfnetstorehdfx.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfusacap.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfusacap.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfusacapinterface.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvfusacapinterface.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_boot.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_boot.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_camera.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_camera.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_force.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_force.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_generic.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_generic.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_gpucompute.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_gpucompute.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_graphics.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_graphics.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_il.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_il.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_spincircle.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_spincircle.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_tbc.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_tbc.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_ui.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvgov_ui.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvid_mapper.so.1.0.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvid_mapper.so.1.0.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-allocator.so.1 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-allocator.so.1 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-egl-gbm.so.1.1.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-egl-gbm.so.1.1.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-egl-wayland.so.1.1.11 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-egl-wayland.so.1.1.11 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-eglcore.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-eglcore.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-glcore.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-glcore.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-glsi.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-glsi.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-glvkspirv.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-glvkspirv.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-gpucomp.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-gpucomp.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-kms.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-kms.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ml.so.1 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-nvvm.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-nvvm.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ptxjitcompiler.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-ptxjitcompiler.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-rmapi-tegra.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-rmapi-tegra.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-rtcore.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-rtcore.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-tls.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-tls.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-vksc-core.so.540.5.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvidia-vksc-core.so.540.5.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvimp.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvimp.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvisp.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvisp.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvisp_utils.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvisp_utils.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvisppg.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvisppg.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvjpeg.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvjpeg.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia2d.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia2d.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_2d.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_2d.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_dla.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_dla.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_eglstream.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_eglstream.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_ide_parser.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_ide_parser.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_ide_sci.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_ide_sci.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_iep_sci.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_iep_sci.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_ijpd_sci.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_ijpd_sci.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_ijpe_sci.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_ijpe_sci.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_iofa_sci.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_iofa_sci.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_isp_ext.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_isp_ext.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_tensor.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedia_tensor.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedialdc.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmedialdc.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmm.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmm.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmm_contentpipe.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmm_contentpipe.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmm_parser.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmm_parser.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmm_utils.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmm_utils.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmmlite.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmmlite.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmmlite_image.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmmlite_image.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmmlite_utils.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmmlite_utils.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmmlite_video.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvmmlite_video.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvodm_imager.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvodm_imager.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvofsdk.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvofsdk.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvomx.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvomx.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvomxilclient.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvomxilclient.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvos.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvos.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvosd.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvosd.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvparser.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvparser.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvphs.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvphs.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvphsd.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvphsd.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvplayfair.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvplayfair.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvpva.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvpva.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvpva_algorithms.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvpva_algorithms.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvpvaintf.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvpvaintf.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvpvaumd.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvpvaumd.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_chip.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_chip.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_gpu.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_gpu.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_host1x.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_host1x.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_mem.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_mem.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_stream.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_stream.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_surface.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_surface.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_sync.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvrm_sync.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscf.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscf.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscibuf.so.1 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscibuf.so.1 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscicommon.so.1 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscicommon.so.1 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscievent.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscievent.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvsciipc.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvsciipc.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscistream.so.1 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscistream.so.1 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscisync.so.1 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvscisync.so.1 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvsocsys.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvsocsys.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvtegrahv.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvtegrahv.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvtracebuf.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvtracebuf.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvtvmr.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvtvmr.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvtvmr_2d.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvtvmr_2d.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvv4l2.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvv4l2.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvv4lconvert.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvv4lconvert.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvvic.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvvic.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvvideo.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvvideo.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvvideoencode_ppe.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libnvvideoencode_ppe.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libsensors.hal-client.nvs.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libsensors.hal-client.nvs.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libsensors.l4t.no_fusion.nvs.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libsensors.l4t.no_fusion.nvs.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libsensors_hal.nvs.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libsensors_hal.nvs.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libtegrav4l2.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libtegrav4l2.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libtegrawfd.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libtegrawfd.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libv4l2_nvargus.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libv4l2_nvargus.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libv4l2_nvcuvidvideocodec.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libv4l2_nvcuvidvideocodec.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libv4l2_nvvideocodec.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libv4l2_nvvideocodec.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libvulkansc.so.1.0.10 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libvulkansc.so.1.0.10 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libwayland-client.so.0.22.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libwayland-client.so.0.22.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libwayland-cursor.so.0.22.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libwayland-cursor.so.0.22.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libwayland-egl.so.1.22.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libwayland-egl.so.1.22.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/libwayland-server.so.0.22.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/libwayland-server.so.0.22.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/nvidia_icd.json + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/nvidia_icd.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/tegra-egl/ld.so.conf + containerPath: /usr/lib/aarch64-linux-gnu/tegra-egl/ld.so.conf + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/tegra-egl/libEGL_nvidia.so.0 + containerPath: /usr/lib/aarch64-linux-gnu/tegra-egl/libEGL_nvidia.so.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/tegra-egl/libGLESv1_CM_nvidia.so.1 + containerPath: /usr/lib/aarch64-linux-gnu/tegra-egl/libGLESv1_CM_nvidia.so.1 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/tegra-egl/libGLESv2_nvidia.so.2 + containerPath: /usr/lib/aarch64-linux-gnu/tegra-egl/libGLESv2_nvidia.so.2 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/tegra-egl/nvidia.json + containerPath: /usr/lib/aarch64-linux-gnu/tegra-egl/nvidia.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/firmware/tegra23x/nvhost_nvdec050_desc_prod.bin + containerPath: /usr/lib/firmware/tegra23x/nvhost_nvdec050_desc_prod.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/EGLWLInputEventExample + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/EGLWLInputEventExample + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/EGLWLMockNavigation + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/EGLWLMockNavigation + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/LayerManagerControl + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/LayerManagerControl + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/color-lcms.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/color-lcms.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/desktop-shell.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/desktop-shell.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/drm-backend.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/drm-backend.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/fullscreen-shell.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/fullscreen-shell.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/gl-renderer.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/gl-renderer.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/headless-backend.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/headless-backend.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/hmi-controller.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/hmi-controller.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/ivi-controller.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/ivi-controller.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/ivi-shell.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/ivi-shell.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libilmClient.so.2.3.2 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libilmClient.so.2.3.2 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libilmCommon.so.2.3.2 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libilmCommon.so.2.3.2 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libilmControl.so.2.3.2 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libilmControl.so.2.3.2 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libilmInput.so.2.3.2 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libilmInput.so.2.3.2 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libweston-13.so.0 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/libweston-13.so.0 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/simple-weston-client + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/simple-weston-client + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/spring-tool + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/spring-tool + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/wayland-backend.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/wayland-backend.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-calibrator + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-calibrator + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-clickdot + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-clickdot + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-cliptest + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-cliptest + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-content-protection + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-content-protection + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-debug + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-debug + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-desktop-shell + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-desktop-shell + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-dnd + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-dnd + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-eventdemo + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-eventdemo + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-flower + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-flower + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-fullscreen + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-fullscreen + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-image + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-image + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-ivi-shell-user-interface + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-ivi-shell-user-interface + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-keyboard + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-keyboard + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-multi-resource + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-multi-resource + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-presentation-shm + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-presentation-shm + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-resizor + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-resizor + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-scaler + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-scaler + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-screenshooter + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-screenshooter + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-damage + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-damage + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-egl + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-egl + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-egldevice + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-egldevice + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-feedback + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-dmabuf-feedback + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-egl + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-egl + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-im + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-im + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-shm + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-shm + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-touch + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-simple-touch + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-smoke + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-smoke + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-stacking + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-stacking + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-subsurfaces + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-subsurfaces + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-tablet + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-tablet + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-terminal + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-terminal + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-touch-calibrator + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-touch-calibrator + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-transformed + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/weston-13.0/weston-transformed + options: + - ro + - nosuid + - nodev + - rbind + - rprivate diff --git a/crates/openshell-sandbox/testdata/cdi/spark-nvidia-policy-baseline.json b/crates/openshell-sandbox/testdata/cdi/spark-nvidia-policy-baseline.json new file mode 100644 index 0000000000..8ed7390b4c --- /dev/null +++ b/crates/openshell-sandbox/testdata/cdi/spark-nvidia-policy-baseline.json @@ -0,0 +1,39 @@ +{ + "device_node_paths": [ + "/dev/dri/card0", + "/dev/dri/renderD128", + "/dev/nvidia-modeset", + "/dev/nvidia-uvm", + "/dev/nvidia-uvm-tools", + "/dev/nvidia0", + "/dev/nvidiactl" + ], + "socket_paths": [ + "/run/nvidia-persistenced/socket" + ], + "read_only_paths": [ + "/etc/vulkan/icd.d/nvidia_icd.json", + "/etc/vulkan/implicit_layer.d/nvidia_layers.json", + "/lib/firmware/nvidia/580.142/gsp_ga10x.bin", + "/lib/firmware/nvidia/580.142/gsp_tu10x.bin", + "/usr/bin/nvidia-cuda-mps-control", + "/usr/bin/nvidia-cuda-mps-server", + "/usr/bin/nvidia-debugdump", + "/usr/bin/nvidia-persistenced", + "/usr/bin/nvidia-smi", + "/usr/lib/aarch64-linux-gnu", + "/usr/lib/aarch64-linux-gnu/nvidia/xorg", + "/usr/lib/aarch64-linux-gnu/vdpau", + "/usr/share/X11/xorg.conf.d/10-nvidia.conf", + "/usr/share/egl/egl_external_platform.d/10_nvidia_wayland.json", + "/usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json", + "/usr/share/glvnd/egl_vendor.d/10_nvidia.json", + "/usr/share/nvidia/nvoptix.bin" + ], + "read_write_mount_paths": [], + "ipc_directory_paths": [], + "additional_gids": [ + 44, + 993 + ] +} diff --git a/crates/openshell-sandbox/testdata/cdi/spark-nvidia.yaml b/crates/openshell-sandbox/testdata/cdi/spark-nvidia.yaml new file mode 100644 index 0000000000..e026124d9b --- /dev/null +++ b/crates/openshell-sandbox/testdata/cdi/spark-nvidia.yaml @@ -0,0 +1,600 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Captured from spark-dev:/var/run/cdi/nvidia.yaml on 2026-09-17. +# SHA-256: 86913dad81e83b8314dbae9eebd5279b606d701b712180a7c390277932cf287e +--- +cdiVersion: 0.7.0 +kind: nvidia.com/gpu +devices: + - name: "0" + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + major: 195 + fileMode: 438 + permissions: rwm + - path: /dev/dri/card0 + major: 226 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/dri/renderD128 + major: 226 + minor: 128 + fileMode: 432 + permissions: rwm + gid: 993 + hooks: + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - ../card0::/dev/dri/by-path/pci-000f:01:00.0-card + - --link + - ../renderD128::/dev/dri/by-path/pci-000f:01:00.0-render + env: + - NVIDIA_CTK_DEBUG=false + additionalGids: + - 44 + - 993 + - name: GPU-017980a8-44ff-7f9c-320d-a7f9fa4d808c + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + major: 195 + fileMode: 438 + permissions: rwm + - path: /dev/dri/card0 + major: 226 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/dri/renderD128 + major: 226 + minor: 128 + fileMode: 432 + permissions: rwm + gid: 993 + hooks: + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - ../card0::/dev/dri/by-path/pci-000f:01:00.0-card + - --link + - ../renderD128::/dev/dri/by-path/pci-000f:01:00.0-render + env: + - NVIDIA_CTK_DEBUG=false + additionalGids: + - 44 + - 993 + - name: all + containerEdits: + deviceNodes: + - path: /dev/nvidia0 + major: 195 + fileMode: 438 + permissions: rwm + - path: /dev/dri/card0 + major: 226 + fileMode: 432 + permissions: rwm + gid: 44 + - path: /dev/dri/renderD128 + major: 226 + minor: 128 + fileMode: 432 + permissions: rwm + gid: 993 + hooks: + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - ../card0::/dev/dri/by-path/pci-000f:01:00.0-card + - --link + - ../renderD128::/dev/dri/by-path/pci-000f:01:00.0-render + env: + - NVIDIA_CTK_DEBUG=false + additionalGids: + - 44 + - 993 +containerEdits: + env: + - NVIDIA_CTK_LIBCUDA_DIR=/usr/lib/aarch64-linux-gnu + - NVIDIA_VISIBLE_DEVICES=void + deviceNodes: + - path: /dev/nvidia-modeset + major: 195 + minor: 254 + fileMode: 438 + permissions: rwm + - path: /dev/nvidia-uvm + major: 498 + fileMode: 438 + permissions: rwm + - path: /dev/nvidia-uvm-tools + major: 498 + minor: 1 + fileMode: 438 + permissions: rwm + - path: /dev/nvidiactl + major: 195 + minor: 255 + fileMode: 438 + permissions: rwm + hooks: + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - ../libnvidia-allocator.so.1::/usr/lib/aarch64-linux-gnu/gbm/nvidia-drm_gbm.so + - --link + - libglxserver_nvidia.so.580.142::/usr/lib/aarch64-linux-gnu/nvidia/xorg/libglxserver_nvidia.so + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - create-symlinks + - --link + - libEGL_nvidia.so.580.142::/usr/lib/aarch64-linux-gnu/libEGL_nvidia.so.0 + - --link + - libGLESv1_CM_nvidia.so.580.142::/usr/lib/aarch64-linux-gnu/libGLESv1_CM_nvidia.so.1 + - --link + - libGLESv2_nvidia.so.580.142::/usr/lib/aarch64-linux-gnu/libGLESv2_nvidia.so.2 + - --link + - libGLX_nvidia.so.580.142::/usr/lib/aarch64-linux-gnu/libGLX_indirect.so.0 + - --link + - libGLX_nvidia.so.580.142::/usr/lib/aarch64-linux-gnu/libGLX_nvidia.so.0 + - --link + - libcuda.so.1::/usr/lib/aarch64-linux-gnu/libcuda.so + - --link + - libcuda.so.580.142::/usr/lib/aarch64-linux-gnu/libcuda.so.1 + - --link + - libcudadebugger.so.580.142::/usr/lib/aarch64-linux-gnu/libcudadebugger.so.1 + - --link + - libnvcuvid.so.580.142::/usr/lib/aarch64-linux-gnu/libnvcuvid.so.1 + - --link + - libnvcuvid.so.1::/usr/lib/aarch64-linux-gnu/libnvcuvid.so + - --link + - libnvidia-allocator.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-allocator.so.1 + - --link + - libnvidia-allocator.so.1::/usr/lib/aarch64-linux-gnu/libnvidia-allocator.so + - --link + - libnvidia-cfg.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-cfg.so.1 + - --link + - libnvidia-cfg.so.1::/usr/lib/aarch64-linux-gnu/libnvidia-cfg.so + - --link + - libnvidia-encode.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-encode.so.1 + - --link + - libnvidia-encode.so.1::/usr/lib/aarch64-linux-gnu/libnvidia-encode.so + - --link + - libnvidia-fbc.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-fbc.so.1 + - --link + - libnvidia-fbc.so.1::/usr/lib/aarch64-linux-gnu/libnvidia-fbc.so + - --link + - libnvidia-ml.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1 + - --link + - libnvidia-ml.so.1::/usr/lib/aarch64-linux-gnu/libnvidia-ml.so + - --link + - libnvidia-ngx.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-ngx.so.1 + - --link + - libnvidia-nvvm.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-nvvm.so.4 + - --link + - libnvidia-nvvm.so.4::/usr/lib/aarch64-linux-gnu/libnvidia-nvvm.so + - --link + - libnvidia-opencl.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-opencl.so.1 + - --link + - libnvidia-opticalflow.so.1::/usr/lib/aarch64-linux-gnu/libnvidia-opticalflow.so + - --link + - libnvidia-opticalflow.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-opticalflow.so.1 + - --link + - libnvidia-ptxjitcompiler.so.580.142::/usr/lib/aarch64-linux-gnu/libnvidia-ptxjitcompiler.so.1 + - --link + - libnvidia-ptxjitcompiler.so.1::/usr/lib/aarch64-linux-gnu/libnvidia-ptxjitcompiler.so + - --link + - libnvoptix.so.580.142::/usr/lib/aarch64-linux-gnu/libnvoptix.so.1 + - --link + - libvdpau_nvidia.so.580.142::/usr/lib/aarch64-linux-gnu/vdpau/libvdpau_nvidia.so.1 + - --link + - libvdpau_nvidia.so.1::/usr/lib/aarch64-linux-gnu/vdpau/libvdpau_nvidia.so + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - enable-cuda-compat + - --host-driver-version=580.142 + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - update-ldcache + - --folder + - /usr/lib/aarch64-linux-gnu + - --folder + - /usr/lib/aarch64-linux-gnu/vdpau + env: + - NVIDIA_CTK_DEBUG=false + - hookName: createContainer + path: /usr/bin/nvidia-cdi-hook + args: + - nvidia-cdi-hook + - disable-device-node-modification + env: + - NVIDIA_CTK_DEBUG=false + mounts: + - hostPath: /run/nvidia-persistenced/socket + containerPath: /run/nvidia-persistenced/socket + options: + - nosuid + - nodev + - rbind + - rprivate + - noexec + - hostPath: /usr/bin/nvidia-cuda-mps-control + containerPath: /usr/bin/nvidia-cuda-mps-control + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/bin/nvidia-cuda-mps-server + containerPath: /usr/bin/nvidia-cuda-mps-server + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/bin/nvidia-debugdump + containerPath: /usr/bin/nvidia-debugdump + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/bin/nvidia-persistenced + containerPath: /usr/bin/nvidia-persistenced + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/bin/nvidia-smi + containerPath: /usr/bin/nvidia-smi + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/share/vulkan/icd.d/nvidia_icd.json + containerPath: /etc/vulkan/icd.d/nvidia_icd.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/share/vulkan/implicit_layer.d/nvidia_layers.json + containerPath: /etc/vulkan/implicit_layer.d/nvidia_layers.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libEGL_nvidia.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libEGL_nvidia.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libGLESv1_CM_nvidia.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libGLESv1_CM_nvidia.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libGLESv2_nvidia.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libGLESv2_nvidia.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libGLX_nvidia.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libGLX_nvidia.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libcuda.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libcuda.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libcudadebugger.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libcudadebugger.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvcuvid.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvcuvid.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-allocator.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-allocator.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-cfg.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-cfg.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-egl-gbm.so.1.1.3 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-egl-gbm.so.1.1.3 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-egl-wayland.so.1.1.21 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-egl-wayland.so.1.1.21 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-eglcore.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-eglcore.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-encode.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-encode.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-fbc.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-fbc.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-glcore.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-glcore.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-glsi.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-glsi.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-glvkspirv.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-glvkspirv.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-gpucomp.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-gpucomp.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-ml.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-ml.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-ngx.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-ngx.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-nvvm.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-nvvm.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-opencl.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-opencl.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-opticalflow.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-opticalflow.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-ptxjitcompiler.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-ptxjitcompiler.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-rtcore.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-rtcore.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvidia-tls.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvidia-tls.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/libnvoptix.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/libnvoptix.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/share/nvidia/nvoptix.bin + containerPath: /usr/share/nvidia/nvoptix.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /lib/firmware/nvidia/580.142/gsp_ga10x.bin + containerPath: /lib/firmware/nvidia/580.142/gsp_ga10x.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /lib/firmware/nvidia/580.142/gsp_tu10x.bin + containerPath: /lib/firmware/nvidia/580.142/gsp_tu10x.bin + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/vdpau/libvdpau_nvidia.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/vdpau/libvdpau_nvidia.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/share/X11/xorg.conf.d/10-nvidia.conf + containerPath: /usr/share/X11/xorg.conf.d/10-nvidia.conf + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/share/egl/egl_external_platform.d/10_nvidia_wayland.json + containerPath: /usr/share/egl/egl_external_platform.d/10_nvidia_wayland.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json + containerPath: /usr/share/egl/egl_external_platform.d/15_nvidia_gbm.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/share/glvnd/egl_vendor.d/10_nvidia.json + containerPath: /usr/share/glvnd/egl_vendor.d/10_nvidia.json + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/xorg/libglxserver_nvidia.so.580.142 + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/xorg/libglxserver_nvidia.so.580.142 + options: + - ro + - nosuid + - nodev + - rbind + - rprivate + - hostPath: /usr/lib/aarch64-linux-gnu/nvidia/xorg/nvidia_drv.so + containerPath: /usr/lib/aarch64-linux-gnu/nvidia/xorg/nvidia_drv.so + options: + - ro + - nosuid + - nodev + - rbind + - rprivate From 05d78e94e934b173bda10d829b13514b02a9d64e Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 14 Oct 2026 21:36:28 +0200 Subject: [PATCH 3/6] feat(sandbox): enforce CDI-derived filesystem access When the boundary configuration carries a CDI context, resolve it at agent start and use the result instead of the hard-coded GPU baseline: - add derived read-only paths, and read-write access for device nodes and IPC directories, unless the authored policy already covers them; - reject an authored read-only entry for a device node or IPC directory instead of overriding it; - validate every CDI target through the descriptor that anchors its Landlock rule at each launch and exec, rejecting broad-root aliases, type changes and changed mount access; - require Landlock enforcement for CDI targets even when the policy is best effort. Missing CDI additionalGids are reported as a warning, because some runtimes do not apply them and devices that do not depend on them stay usable. A required root group remains fatal. Enrichment, failures and the effective Landlock mode are reported as OCSF events. Sandboxes without a CDI context keep the existing GPU baseline. Signed-off-by: Evan Lezar --- crates/openshell-sandbox/src/boundary_exec.rs | 102 +++- .../openshell-sandbox/src/boundary_server.rs | 346 +++++++++++- crates/openshell-sandbox/src/delegated.rs | 3 + crates/openshell-sandbox/src/launch.rs | 85 ++- crates/openshell-sandbox/src/main.rs | 6 +- crates/openshell-sandbox/src/process.rs | 22 +- .../src/sandbox/cdi_paths.rs | 492 ++++++++++++++++++ .../src/sandbox/linux/landlock.rs | 448 +++++++++++++++- .../src/sandbox/linux/mod.rs | 20 +- crates/openshell-sandbox/src/sandbox/mod.rs | 3 + 10 files changed, 1487 insertions(+), 40 deletions(-) create mode 100644 crates/openshell-sandbox/src/sandbox/cdi_paths.rs diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index c0a4930a35..0909c475b8 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -31,6 +31,8 @@ use openshell_isolation_interface::contract::{ #[derive(Clone)] pub struct LocalBoundaryExec { policy: SandboxPolicy, + #[cfg(target_os = "linux")] + launch_context: crate::launch::WorkloadLaunchContext, base_workdir: Option, ca_file_paths: Option>, provider_credentials: ProviderCredentialState, @@ -43,8 +45,10 @@ pub struct LocalBoundaryExec { impl LocalBoundaryExec { /// Construct the executor owned by an active sandbox boundary. #[must_use] + #[allow(clippy::too_many_arguments)] pub fn new( policy: SandboxPolicy, + #[cfg(target_os = "linux")] launch_context: crate::launch::WorkloadLaunchContext, base_workdir: Option, ca_file_paths: Option>, provider_credentials: ProviderCredentialState, @@ -54,6 +58,8 @@ impl LocalBoundaryExec { ) -> Self { Self { policy, + #[cfg(target_os = "linux")] + launch_context, base_workdir, ca_file_paths, provider_credentials, @@ -198,11 +204,16 @@ impl LocalBoundaryExec { &self, workdir: Option<&str>, ) -> Result, BackendError> { - crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); + crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir, &self.launch_context); let runtime_read_only = crate::process::ca_runtime_read_only_paths(self.ca_file_paths.as_deref()); - crate::process::prepare_child_sandbox(&self.policy, workdir, &runtime_read_only) - .map_err(|error| BackendError::Process(error.to_string())) + crate::process::prepare_child_sandbox( + &self.policy, + workdir, + &runtime_read_only, + &self.launch_context, + ) + .map_err(|error| BackendError::Process(error.to_string())) } fn spawn_piped(&self, spec: &ExecSpec) -> Result { @@ -706,6 +717,7 @@ mod tests { landlock: openshell_core::policy::LandlockPolicy::default(), process: openshell_core::policy::ProcessPolicy::default(), }, + crate::launch::WorkloadLaunchContext::default(), None, None, ProviderCredentialState::from_environment( @@ -720,6 +732,90 @@ mod tests { ) } + #[test] + fn cdi_launch_context_is_retained_and_revalidated_by_exec() { + if let Err(error) = crate::sandbox::linux::require_landlock_abi() { + assert!( + std::env::var_os("OPENSHELL_TEST_REQUIRE_LANDLOCK").is_none(), + "{error}" + ); + return; + } + let directory = tempfile::tempdir().unwrap(); + let library = directory.path().join("lib"); + let alias = directory.path().join("alias"); + std::fs::create_dir(&library).unwrap(); + std::os::unix::fs::symlink(&library, &alias).unwrap(); + let mut executor = executor(); + executor.policy.filesystem.read_only = vec![directory.path().to_path_buf()]; + executor.launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + read_only_paths: vec![alias.to_string_lossy().into_owned()], + ..Default::default() + }), + }; + crate::process::prepare_child_sandbox( + &executor.policy, + None, + &[], + &executor.launch_context, + ) + .expect("initial launch validates CDI targets"); + let retained = executor.clone(); + executor + .prepare_sandbox(None) + .expect("original executor retains valid CDI context"); + retained + .prepare_sandbox(None) + .expect("exec validates retained CDI targets"); + std::fs::remove_file(&alias).unwrap(); + std::os::unix::fs::symlink("/", &alias).unwrap(); + let Err(error) = retained.prepare_sandbox(None) else { + panic!("exec must reject a CDI target replaced by a broad root"); + }; + assert!(error.to_string().contains("broad root"), "{error}"); + } + + #[test] + fn missing_cdi_groups_do_not_block_initial_launch_or_exec() { + let mut executor = executor(); + executor.launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + additional_gids: vec![u32::MAX], + ..Default::default() + }), + }; + crate::process::prepare_child_sandbox( + &executor.policy, + None, + &[], + &executor.launch_context, + ) + .expect("initial launch only reports missing CDI groups"); + let retained = executor.clone(); + executor + .prepare_sandbox(None) + .expect("original executor only reports missing CDI groups"); + retained + .prepare_sandbox(None) + .expect("exec only reports missing CDI groups"); + } + + #[test] + fn root_cdi_group_is_rejected_for_initial_launch_and_exec() { + let mut executor = executor(); + executor.launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + additional_gids: vec![0], + ..Default::default() + }), + }; + let Err(error) = executor.prepare_sandbox(None) else { + panic!("exec must reject root GID 0 in CDI groups"); + }; + assert!(error.to_string().contains("root GID 0"), "{error}"); + } + #[test] fn shell_requests_are_resolved_by_the_workload_executor() { let executor = executor(); diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 5f5a6aa2a6..95885f94c9 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -27,6 +27,7 @@ mod linux { use crate::boundary_io::BoundaryRuntimeState; use crate::delegated::{AgentSignaler, spawn_workload}; + use crate::launch::{WorkloadLaunchContext, supplementary_groups_match}; use crate::main_session::{MainOutput, MainSession}; use crate::network_broker::NetworkBroker; use crate::process::ProcessStatus; @@ -112,6 +113,134 @@ mod linux { fn enrich_gpu_filesystem_paths( policy: &mut openshell_core::policy::SandboxPolicy, gpu_requested: bool, + cdi_context: Option<&openshell_core::cdi::CdiContext>, + ) -> Result<(bool, WorkloadLaunchContext), String> { + if let Some(context) = cdi_context { + if !gpu_requested { + return Err("CDI context was provided without a GPU resource claim".to_string()); + } + return enrich_cdi_filesystem_paths(policy, context); + } + Ok(( + enrich_legacy_gpu_filesystem_paths(policy, gpu_requested), + WorkloadLaunchContext::default(), + )) + } + + fn enrich_cdi_filesystem_paths( + policy: &mut openshell_core::policy::SandboxPolicy, + context: &openshell_core::cdi::CdiContext, + ) -> Result<(bool, WorkloadLaunchContext), String> { + let requirements = crate::cdi::resolve_cdi_context(context) + .map_err(|error| format!("resolve workload CDI requirements: {error}"))?; + crate::cdi::validate_cdi_requirements(&requirements) + .map_err(|error| format!("validate workload CDI requirements: {error}"))?; + + let modified = apply_cdi_requirements(policy, &requirements)?; + Ok(( + modified, + WorkloadLaunchContext { + cdi_requirements: Some(requirements), + }, + )) + } + + fn apply_cdi_requirements( + policy: &mut openshell_core::policy::SandboxPolicy, + requirements: &crate::launch::CdiDerivedRequirements, + ) -> Result { + // GPU device nodes and IPC directories need read-write access. Granting + // it would override an exact authored read-only entry, so reject the + // conflict instead. + let authored_read_only = |path: &&String| { + policy + .filesystem + .read_only + .iter() + .any(|allowed| allowed == Path::new(path.as_str())) + }; + if let Some(path) = requirements + .device_node_paths + .iter() + .find(authored_read_only) + { + return Err(format!( + "CDI device node '{path}' is listed in filesystem_policy.read_only; \ + GPU device nodes require read-write access, so remove it from read_only" + )); + } + if let Some(path) = requirements + .ipc_directory_paths + .iter() + .find(authored_read_only) + { + return Err(format!( + "CDI IPC directory '{path}' is listed in filesystem_policy.read_only; \ + GPU IPC directories require read-write access, so remove it from read_only" + )); + } + + let mut read_only = requirements + .read_only_paths + .iter() + .map(std::path::PathBuf::from) + .collect::>(); + read_only.sort_by(path_depth_order); + read_only.dedup(); + + let mut modified = false; + for path in read_only { + if path_is_covered_by_policy(policy, &path) { + continue; + } + policy.filesystem.read_only.push(path); + modified = true; + } + + let mut read_write = requirements + .device_node_paths + .iter() + .chain(&requirements.ipc_directory_paths) + .map(std::path::PathBuf::from) + .collect::>(); + read_write.sort_by(path_depth_order); + read_write.dedup(); + for path in read_write { + if path_is_covered(&path, &policy.filesystem.read_write) { + continue; + } + policy.filesystem.read_write.push(path); + modified = true; + } + + Ok(modified) + } + + fn path_depth_order( + left: &std::path::PathBuf, + right: &std::path::PathBuf, + ) -> std::cmp::Ordering { + left.components() + .count() + .cmp(&right.components().count()) + .then_with(|| left.cmp(right)) + } + + fn path_is_covered_by_policy( + policy: &openshell_core::policy::SandboxPolicy, + candidate: &Path, + ) -> bool { + path_is_covered(candidate, &policy.filesystem.read_only) + || path_is_covered(candidate, &policy.filesystem.read_write) + } + + fn path_is_covered(candidate: &Path, allowed: &[std::path::PathBuf]) -> bool { + allowed.iter().any(|path| candidate.starts_with(path)) + } + + fn enrich_legacy_gpu_filesystem_paths( + policy: &mut openshell_core::policy::SandboxPolicy, + gpu_requested: bool, ) -> bool { if !gpu_requested { return false; @@ -424,17 +553,6 @@ mod linux { .is_some_and(|value| value == "true") } - fn supplementary_groups_match(actual: &[u32], expected: &[u32], allow_extra: bool) -> bool { - if allow_extra { - // GPU runtimes may add host device-access groups while materializing - // an admitted GPU claim. They may extend, but never replace, the - // image-derived identity asserted by the compute driver. - expected.iter().all(|gid| actual.binary_search(gid).is_ok()) - } else { - actual == expected - } - } - #[allow(clippy::similar_names)] fn validate_running_identity( expected: &ResolvedWorkloadIdentity, @@ -2539,19 +2657,56 @@ mod linux { Ok(paths) => paths, Err(error) => return guest_error(BoundaryErrorKind::Process, error), }; - let mut policy = policy.into(); + let mut policy: openshell_core::policy::SandboxPolicy = policy.into(); let gpu_requested = self .config .resource_claims .get(GPU_RESOURCE_CLAIM) .is_some_and(|value| value == "true"); - if enrich_gpu_filesystem_paths(&mut policy, gpu_requested) { + let authored_filesystem = policy.filesystem.clone(); + let (enriched, launch_context) = match enrich_gpu_filesystem_paths( + &mut policy, + gpu_requested, + self.config.cdi_context.as_ref(), + ) { + Ok(enriched) => enriched, + Err(error) => { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Medium) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Other, "failed") + .message(format!("GPU filesystem enrichment failed: {error}")) + .build() + ); + return guest_error(BoundaryErrorKind::Process, error); + } + }; + if enriched { + let added = |authored: &[std::path::PathBuf], enriched: &[std::path::PathBuf]| { + enriched + .iter() + .filter(|path| !authored.contains(path)) + .map(|path| path.to_string_lossy().into_owned()) + .collect::>() + }; + let read_only = added(&authored_filesystem.read_only, &policy.filesystem.read_only); + let read_write = added( + &authored_filesystem.read_write, + &policy.filesystem.read_write, + ); openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Informational) .status(openshell_ocsf::StatusId::Success) .state(openshell_ocsf::StateId::Enabled, "enriched") - .message("Added workload-local GPU filesystem paths".to_string()) + .message(format!( + "Added workload-local GPU filesystem paths [ro:{} rw:{}]", + read_only.len(), + read_write.len() + )) + .unmapped("added_read_only", read_only) + .unmapped("added_read_write", read_write) .build() ); } @@ -2561,6 +2716,7 @@ mod linux { process_id: format!("{}:main:0", self.config.generation), spec, policy, + launch_context, provider_env_revision, provider_env, ca_file_paths, @@ -2936,6 +3092,7 @@ mod linux { process_id: String, spec: AgentSpecWire, policy: openshell_core::policy::SandboxPolicy, + launch_context: WorkloadLaunchContext, provider_env_revision: u64, provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, @@ -2970,6 +3127,7 @@ mod linux { process_id, spec, policy, + launch_context, provider_env_revision, provider_env, ca_file_paths, @@ -2990,6 +3148,7 @@ mod linux { spec.timeout_secs, spec.interactive, &policy, + &launch_context, entrypoint_pid, provider_credentials.clone(), provider_env, @@ -3791,6 +3950,162 @@ mod linux { )); } + fn cdi_test_policy(read_only: &[&str]) -> openshell_core::policy::SandboxPolicy { + openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy { + read_only: read_only.iter().map(std::path::PathBuf::from).collect(), + read_write: Vec::new(), + include_workdir: false, + }, + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + } + } + + fn cdi_test_requirements() -> crate::launch::CdiDerivedRequirements { + crate::launch::CdiDerivedRequirements { + device_node_paths: vec!["/dev/null".into()], + read_only_paths: vec!["/usr/lib".into(), "/etc/hosts".into()], + additional_gids: vec![44], + ..Default::default() + } + } + + #[test] + fn workload_cdi_enrichment_respects_authored_ancestors() { + let requirements = cdi_test_requirements(); + let mut policy = cdi_test_policy(&["/usr", "/proc"]); + + assert!(apply_cdi_requirements(&mut policy, &requirements).unwrap()); + + assert!(policy.filesystem.read_only.contains(&"/proc".into())); + assert!(policy.filesystem.read_only.contains(&"/usr".into())); + assert!(!policy.filesystem.read_only.contains(&"/usr/lib".into())); + assert!(policy.filesystem.read_only.contains(&"/etc/hosts".into())); + assert!(!policy.filesystem.read_only.contains(&"/sys".into())); + assert!(policy.filesystem.read_write.contains(&"/dev/null".into())); + assert!(!policy.filesystem.read_write.contains(&"/proc".into())); + + // Covered paths still carry descriptor validation into later execs. + let context = WorkloadLaunchContext { + cdi_requirements: Some(requirements), + }; + let requirements = context.cdi_requirements.unwrap(); + assert_eq!(requirements.additional_gids, vec![44]); + assert!( + requirements + .read_only_paths + .contains(&"/usr/lib".to_string()) + ); + assert_eq!(requirements.device_node_paths, vec!["/dev/null"]); + } + + #[test] + fn cdi_groups_require_runtime_access() { + assert!(crate::launch::required_cdi_groups_present( + &[44, 107], + 44, + &[107] + )); + assert!(!crate::launch::required_cdi_groups_present( + &[44, 107], + 44, + &[] + )); + assert!(!crate::launch::required_cdi_groups_present( + &[0], + 1000, + &[0] + )); + } + + #[test] + fn workload_cdi_enrichment_rejects_read_only_device_nodes() { + let requirements = cdi_test_requirements(); + let mut policy = cdi_test_policy(&["/usr", "/dev/null"]); + + let error = apply_cdi_requirements(&mut policy, &requirements).unwrap_err(); + assert!(error.contains("/dev/null"), "{error}"); + assert!(error.contains("read_only"), "{error}"); + assert!(policy.filesystem.read_write.is_empty()); + } + + #[test] + fn workload_cdi_enrichment_accepts_read_only_device_ancestors() { + let requirements = cdi_test_requirements(); + let mut policy = cdi_test_policy(&["/usr", "/dev"]); + + assert!(apply_cdi_requirements(&mut policy, &requirements).unwrap()); + assert!(policy.filesystem.read_write.contains(&"/dev/null".into())); + } + + #[test] + fn workload_cdi_enrichment_grants_ipc_directories_read_write() { + let requirements = crate::launch::CdiDerivedRequirements { + ipc_directory_paths: vec!["/tmp/nvidia-mps".into()], + ..Default::default() + }; + let mut policy = cdi_test_policy(&["/tmp"]); + assert!(apply_cdi_requirements(&mut policy, &requirements).unwrap()); + assert!( + policy + .filesystem + .read_write + .contains(&"/tmp/nvidia-mps".into()) + ); + + // An authored read-write ancestor already covers the directory. + let mut policy = cdi_test_policy(&[]); + policy.filesystem.read_write.push("/tmp".into()); + assert!(!apply_cdi_requirements(&mut policy, &requirements).unwrap()); + + let mut policy = cdi_test_policy(&["/tmp/nvidia-mps"]); + let error = apply_cdi_requirements(&mut policy, &requirements).unwrap_err(); + assert!( + error.contains("CDI IPC directory '/tmp/nvidia-mps'"), + "{error}" + ); + } + + #[test] + fn cdi_group_warnings_name_missing_groups() { + assert_eq!( + crate::launch::missing_cdi_groups(&[44, 104, 995], 1000, &[104]), + vec![44, 995] + ); + assert_eq!( + crate::launch::missing_cdi_groups(&[44, 107], 44, &[107]), + Vec::::new() + ); + assert_eq!(crate::launch::missing_cdi_groups(&[0], 0, &[0]), vec![0]); + } + + #[test] + fn workload_cdi_enrichment_is_idempotent() { + let requirements = cdi_test_requirements(); + let mut policy = cdi_test_policy(&["/usr", "/proc"]); + + assert!(apply_cdi_requirements(&mut policy, &requirements).unwrap()); + assert!(!apply_cdi_requirements(&mut policy, &requirements).unwrap()); + } + + #[test] + fn workload_cdi_enrichment_adds_no_ordinary_mount_write_grants() { + let mut policy = cdi_test_policy(&[]); + policy.filesystem.read_write = vec!["/workspace".into()]; + let requirements = crate::launch::CdiDerivedRequirements { + read_write_mount_paths: vec!["/workspace/cache".into(), "/elsewhere/file".into()], + ..Default::default() + }; + assert!(!apply_cdi_requirements(&mut policy, &requirements).unwrap()); + assert_eq!( + policy.filesystem.read_write, + vec![std::path::PathBuf::from("/workspace")] + ); + } + #[test] fn exec_tombstones_expire_without_a_lifetime_limit() { let mut ledger = ExecRequestLedger::default(); @@ -4999,8 +5314,8 @@ mod linux { tls: server_tls.clone(), }, resource_claims: std::collections::BTreeMap::new(), - resource_claim_files: std::collections::BTreeMap::new(), cdi_context: None, + resource_claim_files: std::collections::BTreeMap::new(), workload_identity: test_workload_identity(), outer_fence: test_outer_fence(), child_env: std::collections::HashMap::new(), @@ -5703,6 +6018,7 @@ mod linux { process_id: "generation-retained:main:0".to_string(), spec: agent_spec.clone(), policy, + launch_context: WorkloadLaunchContext::default(), provider_env_revision: 0, provider_env: std::collections::HashMap::new(), ca_file_paths: None, diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index b0282e1016..78f0168352 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -35,6 +35,7 @@ pub async fn spawn_workload( timeout_secs: u64, interactive: bool, policy: &SandboxPolicy, + launch_context: &crate::launch::WorkloadLaunchContext, entrypoint_pid: Arc, provider_credentials: ProviderCredentialState, provider_env: std::collections::HashMap, @@ -77,6 +78,7 @@ pub async fn spawn_workload( let boundary_exec: Arc = Arc::new(crate::boundary_exec::LocalBoundaryExec::new( policy.clone(), + launch_context.clone(), workspace.owned_root(), ca_file_paths.clone().map(Arc::new), provider_credentials, @@ -93,6 +95,7 @@ pub async fn spawn_workload( &workspace, interactive, policy, + launch_context, ca_file_paths.as_ref(), &provider_env, ) diff --git a/crates/openshell-sandbox/src/launch.rs b/crates/openshell-sandbox/src/launch.rs index bddf14bc7a..fa5f3d4b39 100644 --- a/crates/openshell-sandbox/src/launch.rs +++ b/crates/openshell-sandbox/src/launch.rs @@ -1,7 +1,90 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Workload-local requirements derived for launch-time enforcement. +//! Workload-local requirements retained for initial launch and later execs. + +/// Runtime-derived inputs kept separate from the authored sandbox policy. +#[derive(Debug, Clone, Default)] +pub struct WorkloadLaunchContext { + #[cfg(target_os = "linux")] + pub(crate) cdi_requirements: Option, +} + +#[cfg(target_os = "linux")] +impl WorkloadLaunchContext { + /// Check that the runtime applied the CDI `additionalGids`. Older + /// runtimes ignore them, so missing groups are reported rather than + /// fatal: devices that do not depend on them stay usable. + pub(crate) fn validate_running_groups(&self) -> miette::Result<()> { + use miette::IntoDiagnostic as _; + let Some(requirements) = &self.cdi_requirements else { + return Ok(()); + }; + if requirements.additional_gids.is_empty() { + return Ok(()); + } + if requirements.additional_gids.contains(&0) { + return Err(miette::miette!( + "CDI additionalGids must not contain root GID 0" + )); + } + let actual = nix::unistd::getgroups().into_diagnostic()?; + let actual = actual.iter().map(|gid| gid.as_raw()).collect::>(); + let primary = nix::unistd::getegid().as_raw(); + if !required_cdi_groups_present(&requirements.additional_gids, primary, &actual) { + let missing = missing_cdi_groups(&requirements.additional_gids, primary, &actual); + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Low) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Other, "degraded") + .message(format!( + "CDI groups not applied by workload runtime [missing:{missing:?} \ + required:{:?}]; GPU devices restricted to these groups may be \ + inaccessible", + requirements.additional_gids + )) + .build() + ); + } + Ok(()) + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn required_cdi_groups_present(required: &[u32], primary: u32, actual: &[u32]) -> bool { + if required.contains(&0) { + return false; + } + let mut available = actual.to_vec(); + available.push(primary); + available.sort_unstable(); + supplementary_groups_match(&available, required, true) +} + +#[cfg(target_os = "linux")] +pub(crate) fn missing_cdi_groups(required: &[u32], primary: u32, actual: &[u32]) -> Vec { + required + .iter() + .copied() + .filter(|gid| *gid == 0 || (*gid != primary && !actual.contains(gid))) + .collect() +} + +/// Match the sorted runtime groups against required identity groups. Runtime +/// GPU groups may extend, but never replace, the image-derived identity. +#[cfg(target_os = "linux")] +pub(crate) fn supplementary_groups_match( + actual: &[u32], + expected: &[u32], + allow_extra: bool, +) -> bool { + if allow_extra { + expected.iter().all(|gid| actual.binary_search(gid).is_ok()) + } else { + actual == expected + } +} #[cfg(target_os = "linux")] #[derive(Debug, Clone, Default, PartialEq, Eq)] diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 86ef2bb4f5..0edf4033eb 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -315,7 +315,11 @@ fn run_capability_landlock_child(args: &[String]) -> Result<()> { }, process: ProcessPolicy::default(), }; - let prepared = openshell_sandbox::sandbox::linux::prepare_capability_free(&policy, None)?; + let prepared = openshell_sandbox::sandbox::linux::prepare_capability_free( + &policy, + None, + &openshell_sandbox::launch::WorkloadLaunchContext::default(), + )?; openshell_sandbox::sandbox::linux::enforce(prepared)?; if std::fs::read(allowed.join("sentinel")).into_diagnostic()? != b"allowed" { return Err(miette::miette!("Landlock probe allowed-path mismatch")); diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 1b0f853d23..185a05fd4b 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -71,9 +71,12 @@ pub(crate) fn prepare_child_sandbox( policy: &SandboxPolicy, workdir: Option<&str>, runtime_read_only: &[PathBuf], + launch_context: &crate::launch::WorkloadLaunchContext, ) -> Result> { + launch_context.validate_running_groups()?; let effective_policy = policy_with_runtime_read_only(policy, runtime_read_only); - let prepared = sandbox::linux::prepare_capability_free(&effective_policy, workdir)?; + let prepared = + sandbox::linux::prepare_capability_free(&effective_policy, workdir, launch_context)?; Ok(Some(prepared)) } @@ -427,6 +430,7 @@ impl ProcessHandle { workspace: &ResolvedWorkspace, interactive: bool, policy: &SandboxPolicy, + launch_context: &crate::launch::WorkloadLaunchContext, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -437,6 +441,7 @@ impl ProcessHandle { workspace, interactive, policy, + launch_context, ca_paths, provider_env, ) @@ -478,6 +483,7 @@ impl ProcessHandle { workspace: &ResolvedWorkspace, interactive: bool, policy: &SandboxPolicy, + launch_context: &crate::launch::WorkloadLaunchContext, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -539,14 +545,15 @@ impl ProcessHandle { // process where the tracing subscriber is functional. The child's // pre_exec context cannot reliably emit structured logs. #[cfg(target_os = "linux")] - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + sandbox::linux::log_sandbox_readiness(policy, workspace.root(), launch_context); // Prepare the Landlock ruleset as the workload UID. Inaccessible paths // are already unavailable to the child and remain omitted. #[cfg(target_os = "linux")] let runtime_read_only = ca_runtime_read_only_paths(ca_paths); - let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), &runtime_read_only) - .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = + prepare_child_sandbox(policy, workspace.root(), &runtime_read_only, launch_context) + .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; #[cfg(target_os = "linux")] let mut child_hardening = crate::linux::child_seccomp::prepare(std::process::id()) .map_err(|error| miette::miette!("prepare child self-protection filter: {error}"))?; @@ -1566,7 +1573,12 @@ mod tests { }; let runtime_paths = ca_runtime_read_only_paths(Some(&(certificate.clone(), bundle.clone()))); - let Ok(Some(prepared)) = prepare_child_sandbox(&policy, None, &runtime_paths) else { + let Ok(Some(prepared)) = prepare_child_sandbox( + &policy, + None, + &runtime_paths, + &crate::launch::WorkloadLaunchContext::default(), + ) else { return; }; diff --git a/crates/openshell-sandbox/src/sandbox/cdi_paths.rs b/crates/openshell-sandbox/src/sandbox/cdi_paths.rs new file mode 100644 index 0000000000..555855fec1 --- /dev/null +++ b/crates/openshell-sandbox/src/sandbox/cdi_paths.rs @@ -0,0 +1,492 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validate CDI objects using the descriptors that will anchor Landlock rules. + +use std::collections::BTreeMap; +use std::os::fd::AsFd; +use std::path::{Path, PathBuf}; + +use crate::launch::CdiDerivedRequirements; +use miette::{IntoDiagnostic, Result}; +use rustix::fs::{FileType, fstat}; + +/// Deduplicated CDI paths and their combined validation requirements. +/// Stores no descriptors; target handles live only through validation and use. +#[derive(Default)] +pub(super) struct CdiPaths { + paths: BTreeMap, +} + +// Each flag is an independent constraint; duplicate CDI uses of one path must +// all hold, so they are not collapsed into a single kind. +#[allow(clippy::struct_excessive_bools)] +#[derive(Default)] +struct TargetRequirements { + device: bool, + socket: bool, + ipc_directory: bool, + writable_mount: bool, + mount_read_only: Option, +} + +impl CdiPaths { + pub(super) fn new(requirements: &CdiDerivedRequirements) -> Self { + let mut paths = BTreeMap::::new(); + for path in &requirements.read_only_paths { + paths.entry(path.into()).or_default(); + } + for path in &requirements.socket_paths { + paths.entry(path.into()).or_default().socket = true; + } + for path in &requirements.device_node_paths { + paths.entry(path.into()).or_default().device = true; + } + for path in &requirements.ipc_directory_paths { + paths.entry(path.into()).or_default().ipc_directory = true; + } + for path in &requirements.read_write_mount_paths { + paths.entry(path.into()).or_default().writable_mount = true; + } + for mount in &requirements.mounts { + paths + .entry((&mount.path).into()) + .or_default() + .mount_read_only = Some(mount.read_only); + } + Self { paths } + } + + pub(super) fn contains(&self, path: &Path) -> bool { + self.paths.contains_key(path) + } + + pub(super) fn is_empty(&self) -> bool { + self.paths.is_empty() + } + + /// Open, validate and consume one target at a time, including targets + /// covered by authored ancestors. `apply` must install any matching rules + /// using the supplied descriptor before this function closes it. + /// Sockets are validated without inserting ordinary filesystem rules. + /// Symlinks remain supported: validation applies to the opened object, + /// without a canonicalize/reopen race. + pub(super) fn process_validated( + &self, + mut state: S, + mut open: impl FnMut(&Path) -> Result, + mut apply: impl FnMut(S, &Path, &F) -> Result, + ) -> Result { + if self.is_empty() { + return Ok(state); + } + // Retain only root identities. Each root descriptor closes immediately. + let mut broad_roots = Vec::new(); + for root in crate::cdi::FORBIDDEN_CDI_ROOTS { + match open(Path::new(root)) { + Ok(fd) => broad_roots.push(fstat(&fd).into_diagnostic()?), + Err(_) if !Path::new(root).exists() => {} + Err(error) => return Err(error), + } + } + #[cfg(target_os = "linux")] + let mount_table = self + .paths + .values() + .any(|target| target.mount_read_only.is_some()) + .then(crate::cdi::CdiMountTable::read) + .transpose() + .into_diagnostic()?; + for (path, requirements) in &self.paths { + let fd = open(path)?; + let stat = fstat(&fd).into_diagnostic()?; + if broad_roots + .iter() + .any(|root| root.st_dev == stat.st_dev && root.st_ino == stat.st_ino) + { + return Err(miette::miette!( + "CDI path '{}' resolves to a forbidden broad root", + path.display() + )); + } + let kind = FileType::from_raw_mode(stat.st_mode); + if requirements.socket && kind != FileType::Socket { + return Err(miette::miette!( + "CDI socket '{}' is not a Unix socket", + path.display() + )); + } + if requirements.device + && !matches!(kind, FileType::CharacterDevice | FileType::BlockDevice) + { + return Err(miette::miette!( + "CDI device '{}' is not a character or block device", + path.display() + )); + } + if requirements.ipc_directory { + validate_ipc_directory(path, &fd, kind)?; + } + if requirements.writable_mount || requirements.mount_read_only == Some(false) { + return Err(miette::miette!( + "CDI writable mount '{}' is unsupported: ordinary host-backed file and directory mounts must be read-only", + path.display() + )); + } + #[cfg(target_os = "linux")] + if let Some(expected) = requirements.mount_read_only { + let actual = mount_table + .as_ref() + .expect("mount table loaded") + .read_only(&path.to_string_lossy(), &fd) + .into_diagnostic()?; + if actual != expected { + return Err(miette::miette!( + "CDI mount '{}' changed effective access before enforcement", + path.display() + )); + } + } + if !requirements.socket { + state = apply(state, path, &fd)?; + } + // The descriptor closes before the next target is opened. Landlock + // retains its object reference after successful rule insertion. + } + Ok(state) + } +} + +/// IPC directories are writable host mounts, so require the kernel to report +/// the same restrictive flags the spec requested on the opened directory. +fn validate_ipc_directory(path: &Path, fd: &F, kind: FileType) -> Result<()> { + if kind != FileType::Directory { + return Err(miette::miette!( + "CDI IPC directory '{}' is not a directory", + path.display() + )); + } + #[cfg(target_os = "linux")] + { + use rustix::fs::{StatVfsMountFlags, fstatvfs}; + let required = + StatVfsMountFlags::NOSUID | StatVfsMountFlags::NODEV | StatVfsMountFlags::NOEXEC; + let flags = fstatvfs(fd).into_diagnostic()?.f_flag; + if !flags.contains(required) { + return Err(miette::miette!( + "CDI IPC directory '{}' must be mounted with nosuid, nodev and noexec", + path.display() + )); + } + } + #[cfg(not(target_os = "linux"))] + let _ = fd; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use std::os::unix::fs::symlink; + + fn open(path: &Path) -> Result { + File::open(path).into_diagnostic() + } + + fn validate( + requirements: &CdiDerivedRequirements, + open: impl FnMut(&Path) -> Result, + ) -> Result<()> { + CdiPaths::new(requirements).process_validated((), open, |(), _, _| Ok(())) + } + + fn read_only(path: &Path) -> CdiDerivedRequirements { + CdiDerivedRequirements { + read_only_paths: vec![path.to_string_lossy().into_owned()], + ..Default::default() + } + } + + struct TrackedFd { + file: File, + live: std::rc::Rc>, + } + + impl AsFd for TrackedFd { + fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> { + self.file.as_fd() + } + } + + impl Drop for TrackedFd { + fn drop(&mut self) { + self.live.set(self.live.get() - 1); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn socket_targets_are_exact_and_revalidated_without_filesystem_rules() { + let temp = tempfile::tempdir().unwrap(); + let target = temp.path().join("socket"); + let _listener = std::os::unix::net::UnixListener::bind(&target).unwrap(); + let requirements = CdiDerivedRequirements { + socket_paths: vec![target.to_string_lossy().into_owned()], + ..Default::default() + }; + let paths = CdiPaths::new(&requirements); + assert!(paths.contains(&target)); + assert!(!paths.contains(temp.path())); + let open_socket = |path: &Path| { + rustix::fs::open( + path, + rustix::fs::OFlags::PATH | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .into_diagnostic() + }; + paths + .process_validated((), open_socket, |(), _, _| { + panic!("socket must not add a filesystem rule") + }) + .unwrap(); + std::fs::remove_file(&target).unwrap(); + std::fs::write(&target, "replacement").unwrap(); + assert!( + paths + .process_validated((), open_socket, |(), _, _| Ok(())) + .unwrap_err() + .to_string() + .contains("not a Unix socket") + ); + } + + #[test] + fn deduplicates_paths_and_keeps_one_descriptor_live() { + let temp = tempfile::tempdir().unwrap(); + let mut requirements = CdiDerivedRequirements::default(); + for index in 0..512 { + let path = temp.path().join(format!("file-{index}")); + std::fs::write(&path, "").unwrap(); + let path = path.to_string_lossy().into_owned(); + requirements.read_only_paths.extend([path.clone(), path]); + } + let paths = CdiPaths::new(&requirements); + assert!(paths.contains(&temp.path().join("file-0"))); + let live = std::rc::Rc::new(std::cell::Cell::new(0)); + let processed = paths + .process_validated( + 0, + |path| { + // No earlier target (or root) descriptor may survive this open. + assert_eq!(live.get(), 0); + let file = open(path)?; + live.set(live.get() + 1); + Ok(TrackedFd { + file, + live: std::rc::Rc::clone(&live), + }) + }, + |count, _, fd| { + assert_eq!(live.get(), 1); + assert_eq!( + FileType::from_raw_mode(fstat(fd).unwrap().st_mode), + FileType::RegularFile + ); + Ok(count + 1) + }, + ) + .unwrap(); + assert_eq!(processed, 512); + assert_eq!(live.get(), 0); + } + + #[cfg(target_os = "linux")] + #[test] + fn rechecks_effective_mount_access_on_the_consumed_descriptor() { + let temp = tempfile::tempdir().unwrap(); + let target = temp.path().join("target"); + std::fs::write(&target, "").unwrap(); + let requirements = CdiDerivedRequirements { + mounts: vec![crate::launch::CdiMountRequirement { + path: target.to_string_lossy().into_owned(), + read_only: true, + }], + ..Default::default() + }; + let error = validate(&requirements, open).unwrap_err(); + assert!(error.to_string().contains("changed effective access")); + } + + #[test] + fn retains_all_constraints_for_duplicate_paths() { + let requirements = CdiDerivedRequirements { + read_only_paths: vec!["/dev/null".into()], + device_node_paths: vec!["/dev/null".into()], + read_write_mount_paths: vec!["/dev/null".into()], + ..Default::default() + }; + // Valid as a device but invalid as an ordinary writable mount. + let error = validate(&requirements, open).unwrap_err(); + assert!(error.to_string().contains("CDI writable mount")); + } + + #[test] + fn closes_descriptor_when_consumer_fails() { + let temp = tempfile::tempdir().unwrap(); + let live = std::rc::Rc::new(std::cell::Cell::new(0)); + let paths = CdiPaths::new(&read_only(temp.path())); + let result = paths.process_validated( + (), + |path| { + assert_eq!(live.get(), 0); + let file = open(path)?; + live.set(live.get() + 1); + Ok(TrackedFd { + file, + live: std::rc::Rc::clone(&live), + }) + }, + |(), _, _| Err(miette::miette!("rule insertion failed")), + ); + assert!(result.is_err()); + assert_eq!(live.get(), 0); + } + + #[test] + fn rejects_symlink_aliases_to_broad_roots() { + let temp = tempfile::tempdir().unwrap(); + let alias = temp.path().join("library-parent"); + for root in crate::cdi::FORBIDDEN_CDI_ROOTS { + if !Path::new(root).exists() { + continue; + } + symlink(root, &alias).unwrap(); + assert!( + validate(&read_only(&alias), open) + .unwrap_err() + .to_string() + .contains("broad root") + ); + std::fs::remove_file(&alias).unwrap(); + } + } + + #[test] + fn allows_library_directory_symlinks() { + let temp = tempfile::tempdir().unwrap(); + let library = temp.path().join("lib"); + std::fs::create_dir(&library).unwrap(); + let alias = temp.path().join("alias"); + symlink(&library, &alias).unwrap(); + assert!(validate(&read_only(&alias), open).is_ok()); + } + + #[test] + fn validates_types_on_opened_objects() { + let temp = tempfile::tempdir().unwrap(); + let file = temp.path().join("fake-device"); + std::fs::write(&file, "").unwrap(); + let requirements = CdiDerivedRequirements { + device_node_paths: vec![file.to_string_lossy().into_owned()], + ..Default::default() + }; + assert!(validate(&requirements, open).is_err()); + let requirements = CdiDerivedRequirements { + read_write_mount_paths: vec![temp.path().to_string_lossy().into_owned()], + ..Default::default() + }; + assert!(validate(&requirements, open).is_err()); + let requirements = CdiDerivedRequirements { + read_write_mount_paths: vec![file.to_string_lossy().into_owned()], + ..Default::default() + }; + assert!(validate(&requirements, open).is_err()); + // Read-only targets still have to exist when enforcement opens them. + let missing = temp.path().join("missing"); + assert!(validate(&read_only(&missing), open).is_err()); + let requirements = CdiDerivedRequirements { + device_node_paths: vec!["/dev/null".into()], + ..Default::default() + }; + assert!(validate(&requirements, open).is_ok()); + } + + fn ipc_directory(path: &Path) -> CdiDerivedRequirements { + CdiDerivedRequirements { + ipc_directory_paths: vec![path.to_string_lossy().into_owned()], + ..Default::default() + } + } + + #[test] + fn ipc_directories_must_be_directories() { + let temp = tempfile::tempdir().unwrap(); + let file = temp.path().join("nvidia-mps"); + std::fs::write(&file, "").unwrap(); + let error = validate(&ipc_directory(&file), open).unwrap_err(); + assert!(error.to_string().contains("not a directory"), "{error}"); + } + + #[cfg(target_os = "linux")] + #[test] + fn ipc_directories_require_effective_nosuid_nodev_noexec() { + use rustix::fs::{StatVfsMountFlags, statvfs}; + let required = + StatVfsMountFlags::NOSUID | StatVfsMountFlags::NODEV | StatVfsMountFlags::NOEXEC; + let temp = tempfile::tempdir().unwrap(); + let result = validate(&ipc_directory(temp.path()), open); + if statvfs(temp.path()).unwrap().f_flag.contains(required) { + result.unwrap(); + } else { + let error = result.unwrap_err(); + assert!( + error.to_string().contains("nosuid, nodev and noexec"), + "{error}" + ); + } + // Container /dev/shm is normally mounted nosuid,nodev,noexec. + let shm = Path::new("/dev/shm"); + if shm.is_dir() && statvfs(shm).unwrap().f_flag.contains(required) { + let pipes = tempfile::tempdir_in(shm).unwrap(); + validate(&ipc_directory(pipes.path()), open).unwrap(); + } + } + + #[test] + fn pins_target_across_swap_and_revalidates_next_launch() { + let temp = tempfile::tempdir().unwrap(); + let library = temp.path().join("lib"); + std::fs::create_dir(&library).unwrap(); + let alias = temp.path().join("alias"); + symlink(&library, &alias).unwrap(); + let requirements = read_only(&alias); + let expected = fstat(File::open(&library).unwrap()).unwrap(); + let pinned = CdiPaths::new(&requirements) + .process_validated( + None, + |path| { + let fd = open(path)?; + if path == alias { + std::fs::remove_file(&alias).unwrap(); + symlink("/", &alias).unwrap(); + } + Ok(fd) + }, + |_, path, fd| { + assert_eq!(path, alias); + Ok(Some(fstat(fd).unwrap())) + }, + ) + .unwrap() + .unwrap(); + assert_eq!( + (pinned.st_dev, pinned.st_ino), + (expected.st_dev, expected.st_ino) + ); + // Exec clones preserve provenance and prepare fresh rules, so the new + // alias must be rejected rather than using earlier pathname validation. + assert!(validate(&requirements, open).is_err()); + } +} diff --git a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs index 27bca169b2..ea131f8750 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs @@ -112,7 +112,7 @@ enum PathOpenMode { /// Returns `None` if there are no filesystem paths to restrict (no-op). /// Returns `Some(PreparedRuleset)` on success, or an error. pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result> { - prepare_with_path_open_mode(policy, workdir, PathOpenMode::Privileged) + prepare_with_path_open_mode(policy, workdir, PathOpenMode::Privileged, None) } /// Phase 1 for already-unprivileged workloads. @@ -124,8 +124,14 @@ pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result, + launch_context: &crate::launch::WorkloadLaunchContext, ) -> Result> { - prepare_with_path_open_mode(policy, workdir, PathOpenMode::CurrentUser) + prepare_with_path_open_mode( + policy, + workdir, + PathOpenMode::CurrentUser, + launch_context.cdi_requirements.as_ref(), + ) } /// Build the mandatory same-UID self-protection baseline. @@ -236,10 +242,36 @@ fn capability_free_baseline_entries( })) } +/// CDI targets require enforcement, so they override a best-effort policy. +fn effective_compatibility<'a>( + policy: &'a SandboxPolicy, + cdi_paths: &super::super::cdi_paths::CdiPaths, +) -> &'a LandlockCompatibility { + if cdi_paths.is_empty() { + &policy.landlock.compatibility + } else { + &LandlockCompatibility::HardRequirement + } +} + +/// Effective compatibility for a workload launch, for logging before preparation. +pub(super) fn launch_compatibility<'a>( + policy: &'a SandboxPolicy, + launch_context: &crate::launch::WorkloadLaunchContext, +) -> &'a LandlockCompatibility { + let cdi_paths = launch_context + .cdi_requirements + .as_ref() + .map(super::super::cdi_paths::CdiPaths::new) + .unwrap_or_default(); + effective_compatibility(policy, &cdi_paths) +} + fn prepare_with_path_open_mode( policy: &SandboxPolicy, workdir: Option<&str>, path_open_mode: PathOpenMode, + cdi_requirements: Option<&crate::launch::CdiDerivedRequirements>, ) -> Result> { let read_only = policy.filesystem.read_only.clone(); let mut read_write = policy.filesystem.read_write.clone(); @@ -253,11 +285,20 @@ fn prepare_with_path_open_mode( } } - if read_only.is_empty() && read_write.is_empty() { + // Keep only path requirements; CDI descriptors are opened and consumed + // sequentially when building rules, including validation-only targets. + let cdi_paths = cdi_requirements + .map(super::super::cdi_paths::CdiPaths::new) + .unwrap_or_default(); + + if read_only.is_empty() && read_write.is_empty() && cdi_paths.is_empty() { return Ok(None); } - let compatibility = &policy.landlock.compatibility; + // CDI target preparation and enforcement cannot fall back to an + // unrestricted launch. This does not change authored path opening: + // missing or inaccessible authored paths retain the policy's mode. + let compatibility = effective_compatibility(policy, &cdi_paths); // Probe first: kernels without Landlock (e.g. gVisor's sentry returns // ENOSYS) would otherwise log misleading "Applying"+"Built" events. @@ -308,7 +349,7 @@ fn prepare_with_path_open_mode( .state(openshell_ocsf::StateId::Enabled, "applying") .message(format!( "Applying Landlock filesystem sandbox [abi:{abi:?} compat:{:?} ro:{} rw:{}]", - policy.landlock.compatibility, + compatibility, read_only.len(), read_write.len(), )) @@ -325,11 +366,50 @@ fn prepare_with_path_open_mode( .handle_access(access_all) .into_diagnostic()?; - let mut ruleset = ruleset.create().into_diagnostic()?; - let mut rules_applied: usize = 0; + let ruleset = ruleset.create().into_diagnostic()?; + let (mut ruleset, mut rules_applied) = cdi_paths.process_validated( + (ruleset, 0_usize), + // Use a direct O_PATH open for CDI objects. PathFd uses std + // OpenOptions, whose musl opener cannot open Unix socket inodes. + |path| { + rustix::fs::open( + path, + rustix::fs::OFlags::PATH | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .into_diagnostic() + }, + |(ruleset, count), path, fd| { + // A covered CDI target still needs validation, but no extra + // rule. Match exact policy entries without reopening the path. + let access = if read_write.iter().any(|allowed| allowed == path) { + Some(access_all) + } else if read_only.iter().any(|allowed| allowed == path) { + Some(access_read) + } else { + None + }; + let Some(access) = access else { + return Ok((ruleset, count)); + }; + let allowed_access = access_for_path_fd(fd, access, abi)?; + debug!(path = %path.display(), "Landlock allow validated CDI target"); + Ok(( + ruleset + .add_rule(PathBeneath::new(fd, allowed_access)) + .into_diagnostic()?, + count + 1, + )) + }, + )?; for path in &read_only { - if let Some(path_fd) = try_open_path(path, compatibility, path_open_mode)? { + if cdi_paths.contains(path) { + continue; + } + if let Some(path_fd) = + try_open_path(path, &policy.landlock.compatibility, path_open_mode)? + { let allowed_access = access_for_path_fd(&path_fd, access_read, abi)?; debug!(path = %path.display(), "Landlock allow read-only"); ruleset = ruleset @@ -340,7 +420,12 @@ fn prepare_with_path_open_mode( } for path in &read_write { - if let Some(path_fd) = try_open_path(path, compatibility, path_open_mode)? { + if cdi_paths.contains(path) { + continue; + } + if let Some(path_fd) = + try_open_path(path, &policy.landlock.compatibility, path_open_mode)? + { let allowed_access = access_for_path_fd(&path_fd, access_all, abi)?; debug!(path = %path.display(), "Landlock allow read-write"); ruleset = ruleset @@ -608,6 +693,324 @@ mod tests { } } + #[test] + fn cdi_targets_make_the_logged_compatibility_a_hard_requirement() { + let mut policy = hard_requirement_policy(vec![], vec![]); + policy.landlock.compatibility = LandlockCompatibility::BestEffort; + let without_cdi = crate::launch::WorkloadLaunchContext::default(); + assert!(matches!( + launch_compatibility(&policy, &without_cdi), + LandlockCompatibility::BestEffort + )); + let empty_cdi = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements::default()), + }; + assert!(matches!( + launch_compatibility(&policy, &empty_cdi), + LandlockCompatibility::BestEffort + )); + let with_cdi = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + device_node_paths: vec!["/dev/null".into()], + ..Default::default() + }), + }; + assert!(matches!( + launch_compatibility(&policy, &with_cdi), + LandlockCompatibility::HardRequirement + )); + } + + #[test] + fn cdi_preserves_authored_path_compatibility() { + if let Err(error) = require_abi() { + assert!( + std::env::var_os("OPENSHELL_TEST_REQUIRE_LANDLOCK").is_none(), + "{error}" + ); + return; + } + let directory = tempfile::tempdir().unwrap(); + let library = directory.path().join("lib"); + std::fs::create_dir(&library).unwrap(); + let missing = directory.path().join("absent-from-image"); + let requirements = crate::launch::CdiDerivedRequirements { + read_only_paths: vec![library.to_string_lossy().into_owned()], + ..Default::default() + }; + let mut policy = hard_requirement_policy(vec![library.clone(), missing], vec![]); + policy.landlock.compatibility = LandlockCompatibility::BestEffort; + let prepared = prepare_with_path_open_mode( + &policy, + None, + PathOpenMode::Privileged, + Some(&requirements), + ) + .unwrap() + .unwrap(); + assert!(matches!( + prepared.compatibility, + LandlockCompatibility::HardRequirement + )); + policy.landlock.compatibility = LandlockCompatibility::HardRequirement; + assert!( + prepare_with_path_open_mode( + &policy, + None, + PathOpenMode::Privileged, + Some(&requirements), + ) + .is_err() + ); + policy.landlock.compatibility = LandlockCompatibility::BestEffort; + std::fs::remove_dir(&library).unwrap(); + assert!( + prepare_with_path_open_mode( + &policy, + None, + PathOpenMode::Privileged, + Some(&requirements), + ) + .is_err(), + "missing CDI targets remain fatal" + ); + } + + #[test] + fn cdi_socket_communication_does_not_grant_parent_file_access() { + use std::io::{Read, Write}; + + let availability = probe_availability(); + if !matches!(availability, LandlockAvailability::Available { abi } if abi >= 3) { + assert!( + std::env::var_os("OPENSHELL_TEST_REQUIRE_LANDLOCK").is_none(), + "Landlock ABI v3 required: {availability}" + ); + return; + } + let temp = tempfile::tempdir().unwrap(); + let target = temp.path().join("socket"); + let sibling = temp.path().join("secret"); + std::fs::write(&sibling, b"secret").unwrap(); + let listener = std::os::unix::net::UnixListener::bind(&target).unwrap(); + let allowed = temp.path().join("allowed"); + std::fs::write(&allowed, b"allowed").unwrap(); + let policy = hard_requirement_policy(vec![allowed], vec![]); + let launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + socket_paths: vec![target.to_string_lossy().into_owned()], + ..Default::default() + }), + }; + std::thread::spawn(move || { + let prepared = prepare_current_user(&policy, None, &launch_context) + .unwrap() + .unwrap(); + enforce(prepared).unwrap(); + let mut stream = std::os::unix::net::UnixStream::connect(&target).unwrap(); + stream.write_all(b"ping").unwrap(); + assert_eq!( + std::fs::read(&sibling).unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + assert_eq!( + std::fs::write(&sibling, b"changed").unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + }) + .join() + .unwrap(); + let (mut stream, _) = listener.accept().unwrap(); + let mut message = [0; 4]; + stream.read_exact(&mut message).unwrap(); + assert_eq!(&message, b"ping"); + } + + #[test] + fn cdi_ipc_directory_grants_pipe_access_without_widening_siblings() { + use rustix::fs::{Mode, OFlags, StatVfsMountFlags, mkfifoat, statvfs}; + let availability = probe_availability(); + if !matches!(availability, LandlockAvailability::Available { abi } if abi >= 3) { + assert!( + std::env::var_os("OPENSHELL_TEST_REQUIRE_LANDLOCK").is_none(), + "Landlock ABI v3 required: {availability}" + ); + return; + } + // IPC directories must be on a nosuid,nodev,noexec mount, as the + // container /dev/shm normally is. + let shm = Path::new("/dev/shm"); + let required = + StatVfsMountFlags::NOSUID | StatVfsMountFlags::NODEV | StatVfsMountFlags::NOEXEC; + if !shm.is_dir() || !statvfs(shm).unwrap().f_flag.contains(required) { + return; + } + let pipes = tempfile::tempdir_in(shm).unwrap(); + let control = pipes.path().join("control"); + mkfifoat(rustix::fs::CWD, &control, Mode::from_bits_truncate(0o600)).unwrap(); + let outside = tempfile::tempdir().unwrap(); + let sibling = outside.path().join("secret"); + std::fs::write(&sibling, b"secret").unwrap(); + // Policy enrichment adds the exact IPC directory as read-write. + let policy = hard_requirement_policy(vec![], vec![pipes.path().to_path_buf()]); + let launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + ipc_directory_paths: vec![pipes.path().to_string_lossy().into_owned()], + ..Default::default() + }), + }; + let client = pipes.path().join("client"); + std::thread::spawn(move || { + let prepared = prepare_current_user(&policy, None, &launch_context) + .unwrap() + .unwrap(); + enforce(prepared).unwrap(); + // Opening a FIFO read-write needs no peer and requires write access. + rustix::fs::open(&control, OFlags::RDWR | OFlags::CLOEXEC, Mode::empty()).unwrap(); + std::fs::write(&client, b"client").unwrap(); + assert_eq!( + std::fs::read(&sibling).unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + }) + .join() + .unwrap(); + } + + #[test] + fn cdi_rules_remain_pinned_after_symlink_swap() { + let availability = probe_availability(); + if !matches!(availability, LandlockAvailability::Available { abi } if abi >= 3) { + assert!( + std::env::var_os("OPENSHELL_TEST_REQUIRE_LANDLOCK").is_none(), + "Landlock ABI v3 required for this regression: {availability}" + ); + return; + } + let temp = tempfile::tempdir().unwrap(); + let library = temp.path().join("lib"); + let alias = temp.path().join("alias"); + std::fs::create_dir(&library).unwrap(); + std::fs::write(library.join("sentinel"), b"allowed").unwrap(); + std::os::unix::fs::symlink(&library, &alias).unwrap(); + let mut policy = hard_requirement_policy(vec![alias.clone()], vec![]); + policy.landlock.compatibility = LandlockCompatibility::BestEffort; + let launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + read_only_paths: vec![alias.to_string_lossy().into_owned()], + ..Default::default() + }), + }; + std::thread::spawn(move || { + let prepared = prepare_current_user(&policy, None, &launch_context) + .unwrap() + .unwrap(); + std::fs::remove_file(&alias).unwrap(); + std::os::unix::fs::symlink("/", &alias).unwrap(); + // Later exec preparation must validate the swapped target afresh. + assert!(prepare_current_user(&policy, None, &launch_context).is_err()); + enforce(prepared).unwrap(); + assert_eq!(std::fs::read(library.join("sentinel")).unwrap(), b"allowed"); + for path in [PathBuf::from("/etc/hosts"), alias.join("etc/hosts")] { + assert_eq!( + std::fs::read(path).unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + } + }) + .join() + .unwrap(); + } + + #[test] + fn cdi_alias_validation_is_fatal_in_best_effort_mode() { + let temp = tempfile::tempdir().unwrap(); + let alias = temp.path().join("lib"); + std::os::unix::fs::symlink("/", &alias).unwrap(); + let mut policy = hard_requirement_policy(vec![alias.clone()], vec![]); + policy.landlock.compatibility = LandlockCompatibility::BestEffort; + let launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + read_only_paths: vec![alias.to_string_lossy().into_owned()], + ..Default::default() + }), + }; + assert!(prepare_current_user(&policy, None, &launch_context).is_err()); + } + + #[test] + fn cdi_targets_covered_by_authored_ancestors_are_still_validated() { + if !matches!(probe_availability(), LandlockAvailability::Available { .. }) { + return; + } + let temp = tempfile::tempdir().unwrap(); + let alias = temp.path().join("lib"); + std::os::unix::fs::symlink("/", &alias).unwrap(); + let policy = hard_requirement_policy(vec![temp.path().to_path_buf()], vec![]); + let launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + read_only_paths: vec![alias.to_string_lossy().into_owned()], + ..Default::default() + }), + }; + let Err(error) = prepare_current_user(&policy, None, &launch_context) else { + panic!("covered CDI target must be validated"); + }; + assert!(error.to_string().contains("broad root")); + } + + #[test] + fn cdi_missing_path_is_fatal_in_current_user_mode() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("missing"); + let mut policy = hard_requirement_policy(vec![path.clone()], vec![]); + policy.landlock.compatibility = LandlockCompatibility::BestEffort; + let launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + read_only_paths: vec![path.to_string_lossy().into_owned()], + ..Default::default() + }), + }; + assert!(prepare_current_user(&policy, None, &launch_context).is_err()); + } + + #[test] + fn cdi_writable_mounts_reject_authored_write_grants_at_enforcement() { + let temp = tempfile::tempdir().unwrap(); + let allowed = temp.path().join("allowed"); + let cache = allowed.join("cache"); + std::fs::create_dir_all(&cache).unwrap(); + let file = cache.join("cache.db"); + std::fs::write(&file, b"original").unwrap(); + for path in [cache, file] { + for grant in [allowed.clone(), path.clone()] { + let policy = hard_requirement_policy(vec![temp.path().into()], vec![grant]); + // Check both a persisted writable requirement and mount metadata + // alone, so neither representation can bypass enforcement. + for explicit_requirement in [false, true] { + let launch_context = crate::launch::WorkloadLaunchContext { + cdi_requirements: Some(crate::launch::CdiDerivedRequirements { + read_write_mount_paths: if explicit_requirement { + vec![path.to_string_lossy().into_owned()] + } else { + Vec::new() + }, + mounts: vec![crate::launch::CdiMountRequirement { + path: path.to_string_lossy().into_owned(), + read_only: false, + }], + ..Default::default() + }), + }; + let error = prepare_current_user(&policy, None, &launch_context) + .err() + .unwrap(); + assert!(error.to_string().contains("CDI writable mount")); + } + } + } + } + #[test] fn prepare_hard_requirement_accepts_device_paths() { if !matches!(probe_availability(), LandlockAvailability::Available { .. }) { @@ -720,11 +1123,25 @@ mod tests { std::fs::create_dir(&private).unwrap(); std::fs::write(public.join("sentinel"), b"allowed").unwrap(); std::fs::write(private.join("secret"), b"protected").unwrap(); + let spec = root + .path() + .join(openshell_core::cdi::cdi_spec_mount_path(0).trim_start_matches('/')) + .join("other-vendor.yaml"); + std::fs::create_dir_all(spec.parent().unwrap()).unwrap(); + std::fs::write(&spec, b"trusted runtime CDI metadata").unwrap(); std::os::unix::fs::symlink(root.path(), root.path().join("root-alias")).unwrap(); std::os::unix::fs::symlink(&private, root.path().join("private-alias")).unwrap(); let path = root.path().to_path_buf(); std::thread::spawn(move || { + let authored = prepare(&hard_requirement_policy(vec![path.clone()], vec![]), None) + .unwrap() + .unwrap(); enforce(prepare_capability_free_baseline_at(&path).unwrap()).unwrap(); + enforce(authored).unwrap(); + assert_eq!( + std::fs::read(&spec).unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); assert_eq!( std::fs::read(path.join("public/sentinel")).unwrap(), b"allowed" @@ -733,6 +1150,8 @@ mod tests { ".openshell/secret", "root-alias/.openshell/secret", "private-alias/secret", + ".openshell/cdi-specs/0/other-vendor.yaml", + "private-alias/cdi-specs/0/other-vendor.yaml", ] { let target = path.join(name); assert_eq!( @@ -751,7 +1170,16 @@ mod tests { ); } let policy = hard_requirement_policy(vec![path.join("public")], Vec::new()); - enforce(prepare_current_user(&policy, None).unwrap().unwrap()).unwrap(); + enforce( + prepare_current_user( + &policy, + None, + &crate::launch::WorkloadLaunchContext::default(), + ) + .unwrap() + .unwrap(), + ) + .unwrap(); let read_only = std::ffi::CString::new(path.join("public/sentinel").as_os_str().as_encoded_bytes()) .unwrap(); diff --git a/crates/openshell-sandbox/src/sandbox/linux/mod.rs b/crates/openshell-sandbox/src/sandbox/linux/mod.rs index ef68c81f5c..17f4857651 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/mod.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/mod.rs @@ -40,7 +40,11 @@ pub fn prepare_current_user( policy: &SandboxPolicy, workdir: Option<&str>, ) -> Result { - let landlock = landlock::prepare_current_user(policy, workdir)?; + let landlock = landlock::prepare_current_user( + policy, + workdir, + &crate::launch::WorkloadLaunchContext::default(), + )?; Ok(PreparedSandbox { landlock: landlock.into_iter().collect(), policy: policy.clone(), @@ -58,9 +62,10 @@ pub fn prepare_current_user( pub fn prepare_capability_free( policy: &SandboxPolicy, workdir: Option<&str>, + launch_context: &crate::launch::WorkloadLaunchContext, ) -> Result { let baseline = landlock::prepare_capability_free_baseline()?; - let user = landlock::prepare_current_user(policy, workdir)?; + let user = landlock::prepare_current_user(policy, workdir, launch_context)?; let mut landlock = vec![baseline]; landlock.extend(user); Ok(PreparedSandbox { @@ -110,7 +115,11 @@ pub fn apply_supervisor_prelude() -> Result<()> { /// This must be called **before** `pre_exec` / `fork()` so that the OCSF events /// are emitted through the parent's tracing subscriber (the child process after /// fork does not have a working tracing pipeline). -pub fn log_sandbox_readiness(policy: &SandboxPolicy, workdir: Option<&str>) { +pub fn log_sandbox_readiness( + policy: &SandboxPolicy, + workdir: Option<&str>, + launch_context: &crate::launch::WorkloadLaunchContext, +) { static PROBED: Once = Once::new(); let mut already_probed = true; PROBED.call_once(|| already_probed = false); @@ -144,6 +153,7 @@ pub fn log_sandbox_readiness(policy: &SandboxPolicy, workdir: Option<&str>) { return; } + let compatibility = landlock::launch_compatibility(policy, launch_context); let availability = landlock::probe_availability(); if let landlock::LandlockAvailability::Available { abi } = &availability { openshell_ocsf::ocsf_emit!( @@ -154,7 +164,7 @@ pub fn log_sandbox_readiness(policy: &SandboxPolicy, workdir: Option<&str>) { .message(format!( "Landlock filesystem sandbox available \ [abi:v{abi} compat:{:?} ro:{} rw:{}]", - policy.landlock.compatibility, + compatibility, read_only.len(), read_write.len(), )) @@ -164,7 +174,7 @@ pub fn log_sandbox_readiness(policy: &SandboxPolicy, workdir: Option<&str>) { // Landlock is NOT available — this is the critical log that was // previously invisible because it only fired inside pre_exec. let is_best_effort = matches!( - policy.landlock.compatibility, + compatibility, openshell_core::policy::LandlockCompatibility::BestEffort ); let (desc, msg) = if is_best_effort { diff --git a/crates/openshell-sandbox/src/sandbox/mod.rs b/crates/openshell-sandbox/src/sandbox/mod.rs index bcdb800f79..3065527ce7 100644 --- a/crates/openshell-sandbox/src/sandbox/mod.rs +++ b/crates/openshell-sandbox/src/sandbox/mod.rs @@ -10,6 +10,9 @@ use openshell_core::policy::SandboxPolicy; #[cfg(target_os = "linux")] pub mod linux; +#[cfg(all(unix, any(target_os = "linux", test)))] +mod cdi_paths; + /// Apply sandboxing rules for the current platform. /// /// # Errors From 7e46ebbef8c534242627f1389f177133892a5bf8 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 14 Oct 2026 21:37:25 +0200 Subject: [PATCH 4/6] feat(docker): project CDI spec directories and record the CDI context For GPU sandboxes, record the selected CDI device IDs and the spec directories the Docker daemon currently reports in the protected boundary configuration, and project those directories read-only under /.openshell/cdi-specs so the sandbox runtime can resolve the same specs. User mounts, image volumes and the workspace cannot overlap the projection path. Before creating the sandbox, probe each directory with a stopped container from the prepared workload image and skip directories that do not exist on the Docker host. Probes are bounded by a timeout, labelled for cleanup with their sandbox, and remove their anonymous volumes. A probe that cannot be removed no longer fails creation. With resource admission enabled, containers with a CDI context must keep a CDI device request matching the recorded selection and one read-only, non-recursive, private bind per recorded directory. Admission reads the bootstrap only for CDI containers, and boundary files are replaced atomically so it never observes a partial write. The Docker e2e gateway enables resource admission in GPU mode. Signed-off-by: Evan Lezar --- .../openshell-driver-docker/src/isolation.rs | 20 +- crates/openshell-driver-docker/src/lib.rs | 582 ++++++++- crates/openshell-driver-docker/src/tests.rs | 1050 ++++++++++++++++- e2e/with-docker-gateway.sh | 6 +- 4 files changed, 1617 insertions(+), 41 deletions(-) diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs index b5d4544e00..4150ca58f5 100644 --- a/crates/openshell-driver-docker/src/isolation.rs +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -11,6 +11,7 @@ use std::collections::{BTreeMap, HashMap}; use std::net::IpAddr; use std::path::PathBuf; +use openshell_core::cdi::CdiContext; use openshell_isolation_interface::contract::{ BackendError, OuterFenceGuarantee, OuterFenceGuarantees, ResolvedWorkloadIdentity, }; @@ -70,6 +71,7 @@ pub struct DockerBoundarySpec { pub container_id: String, pub image_identity: String, pub gpu_requested: bool, + pub cdi_context: Option, pub listener_socket: PathBuf, pub control_socket: PathBuf, pub sandbox_tls: SandboxTlsServerConfig, @@ -118,7 +120,7 @@ impl DockerBoundarySpec { }, resource_claims: resource_claims.clone(), resource_claim_files: BTreeMap::new(), - cdi_context: None, + cdi_context: self.cdi_context, workload_identity: self.workload_identity.clone(), outer_fence: outer_fence.clone(), child_env: self.child_env, @@ -188,6 +190,13 @@ mod tests { container_id: "sha256:container".to_string(), image_identity: "sha256:image".to_string(), gpu_requested: true, + cdi_context: Some(CdiContext::new( + vec!["nvidia.com/gpu=0".to_string()], + vec![openshell_core::cdi::CdiSpecDirectory::new( + openshell_core::cdi::cdi_spec_mount_path(0), + "/var/run/cdi", + )], + )), listener_socket: PathBuf::from("/run/openshell/boundary/control.sock"), control_socket: PathBuf::from("/host/control.sock"), sandbox_tls: SandboxTlsServerConfig { @@ -224,6 +233,15 @@ mod tests { provisioned.runtime_descriptor.resource_claims[GPU_RESOURCE_CLAIM], "true" ); + assert_eq!( + provisioned + .boundary_config + .cdi_context + .as_ref() + .unwrap() + .selected_devices, + ["nvidia.com/gpu=0"] + ); assert_eq!( provisioned.boundary_config.outer_fence, provisioned.runtime_descriptor.outer_fence diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 14da9d7584..563f356b14 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -13,8 +13,9 @@ use bollard::errors::Error as BollardError; use bollard::models::{ ContainerCreateBody, ContainerState, ContainerStateStatusEnum, ContainerSummary, ContainerSummaryStateEnum, ContainerWaitResponse, CreateImageInfo, DeviceRequest, HealthConfig, - HealthStatusEnum, HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, - ProgressDetail, SystemInfo, VolumeCreateRequest, + HealthStatusEnum, HostConfig, Mount, MountBindOptions, MountBindOptionsPropagationEnum, + MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, ProgressDetail, SystemInfo, + VolumeCreateRequest, }; use bollard::query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, @@ -23,6 +24,7 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; +use openshell_core::cdi::{CdiContext, CdiSpecDirectory, cdi_spec_mount_path}; use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ @@ -76,7 +78,7 @@ use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::AsyncWriteExt as _; use tokio::sync::{Mutex, broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; @@ -98,6 +100,10 @@ const SUPERVISOR_HEALTH_INTERVAL_NS: i64 = 250_000_000; const SUPERVISOR_HEALTH_TIMEOUT_NS: i64 = 2_000_000_000; const SUPERVISOR_HEALTH_START_PERIOD_NS: i64 = 60_000_000_000; +/// Driver-owned paths in sandbox containers that user mounts, image volumes +/// and the workspace must not overlap. +const DOCKER_RESERVED_CONTROL_PATHS: &[&str] = + &[BOUNDARY_MOUNT_PATH, openshell_core::cdi::CDI_SPEC_DIR_BASE]; const SANDBOX_BINARY_PATH: &str = "/.openshell/runtime/openshell-sandbox"; const SUPERVISOR_IMAGE_CONTROL_BINARY_PATH: &str = "/openshell-supervisor"; const SUPERVISOR_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock"; @@ -122,6 +128,7 @@ const LABEL_ISOLATION_ROLE_SANDBOX: &str = "sandbox"; const LABEL_ISOLATION_ROLE_SUPERVISOR: &str = "supervisor"; const LABEL_ISOLATION_ROLE_STAGING: &str = "staging"; const LABEL_ISOLATION_ROLE_IDENTITY: &str = "identity"; +const LABEL_ISOLATION_ROLE_CDI_PROBE: &str = "cdi-probe"; const RUNTIME_DESCRIPTOR_FILE: &str = "runtime-descriptor.json"; const MAIN_PROCESS_SPEC_FILE: &str = "main-process.json"; const WORKSPACE_ROOT_FILE: &str = "workspace-root"; @@ -344,12 +351,272 @@ struct DockerDriverRuntimeConfig { app_armor_profile: Option, } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] struct DockerGpuRuntimeCapabilities { cdi_supported: bool, + /// Startup snapshot used by tests. Sandbox creation uses the directories + /// the daemon currently reports. + #[cfg(test)] + cdi_spec_dirs: Vec, wsl_all_gpu_fallback_enabled: bool, } +impl DockerGpuRuntimeCapabilities { + #[cfg(test)] + fn cdi_context(&self, gpu_device_ids: Option<&[String]>) -> Result, Status> { + cdi_context_for_spec_dirs(gpu_device_ids, &self.cdi_spec_dirs) + } +} + +/// Build the CDI context from the spec directories the daemon currently reports. +fn cdi_context_for_spec_dirs( + gpu_device_ids: Option<&[String]>, + spec_dirs: &[String], +) -> Result, Status> { + let Some(gpu_device_ids) = gpu_device_ids.filter(|device_ids| !device_ids.is_empty()) else { + return Ok(None); + }; + require_cdi_spec_dirs(spec_dirs)?; + Ok(Some(CdiContext::new( + gpu_device_ids.to_vec(), + spec_dirs + .iter() + .enumerate() + .map(|(index, source)| CdiSpecDirectory::new(cdi_spec_mount_path(index), source)) + .collect(), + ))) +} + +fn require_cdi_spec_dirs(spec_dirs: &[String]) -> Result<(), Status> { + if spec_dirs.is_empty() { + return Err(Status::failed_precondition( + "docker GPU sandboxes require Docker CDI spec directories reported by the daemon", + )); + } + Ok(()) +} + +fn cdi_spec_mount(target: &str, source: &str) -> Result { + driver_mounts::validate_absolute_mount_source(source, "CDI spec directory") + .map_err(Status::failed_precondition)?; + // Validate existence on the daemon host, which may have a different + // filesystem from the gateway. Structured binds reject missing sources and + // do not implicitly relabel them. + Ok(Mount { + typ: Some(MountTypeEnum::BIND), + source: Some(source.to_string()), + target: Some(target.to_string()), + read_only: Some(true), + bind_options: Some(MountBindOptions { + propagation: Some(MountBindOptionsPropagationEnum::RPRIVATE), + create_mountpoint: Some(false), + non_recursive: Some(true), + ..Default::default() + }), + ..Default::default() + }) +} + +const CDI_SPEC_DIRECTORY_CACHE_TTL: Duration = Duration::from_secs(30); +/// Probes run while holding the directory cache lock, so bound each one to keep +/// a stalled daemon from blocking every GPU sandbox creation. +const CDI_SPEC_PROBE_TIMEOUT: Duration = Duration::from_secs(20); +const CDI_SPEC_PROBE_TARGET: &str = "/.openshell/cdi-probe"; + +#[derive(Default)] +struct CdiSpecDirectoryCache { + entries: HashMap, + inventory: Option, + daemon_id: Option, + spec_dirs: Vec, +} + +impl CdiSpecDirectoryCache { + fn observe_daemon(&mut self, info: &SystemInfo) { + let inventory = docker_cdi_gpu_inventory(info); + let dirs = info.cdi_spec_dirs.clone().unwrap_or_default(); + if self.inventory.as_ref() != Some(&inventory) + || self.daemon_id != info.id + || self.spec_dirs != dirs + { + self.entries.clear(); + self.inventory = Some(inventory); + self.daemon_id.clone_from(&info.id); + self.spec_dirs = dirs; + } + } + + async fn available_context( + &mut self, + mut context: CdiContext, + now: Instant, + mut probe: F, + ) -> Result + where + F: FnMut(CdiSpecDirectory) -> Fut, + Fut: Future>, + { + self.entries.retain(|_, (checked, _)| { + now.saturating_duration_since(*checked) < CDI_SPEC_DIRECTORY_CACHE_TTL + }); + let mut available = Vec::new(); + for directory in context.spec_dirs { + let present = if let Some((_, present)) = self.entries.get(&directory.source) { + *present + } else { + let present = probe(directory.clone()).await?; + self.entries + .insert(directory.source.clone(), (now, present)); + present + }; + if present { + available.push(directory); + } + } + if available.is_empty() { + return Err(Status::failed_precondition( + "no configured CDI spec directory exists on the Docker daemon host", + )); + } + context.spec_dirs = available; + Ok(context) + } +} + +/// Replace any CDI spec projections in the create body with exactly the +/// directories recorded in the context, so the mounts and the persisted +/// context cannot diverge. +fn retain_cdi_spec_projections( + body: &mut ContainerCreateBody, + context: &CdiContext, +) -> Result<(), Status> { + let projections = context + .spec_dirs + .iter() + .map(|dir| cdi_spec_mount(&dir.path, &dir.source)) + .collect::, _>>()?; + let mounts = body + .host_config + .get_or_insert_with(Default::default) + .mounts + .get_or_insert_with(Vec::new); + mounts.retain(|mount| { + !mount.target.as_deref().is_some_and(|target| { + Path::new(target).starts_with(openshell_core::cdi::CDI_SPEC_DIR_BASE) + }) + }); + mounts.extend(projections); + Ok(()) +} + +fn is_missing_bind_source_error(error: &BollardError, source: &str) -> bool { + // Match the stable phrase and the source rather than the full daemon + // message, whose prefix varies across Docker versions and API proxies. + matches!( + error, + BollardError::DockerResponseServerError { status_code: 400, message } + if message.contains("bind source path does not exist") && message.contains(source) + ) +} + +/// Validate one daemon-host source without starting any process. Deleting the +/// stopped container also removes anonymous volumes declared by the image. +async fn probe_cdi_spec_directory( + docker: &Docker, + image: &str, + labels: HashMap, + directory: CdiSpecDirectory, +) -> Result { + // Finish cleanup even if sandbox provisioning is cancelled mid-probe or + // the probe times out. + let docker = docker.clone(); + let image = image.to_string(); + let task = tokio::spawn( + async move { run_cdi_spec_directory_probe(&docker, &image, labels, directory).await } + .in_current_span(), + ); + await_cdi_spec_directory_probe(task, CDI_SPEC_PROBE_TIMEOUT).await +} + +async fn await_cdi_spec_directory_probe( + task: JoinHandle>, + timeout: Duration, +) -> Result { + match tokio::time::timeout(timeout, task).await { + Ok(joined) => joined.map_err(|error| { + Status::internal(format!("CDI spec directory probe task failed: {error}")) + })?, + Err(_) => Err(Status::unavailable(format!( + "CDI spec directory probe did not finish within {}s", + timeout.as_secs() + ))), + } +} + +async fn run_cdi_spec_directory_probe( + docker: &Docker, + image: &str, + labels: HashMap, + directory: CdiSpecDirectory, +) -> Result { + driver_mounts::validate_absolute_mount_source(&directory.source, "CDI spec directory") + .map_err(Status::failed_precondition)?; + // Auxiliary labels let the stale-resource sweep remove a probe left behind + // by a failed removal or a gateway crash. + let body = ContainerCreateBody { + image: Some(image.into()), + labels: Some(labels), + host_config: Some(HostConfig { + network_mode: Some("none".into()), + readonly_rootfs: Some(true), + mounts: Some(vec![Mount { + typ: Some(MountTypeEnum::BIND), + source: Some(directory.source.clone()), + target: Some(CDI_SPEC_PROBE_TARGET.into()), + read_only: Some(true), + bind_options: Some(MountBindOptions { + propagation: Some(MountBindOptionsPropagationEnum::RPRIVATE), + create_mountpoint: Some(false), + non_recursive: Some(true), + ..Default::default() + }), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }; + match docker + .create_container( + None::, + body, + ) + .await + { + Ok(created) => { + // The directory exists once the probe is created. A probe left by a + // failed removal carries the sandbox's auxiliary labels, so it is + // removed with the sandbox or by the startup sweep. + if let Err(error) = docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().v(true).build()), + ) + .await + { + warn!( + container_id = %created.id, + error = %error, + "Failed to remove CDI spec directory probe; it is removed with the sandbox" + ); + } + Ok(true) + } + Err(error) if is_missing_bind_source_error(&error, &directory.source) => Ok(false), + Err(error) => Err(internal_status("probe CDI spec directory", error)), + } +} + #[derive(Clone)] pub struct DockerComputeDriver { docker: Arc, @@ -357,6 +624,7 @@ pub struct DockerComputeDriver { events: broadcast::Sender, pending: Arc>>, gpu_selector: Arc, + cdi_spec_directory_cache: Arc>, lifecycle_event_fences: DockerLifecycleEventFences, control_processes: Arc>>, runtime_failures: Arc>>, @@ -893,14 +1161,14 @@ impl DockerComputeDriver { let info = docker.info().await.map_err(|err| { Error::execution(format!("failed to query Docker daemon info: {err}")) })?; - let cdi_supported = info - .cdi_spec_dirs - .as_ref() - .is_some_and(|dirs| !dirs.is_empty()); + let cdi_spec_dirs = info.cdi_spec_dirs.clone().unwrap_or_default(); + let cdi_supported = !cdi_spec_dirs.is_empty(); let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); let wsl_all_gpu_fallback_enabled = docker_info_reports_wsl2(&info); let gpu = DockerGpuRuntimeCapabilities { cdi_supported, + #[cfg(test)] + cdi_spec_dirs, wsl_all_gpu_fallback_enabled, }; validate_docker_app_armor_profile(docker_config.app_armor_profile.as_ref(), &info)?; @@ -967,7 +1235,7 @@ impl DockerComputeDriver { supervisor_gateway_name: docker_config.supervisor_gateway_name.clone(), ssh_socket_path: docker_config.ssh_socket_path.clone(), guest_tls, - gpu, + gpu: gpu.clone(), sandbox_pids_limit: docker_config.sandbox_pids_limit, enable_bind_mounts: docker_config.enable_bind_mounts, allow_driver_config: docker_config.allow_driver_config, @@ -985,6 +1253,7 @@ impl DockerComputeDriver { cdi_gpu_inventory, gpu.wsl_all_gpu_fallback_enabled, )), + cdi_spec_directory_cache: Arc::new(Mutex::new(CdiSpecDirectoryCache::default())), lifecycle_event_fences: DockerLifecycleEventFences::default(), control_processes: Arc::new(Mutex::new(HashMap::new())), runtime_failures: Arc::new(Mutex::new(HashMap::new())), @@ -1202,6 +1471,10 @@ impl DockerComputeDriver { .info() .await .map_err(|err| internal_status("query Docker daemon info", err))?; + self.cdi_spec_directory_cache + .lock() + .await + .observe_daemon(&info); self.gpu_selector.refresh( docker_cdi_gpu_inventory(&info), self.config.gpu.wsl_all_gpu_fallback_enabled, @@ -1212,6 +1485,31 @@ impl DockerComputeDriver { async fn admit_container_resources( &self, container: &bollard::models::ContainerInspectResponse, + ) -> Result<(), Status> { + // Only CDI containers carry a projection context, so other containers + // are admitted without depending on the bootstrap file. + if !self.config.resource_admission.enabled || !has_cdi_device_request(container) { + return self + .admit_container_resources_with_cdi_context(container, None) + .await; + } + let sandbox_id = container + .config + .as_ref() + .and_then(|config| config.labels.as_ref()) + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .ok_or_else(|| Status::failed_precondition("sandbox lacks identity"))?; + let path = + docker_boundary_state_dir_by_id(sandbox_id, &self.config)?.join(BOUNDARY_CONFIG_FILE); + let cdi_context = read_docker_cdi_context(&path).await?; + self.admit_container_resources_with_cdi_context(container, cdi_context.as_ref()) + .await + } + + async fn admit_container_resources_with_cdi_context( + &self, + container: &bollard::models::ContainerInspectResponse, + cdi_context: Option<&CdiContext>, ) -> Result<(), Status> { if self.config.allow_driver_config && !self.config.resource_admission.enabled { return Ok(()); @@ -1240,12 +1538,67 @@ impl DockerComputeDriver { .get(openshell_core::resource_admission::IDENTITIES_LABEL) .and_then(|value| serde_json::from_str(value).ok()) .ok_or_else(|| Status::failed_precondition("sandbox lacks resource identity record"))?; + // The driver-owned context defines the projections for this container. + // A legacy GPU container without context retains baseline handling; a + // device request alone does not imply that spec directories were mounted. + // Caller-provided bind mounts still require ordinary admission checks. + let cdi_requested = has_cdi_device_request(container); + let mut cdi_projections = std::collections::BTreeMap::new(); + if let Some(context) = cdi_context { + if !cdi_requested || context.spec_dirs.is_empty() { + return Err(Status::failed_precondition( + "CDI context requires a device request and spec projections", + )); + } + if context.version != openshell_core::cdi::CDI_CONTEXT_VERSION { + return Err(Status::failed_precondition(format!( + "unsupported CDI context version {}", + context.version + ))); + } + let selected = context + .selected_devices + .iter() + .map(String::as_str) + .collect::>(); + if cdi_device_request_ids(container) != selected { + return Err(Status::failed_precondition( + "CDI device request does not match the recorded CDI context", + )); + } + for directory in &context.spec_dirs { + validate_cdi_spec_mount_config(container, &directory.path)?; + cdi_projections.insert(directory.path.clone(), directory.source.as_str()); + } + } let mut actual = std::collections::BTreeMap::new(); let anonymous_targets: Vec = labels .get("openshell.ai/private-image-volume-targets") .and_then(|value| serde_json::from_str(value).ok()) .unwrap_or_default(); for mount in container.mounts.as_deref().unwrap_or_default() { + if let Some(source) = mount + .destination + .as_deref() + .and_then(|destination| cdi_projections.get(destination)) + { + if mount.typ != Some(bollard::models::MountPointTypeEnum::BIND) + || mount.source.as_deref() != Some(*source) + || mount.rw != Some(false) + || mount.propagation.as_deref() != Some("rprivate") + { + return Err(Status::failed_precondition( + "CDI spec projection source, type, access or propagation changed", + )); + } + cdi_projections.remove( + mount + .destination + .as_deref() + .expect("projection destination"), + ); + continue; + } match mount.typ { Some(bollard::models::MountPointTypeEnum::VOLUME) => { let name = mount @@ -1309,6 +1662,11 @@ impl DockerComputeDriver { .reject_unlabelable("effective container mount")?, } } + if !cdi_projections.is_empty() { + return Err(Status::failed_precondition( + "CDI spec projection is missing", + )); + } if actual != expected { return Err(Status::failed_precondition( "external volume identity or attachment inventory changed", @@ -1333,6 +1691,9 @@ impl DockerComputeDriver { "driver_config.cdi_devices", ) .map_err(Status::invalid_argument)?; + // Observe the daemon so CDI spec directories and cache + // invalidation reflect its current configuration. + self.refresh_gpu_inventory().await?; return Ok(Some(cdi_devices.to_vec())); } @@ -1660,6 +2021,45 @@ impl DockerComputeDriver { )); } }; + let cdi_context = if gpu_devices + .as_ref() + .is_some_and(|devices| !devices.is_empty()) + { + let mut cache = self.cdi_spec_directory_cache.lock().await; + let probe_labels = docker_auxiliary_container_labels( + sandbox, + &self.config, + LABEL_ISOLATION_ROLE_CDI_PROBE, + ); + match cdi_context_for_spec_dirs(gpu_devices.as_deref(), &cache.spec_dirs) { + Ok(Some(context)) => cache + .available_context(context, Instant::now(), |directory| { + probe_cdi_spec_directory( + &self.docker, + &image.id, + probe_labels.clone(), + directory, + ) + }) + .await + .map(Some), + result => result, + } + } else { + Ok(None) + }; + let cdi_context = match cdi_context { + Ok(context) => context, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "CdiSpecDirectoryProbeFailed", + status.message(), + )); + } + }; let mut create_body = match build_container_create_body_for_image( sandbox, &self.config, @@ -1679,6 +2079,17 @@ impl DockerComputeDriver { )); } }; + if let Some(context) = &cdi_context + && let Err(status) = retain_cdi_spec_projections(&mut create_body, context) + { + let _ = + remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config).await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ContainerCreateFailed", + status.message(), + )); + } let identities = match self .validate_user_volume_mounts_available(&validated.driver_config, &sandbox.workspace) .await @@ -1754,7 +2165,10 @@ impl DockerComputeDriver { } }; let outer_fence_error = match validate_docker_outer_fence(&inspected) { - Ok(()) => self.admit_container_resources(&inspected).await.err(), + Ok(()) => self + .admit_container_resources_with_cdi_context(&inspected, cdi_context.as_ref()) + .await + .err(), Err(error) => Some(error), }; drop(inspected); @@ -1788,7 +2202,7 @@ impl DockerComputeDriver { &created.id, &image, &workload_identity, - gpu_devices.is_some(), + cdi_context.as_ref(), ) .await { @@ -1993,6 +2407,7 @@ impl DockerComputeDriver { LABEL_ISOLATION_ROLE_SUPERVISOR | LABEL_ISOLATION_ROLE_STAGING | LABEL_ISOLATION_ROLE_IDENTITY + | LABEL_ISOLATION_ROLE_CDI_PROBE ) ) { continue; @@ -2001,10 +2416,7 @@ impl DockerComputeDriver { continue; }; self.docker - .remove_container( - &target, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) + .remove_container(&target, Some(auxiliary_container_remove_options(role))) .await .or_else(|error| { if is_not_found_error(&error) || is_removal_in_progress_error(&error) { @@ -2055,6 +2467,7 @@ impl DockerComputeDriver { LABEL_ISOLATION_ROLE_SUPERVISOR | LABEL_ISOLATION_ROLE_STAGING | LABEL_ISOLATION_ROLE_IDENTITY + | LABEL_ISOLATION_ROLE_CDI_PROBE ) ) { continue; @@ -2063,10 +2476,7 @@ impl DockerComputeDriver { continue; }; self.docker - .remove_container( - &target, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) + .remove_container(&target, Some(auxiliary_container_remove_options(role))) .await .or_else(|error| { if is_not_found_error(&error) { @@ -4253,18 +4663,107 @@ fn prepare_docker_boundary_state_dir( Ok(directory) } +/// Read the complete driver-owned bootstrap; unavailable or malformed state is +/// an inspection failure, never evidence that a container uses the legacy path. +fn has_cdi_device_request(container: &bollard::models::ContainerInspectResponse) -> bool { + !cdi_device_request_ids(container).is_empty() +} + +fn cdi_device_request_ids( + container: &bollard::models::ContainerInspectResponse, +) -> std::collections::BTreeSet<&str> { + container + .host_config + .as_ref() + .and_then(|host| host.device_requests.as_deref()) + .unwrap_or_default() + .iter() + .filter(|request| request.driver.as_deref() == Some("cdi")) + .flat_map(|request| request.device_ids.as_deref().unwrap_or_default()) + .map(String::as_str) + .collect() +} + +/// The effective mount list does not report bind recursion, so check the +/// container's mount configuration for each recorded projection. +fn validate_cdi_spec_mount_config( + container: &bollard::models::ContainerInspectResponse, + target: &str, +) -> Result<(), Status> { + let mut configured = container + .host_config + .as_ref() + .and_then(|host| host.mounts.as_deref()) + .unwrap_or_default() + .iter() + .filter(|mount| mount.target.as_deref() == Some(target)); + let valid = configured.next().is_some_and(|mount| { + mount.typ == Some(MountTypeEnum::BIND) + && mount.read_only == Some(true) + && mount + .bind_options + .as_ref() + .is_some_and(|options| options.non_recursive == Some(true)) + }) && configured.next().is_none(); + if valid { + Ok(()) + } else { + Err(Status::failed_precondition( + "CDI spec projection must be configured as one read-only non-recursive bind", + )) + } +} + +/// A missing or undecodable bootstrap is persistent and fails admission. +/// Other read errors may be transient and defer admission to the next pass. +async fn read_docker_cdi_context(path: &Path) -> Result, Status> { + let contents = tokio::fs::read(path).await.map_err(|error| { + let message = format!("read Docker sandbox bootstrap for admission: {error}"); + if error.kind() == std::io::ErrorKind::NotFound { + Status::failed_precondition(message) + } else { + Status::unavailable(message) + } + })?; + let config: BoundaryConfig = serde_json::from_slice(&contents).map_err(|error| { + Status::failed_precondition(format!( + "decode Docker sandbox bootstrap for admission: {error}" + )) + })?; + Ok(config.cdi_context) +} + +/// Replace the file atomically so concurrent readers never see a partial write. async fn write_docker_boundary_file(path: &Path, contents: &[u8]) -> Result<(), Status> { - tokio::fs::write(path, contents).await.map_err(|error| { + let file_name = path.file_name().ok_or_else(|| { Status::internal(format!( - "write Docker boundary file {}: {error}", + "write Docker boundary file {}: path has no file name", path.display() )) })?; - openshell_core::paths::set_file_owner_only(path).map_err(|error| { + let mut temporary_name = std::ffi::OsString::from("."); + temporary_name.push(file_name); + temporary_name.push(".tmp"); + let temporary = path.with_file_name(temporary_name); + tokio::fs::write(&temporary, contents) + .await + .map_err(|error| { + Status::internal(format!( + "write Docker boundary file {}: {error}", + path.display() + )) + })?; + openshell_core::paths::set_file_owner_only(&temporary).map_err(|error| { Status::internal(format!( "restrict Docker boundary file {}: {error}", path.display() )) + })?; + tokio::fs::rename(&temporary, path).await.map_err(|error| { + Status::internal(format!( + "replace Docker boundary file {}: {error}", + path.display() + )) }) } @@ -4548,7 +5047,7 @@ async fn prepare_docker_boundary_files( container_id: &str, image: &DockerImageMetadata, workload_identity: &ResolvedWorkloadIdentity, - gpu_requested: bool, + cdi_context: Option<&CdiContext>, ) -> Result<(), Status> { let directory = docker_boundary_state_dir(sandbox, config)?; let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) @@ -4578,7 +5077,8 @@ async fn prepare_docker_boundary_files( verification_keys, container_id: container_id.to_string(), image_identity: image.id.clone(), - gpu_requested, + gpu_requested: cdi_context.is_some(), + cdi_context: cdi_context.cloned(), listener_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), control_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), sandbox_tls: SandboxTlsServerConfig { @@ -4979,6 +5479,17 @@ async fn stage_docker_supervisor_bundle( cleanup } +/// Probe containers are created from the workload image, so removing them +/// must also remove the anonymous volumes the image declares. +fn auxiliary_container_remove_options( + role: Option<&str>, +) -> bollard::query_parameters::RemoveContainerOptions { + RemoveContainerOptionsBuilder::default() + .force(true) + .v(role == Some(LABEL_ISOLATION_ROLE_CDI_PROBE)) + .build() +} + fn docker_auxiliary_container_labels( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, @@ -5765,7 +6276,7 @@ fn build_container_create_body_with_gpu_devices( template.image.clone(), ) .map_err(|error| Status::internal(error.to_string()))?; - build_container_create_body_for_image( + let mut body = build_container_create_body_for_image( sandbox, config, driver_config, @@ -5777,7 +6288,12 @@ fn build_container_create_body_with_gpu_devices( volumes: Vec::new(), }, &workload_identity, - ) + )?; + // Mirror sandbox creation, which adds projections from the CDI context. + if let Some(context) = config.gpu.cdi_context(gpu_device_ids)? { + retain_cdi_spec_projections(&mut body, &context)?; + } + Ok(body) } fn build_container_create_body_for_image( @@ -5803,8 +6319,10 @@ fn build_container_create_body_for_image( let resource_limits = docker_resource_limits(template)?; let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) .map_err(Status::failed_precondition)?; - driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) - .map_err(Status::failed_precondition)?; + for control_path in DOCKER_RESERVED_CONTROL_PATHS { + driver_mounts::validate_workspace_control_path(&workspace_root, control_path) + .map_err(Status::failed_precondition)?; + } for volume in &image.volumes { driver_mounts::validate_container_mount_target(volume).map_err(|error| { Status::failed_precondition(format!( @@ -5816,8 +6334,10 @@ fn build_container_create_body_for_image( "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" )) })?; - driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) - .map_err(Status::failed_precondition)?; + for control_path in DOCKER_RESERVED_CONTROL_PATHS { + driver_mounts::validate_mount_control_path(volume, control_path) + .map_err(Status::failed_precondition)?; + } } let mount_targets = driver_config.mounts.iter().map(|mount| { match mount { @@ -5831,7 +6351,7 @@ fn build_container_create_body_for_image( driver_mounts::validate_workspace_mount_targets( mount_targets, &workspace_root, - &[BOUNDARY_MOUNT_PATH], + DOCKER_RESERVED_CONTROL_PATHS, ) .map_err(Status::failed_precondition)?; let mut user_mounts = docker_driver_mounts(driver_config)?; diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index f690aae921..4411179101 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -49,6 +49,9 @@ fn startup_error_log_tails_fit_grpc_header_budget() { assert!(format_log_tail("").is_empty()); } +const TEST_CDI_SPEC_DIR: &str = "/opt/openshell-test/cdi"; +const TEST_CDI_SPEC_DIR_ALT: &str = "/srv/openshell-test/cdi"; + fn test_launch_authentication() -> Vec { serde_json::to_vec(&SandboxLaunchAuthentication { supervisor: SupervisorAuthBundle { @@ -107,6 +110,394 @@ fn test_sandbox() -> DriverSandbox { } } +#[tokio::test] +async fn admission_accepts_only_exact_read_only_cdi_spec_projections() { + let mut config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR]); + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + let driver = test_driver_with_config(config); + let context = driver + .config + .gpu + .cdi_context(Some(&["nvidia.com/gpu=0".into()])) + .unwrap() + .unwrap(); + let value = serde_json::json!({ + "Config": {"Labels": { + LABEL_SANDBOX_WORKSPACE: "team-a", + LABEL_SANDBOX_ID: "cdi-admission", + openshell_core::resource_admission::CONFIG_USED_LABEL: "false", + openshell_core::resource_admission::IDENTITIES_LABEL: "{}" + }}, + "HostConfig": { + "DeviceRequests": [{"Driver": "cdi", "DeviceIDs": ["nvidia.com/gpu=0"]}], + "Mounts": [cdi_spec_mount_config(TEST_CDI_SPEC_DIR, &cdi_spec_mount_path(0))] + }, + "Mounts": [{"Type": "bind", "Source": TEST_CDI_SPEC_DIR, + "Destination": cdi_spec_mount_path(0), "RW": false, "Propagation": "rprivate"}] + }); + let inspect = serde_json::from_value(value.clone()).unwrap(); + driver + .admit_container_resources_with_cdi_context(&inspect, Some(&context)) + .await + .unwrap(); + // The configured mount must stay a single read-only, non-recursive bind. + for (pointer, replacement) in [ + ( + "/HostConfig/Mounts/0/BindOptions/NonRecursive", + serde_json::json!(false), + ), + ("/HostConfig/Mounts/0/ReadOnly", serde_json::json!(false)), + ("/HostConfig/Mounts/0/Type", serde_json::json!("volume")), + ("/HostConfig/Mounts", serde_json::json!([])), + ( + "/HostConfig/Mounts", + serde_json::json!([ + cdi_spec_mount_config(TEST_CDI_SPEC_DIR, &cdi_spec_mount_path(0)), + cdi_spec_mount_config(TEST_CDI_SPEC_DIR, &cdi_spec_mount_path(0)), + ]), + ), + ] { + let mut changed = value.clone(); + *changed.pointer_mut(pointer).unwrap() = replacement; + let inspect = serde_json::from_value(changed).unwrap(); + let error = driver + .admit_container_resources_with_cdi_context(&inspect, Some(&context)) + .await + .unwrap_err(); + assert!( + error.message().contains("non-recursive bind"), + "{pointer}: {error}" + ); + } + // The device request must match the recorded selection exactly. + for device_ids in [ + serde_json::json!(["nvidia.com/gpu=1"]), + serde_json::json!(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]), + ] { + let mut changed = value.clone(); + changed["HostConfig"]["DeviceRequests"][0]["DeviceIDs"] = device_ids.clone(); + let inspect = serde_json::from_value(changed).unwrap(); + let error = driver + .admit_container_resources_with_cdi_context(&inspect, Some(&context)) + .await + .unwrap_err(); + assert!(error.message().contains("does not match"), "{device_ids}"); + } + let mut future_context = context.clone(); + future_context.version += 1; + let error = driver + .admit_container_resources_with_cdi_context(&inspect, Some(&future_context)) + .await + .unwrap_err(); + assert!(error.message().contains("unsupported CDI context version")); + for (field, replacement) in [ + ("Source", serde_json::json!("/untrusted")), + ("Destination", serde_json::json!("/elsewhere")), + ("RW", serde_json::json!(true)), + ("Type", serde_json::json!("volume")), + ("Propagation", serde_json::json!("rshared")), + ] { + let mut changed = value.clone(); + changed["Mounts"][0][field] = replacement; + // Avoid a daemon lookup for the wrong mount type. + if field == "Type" { + changed["Mounts"][0]["Name"] = serde_json::json!("unexpected"); + } + let inspect = serde_json::from_value(changed).unwrap(); + assert!( + driver + .admit_container_resources_with_cdi_context(&inspect, Some(&context)) + .await + .is_err(), + "{field}" + ); + } + for change in ["missing", "duplicate", "extra", "no-request"] { + let mut changed = value.clone(); + match change { + "missing" => changed["Mounts"] = serde_json::json!([]), + "duplicate" => changed["Mounts"] + .as_array_mut() + .unwrap() + .push(value["Mounts"][0].clone()), + "extra" => changed["Mounts"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "Type": "bind", "Source": "/data", "Destination": "/data", "RW": false + })), + "no-request" => changed["HostConfig"]["DeviceRequests"] = serde_json::json!([]), + _ => unreachable!(), + } + let inspect = serde_json::from_value(changed).unwrap(); + assert!( + driver + .admit_container_resources_with_cdi_context(&inspect, Some(&context)) + .await + .is_err(), + "{change}" + ); + } +} + +fn cdi_spec_mount_config(source: &str, target: &str) -> serde_json::Value { + serde_json::json!({ + "Type": "bind", "Source": source, "Target": target, "ReadOnly": true, + "BindOptions": {"Propagation": "rprivate", "NonRecursive": true, "CreateMountpoint": false} + }) +} + +#[test] +fn cdi_context_rejects_unknown_fields() { + let mut value = serde_json::to_value(CdiContext::new( + vec!["nvidia.com/gpu=0".into()], + vec![CdiSpecDirectory::new( + cdi_spec_mount_path(0), + TEST_CDI_SPEC_DIR, + )], + )) + .unwrap(); + value["unexpected"] = serde_json::json!(true); + assert!(serde_json::from_value::(value.clone()).is_err()); + value.as_object_mut().unwrap().remove("unexpected"); + value["spec_dirs"][0]["unexpected"] = serde_json::json!(true); + assert!(serde_json::from_value::(value).is_err()); +} + +fn admission_boundary_config(cdi_context: Option) -> BoundaryConfig { + let auth = decode_docker_launch_authentication(&test_launch_authentication()).unwrap(); + isolation::DockerBoundarySpec { + boundary_id: "cdi-admission".into(), + generation: auth.supervisor.runtime_generation.to_string(), + session_id: auth.supervisor.session_id, + session_rotation: auth.supervisor.session_rotation, + auth_epoch: auth.supervisor.auth_epoch, + gateway_id: auth.gateway_id, + verification_keys: gateway_verification_keys(&auth.verification_keys).unwrap(), + container_id: "container-test".into(), + image_identity: "sha256:image".into(), + gpu_requested: true, + cdi_context, + listener_socket: BOUNDARY_SOCKET_MOUNT_PATH.into(), + control_socket: BOUNDARY_SOCKET_MOUNT_PATH.into(), + sandbox_tls: SandboxTlsServerConfig { + certificate_chain_path: BOUNDARY_CERTIFICATE_MOUNT_PATH.into(), + private_key_path: BOUNDARY_PRIVATE_KEY_MOUNT_PATH.into(), + }, + supervisor_tls: SandboxTlsClientConfig { + server_name: "sandbox-test".into(), + trust_anchor_pem: "test-ca".into(), + }, + host_gateway_ip: None, + workload_identity: ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "image".into(), + "sha256:image".into(), + ) + .unwrap(), + child_env: HashMap::new(), + } + .provision() + .unwrap() + .boundary_config +} + +#[tokio::test] +async fn admission_uses_persisted_cdi_context_for_existing_containers() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join(BOUNDARY_CONFIG_FILE); + let mut config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR_ALT]); + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + let driver = test_driver_with_config(config); + let mut value = serde_json::json!({ + "Config": {"Labels": { + LABEL_SANDBOX_WORKSPACE: "team-a", + LABEL_SANDBOX_ID: "cdi-admission", + openshell_core::resource_admission::CONFIG_USED_LABEL: "false", + openshell_core::resource_admission::IDENTITIES_LABEL: "{}" + }}, + "HostConfig": {"DeviceRequests": [{"Driver": "cdi", "DeviceIDs": ["nvidia.com/gpu=0"]}]}, + "Mounts": [] + }); + // Older bootstraps omit the field entirely. GPU and ordinary containers + // without context need no projections, even on a now-CDI-capable daemon. + let mut legacy = serde_json::to_value(admission_boundary_config(None)).unwrap(); + legacy.as_object_mut().unwrap().remove("cdi_context"); + fs::write(&path, serde_json::to_vec(&legacy).unwrap()).unwrap(); + let context = read_docker_cdi_context(&path).await.unwrap(); + assert!(context.is_none()); + for requests in [ + value["HostConfig"]["DeviceRequests"].clone(), + serde_json::json!([]), + ] { + value["HostConfig"]["DeviceRequests"] = requests; + let inspect = serde_json::from_value(value.clone()).unwrap(); + driver + .admit_container_resources_with_cdi_context(&inspect, context.as_ref()) + .await + .unwrap(); + } + value["HostConfig"]["DeviceRequests"] = serde_json::json!([ + {"Driver": "cdi", "DeviceIDs": ["nvidia.com/gpu=0"]} + ]); + value["Mounts"] = serde_json::json!([ + {"Type": "bind", "Source": "/data", "Destination": "/data", "RW": false} + ]); + let inspect = serde_json::from_value(value.clone()).unwrap(); + assert!( + driver + .admit_container_resources_with_cdi_context(&inspect, None) + .await + .is_err() + ); + + // The original context remains authoritative after daemon settings change. + let context = CdiContext::new( + vec!["nvidia.com/gpu=0".into()], + vec![CdiSpecDirectory::new( + cdi_spec_mount_path(0), + TEST_CDI_SPEC_DIR, + )], + ); + fs::write( + &path, + admission_boundary_config(Some(context)).encode().unwrap(), + ) + .unwrap(); + let context = read_docker_cdi_context(&path).await.unwrap(); + value["HostConfig"]["Mounts"] = serde_json::json!([cdi_spec_mount_config( + TEST_CDI_SPEC_DIR, + &cdi_spec_mount_path(0) + )]); + value["Mounts"] = serde_json::json!([]); + let inspect = serde_json::from_value(value.clone()).unwrap(); + let error = driver + .admit_container_resources_with_cdi_context(&inspect, context.as_ref()) + .await + .unwrap_err(); + assert_eq!(error.message(), "CDI spec projection is missing"); + value["Mounts"] = serde_json::json!([ + {"Type": "bind", "Source": TEST_CDI_SPEC_DIR, "Destination": cdi_spec_mount_path(0), + "RW": false, "Propagation": "rprivate"} + ]); + let inspect = serde_json::from_value(value.clone()).unwrap(); + driver + .admit_container_resources_with_cdi_context(&inspect, context.as_ref()) + .await + .unwrap(); + value["Mounts"][0]["Source"] = serde_json::json!(TEST_CDI_SPEC_DIR_ALT); + let inspect = serde_json::from_value(value).unwrap(); + assert!( + driver + .admit_container_resources_with_cdi_context(&inspect, context.as_ref()) + .await + .is_err() + ); +} + +#[tokio::test] +async fn stalled_cdi_spec_directory_probe_times_out_without_cancelling_cleanup() { + let (finished_tx, finished_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel::<()>(); + let task = tokio::spawn(async move { + let _ = release_rx.await; + let _ = finished_tx.send(()); + Ok(true) + }); + let error = await_cdi_spec_directory_probe(task, Duration::from_millis(10)) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::Unavailable); + // The detached probe still runs to completion so it can remove its container. + release_tx.send(()).unwrap(); + finished_rx.await.unwrap(); +} + +#[tokio::test] +async fn admission_reads_bootstrap_only_for_cdi_containers() { + let mut config = runtime_config(); + config.resource_admission = + openshell_core::resource_admission::ResourceAdmissionConfig::default(); + let driver = test_driver_with_config(config); + // No bootstrap exists for this sandbox. Ordinary containers must still be + // admitted, while CDI containers fail closed. + let mut value = serde_json::json!({ + "Config": {"Labels": { + LABEL_SANDBOX_WORKSPACE: "team-a", + LABEL_SANDBOX_ID: "admission-without-bootstrap-0f3c9a", + openshell_core::resource_admission::CONFIG_USED_LABEL: "false", + openshell_core::resource_admission::IDENTITIES_LABEL: "{}" + }}, + "HostConfig": {"DeviceRequests": []}, + "Mounts": [] + }); + let inspect = serde_json::from_value(value.clone()).unwrap(); + driver.admit_container_resources(&inspect).await.unwrap(); + + value["HostConfig"]["DeviceRequests"] = serde_json::json!([ + {"Driver": "cdi", "DeviceIDs": ["nvidia.com/gpu=0"]} + ]); + let inspect = serde_json::from_value(value).unwrap(); + assert_eq!( + driver + .admit_container_resources(&inspect) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); +} + +#[tokio::test] +async fn boundary_files_are_replaced_atomically() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join(BOUNDARY_CONFIG_FILE); + write_docker_boundary_file(&path, b"first").await.unwrap(); + write_docker_boundary_file(&path, b"second").await.unwrap(); + assert_eq!(fs::read(&path).unwrap(), b"second"); + let entries = fs::read_dir(directory.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>(); + assert_eq!( + entries, + vec![std::ffi::OsString::from(BOUNDARY_CONFIG_FILE)] + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } +} + +#[tokio::test] +async fn admission_cdi_context_lookup_failures_do_not_select_legacy_handling() { + let directory = TempDir::new().unwrap(); + let path = directory.path().join(BOUNDARY_CONFIG_FILE); + // Missing and malformed state is persistent and fails admission; other + // read errors are deferred to the next pass. + for contents in [None, Some("invalid json"), Some("{}")] { + if let Some(contents) = contents { + fs::write(&path, contents).unwrap(); + } + let error = read_docker_cdi_context(&path).await.unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + } + fs::remove_file(&path).unwrap(); + fs::create_dir(&path).unwrap(); + assert_eq!( + read_docker_cdi_context(&path).await.unwrap_err().code(), + tonic::Code::Unavailable + ); +} + #[test] fn admission_defaults_reject_even_gpu_driver_json_but_not_public_gpu_requests() { let mut config = runtime_config(); @@ -221,6 +612,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { }), gpu: DockerGpuRuntimeCapabilities { cdi_supported: false, + cdi_spec_dirs: Vec::new(), wsl_all_gpu_fallback_enabled: false, }, sandbox_pids_limit: openshell_core::config::default_sandbox_pids_limit(), @@ -394,6 +786,571 @@ fn sandbox_driver_config_cannot_override_proxy_ca_bundle() { assert!(error.message().contains("proxy_ca_bundle"), "{error}"); } +fn runtime_config_with_cdi_spec_dirs(cdi_spec_dirs: &[&str]) -> DockerDriverRuntimeConfig { + let mut config = runtime_config(); + config.gpu.cdi_supported = !cdi_spec_dirs.is_empty(); + config.gpu.cdi_spec_dirs = cdi_spec_dirs + .iter() + .map(|path| (*path).to_string()) + .collect(); + config +} + +#[test] +fn gpu_runtime_builds_cdi_context_from_selected_devices_and_daemon_dirs() { + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR, TEST_CDI_SPEC_DIR_ALT]); + let selected_devices = vec![ + "nvidia.com/gpu=0".to_string(), + "nvidia.com/gpu=1".to_string(), + ]; + + let context = config + .gpu + .cdi_context(Some(&selected_devices)) + .unwrap() + .expect("GPU selection should produce a CDI context"); + + assert_eq!(context.selected_devices, selected_devices); + assert_eq!( + context.spec_dirs, + vec![ + CdiSpecDirectory::new(cdi_spec_mount_path(0), TEST_CDI_SPEC_DIR), + CdiSpecDirectory::new(cdi_spec_mount_path(1), TEST_CDI_SPEC_DIR_ALT), + ] + ); +} + +#[test] +fn gpu_runtime_omits_cdi_projection_without_selected_devices() { + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR]); + assert_eq!(config.gpu.cdi_context(None).unwrap(), None); +} + +#[test] +fn gpu_runtime_builds_read_only_cdi_bind_mounts() { + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR, TEST_CDI_SPEC_DIR_ALT]); + let context = config + .gpu + .cdi_context(Some(&["nvidia.com/gpu=0".into()])) + .unwrap() + .unwrap(); + let mut body = ContainerCreateBody::default(); + retain_cdi_spec_projections(&mut body, &context).unwrap(); + let mounts = body.host_config.unwrap().mounts.unwrap(); + assert_eq!(mounts.len(), 2); + for (index, (mount, source)) in mounts + .iter() + .zip([TEST_CDI_SPEC_DIR, TEST_CDI_SPEC_DIR_ALT]) + .enumerate() + { + assert_eq!(mount.typ, Some(MountTypeEnum::BIND)); + assert_eq!(mount.source.as_deref(), Some(source)); + assert_eq!( + mount.target.as_deref(), + Some(cdi_spec_mount_path(index).as_str()) + ); + assert_eq!(mount.read_only, Some(true)); + let options = mount.bind_options.as_ref().unwrap(); + assert_eq!(options.create_mountpoint, Some(false)); + assert_eq!(options.non_recursive, Some(true)); + assert_eq!( + options.propagation, + Some(MountBindOptionsPropagationEnum::RPRIVATE) + ); + } +} + +#[test] +fn cdi_projections_delegate_missing_source_checks_to_the_daemon() { + // The source is a daemon-host path, not a gateway-local path. It must not + // be passed as a legacy bind that would create it on the daemon host. + let config = runtime_config_with_cdi_spec_dirs(&["/missing/openshell/cdi-specs"]); + let mut sandbox = test_sandbox(); + sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); + let body = build_container_create_body_with_gpu_devices( + &sandbox, + &config, + &DockerSandboxDriverConfig::default(), + Some(&["nvidia.com/gpu=0".into()]), + ) + .unwrap(); + let host = body.host_config.unwrap(); + assert!( + host.binds + .unwrap_or_default() + .iter() + .all(|bind| !bind.contains("cdi-specs")) + ); + let mount = host + .mounts + .unwrap() + .into_iter() + .find(|mount| mount.source.as_deref() == Some("/missing/openshell/cdi-specs")) + .unwrap(); + assert_eq!(mount.bind_options.unwrap().create_mountpoint, Some(false)); +} + +#[test] +fn cdi_projection_mounts_match_probed_context_without_changing_gpu_selection() { + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR, TEST_CDI_SPEC_DIR_ALT]); + let devices = vec!["nvidia.com/gpu=0".into()]; + let mut body = build_container_create_body_with_gpu_devices( + &test_sandbox(), + &config, + &DockerSandboxDriverConfig::default(), + Some(&devices), + ) + .unwrap(); + let mut context = config.gpu.cdi_context(Some(&devices)).unwrap().unwrap(); + context.spec_dirs.remove(0); + let host = body.host_config.as_mut().unwrap(); + host.mounts.as_mut().unwrap().push(Mount { + typ: Some(MountTypeEnum::BIND), + source: Some(TEST_CDI_SPEC_DIR.into()), + target: Some("/unrelated".into()), + ..Default::default() + }); + // A stale projection for a directory the probe dropped must be removed. + host.mounts + .as_mut() + .unwrap() + .push(cdi_spec_mount(&cdi_spec_mount_path(0), TEST_CDI_SPEC_DIR).unwrap()); + let requests = host.device_requests.clone(); + retain_cdi_spec_projections(&mut body, &context).unwrap(); + let host = body.host_config.unwrap(); + assert_eq!(host.device_requests, requests); + let mounts = host.mounts.unwrap(); + assert!( + !mounts + .iter() + .any(|mount| mount.target.as_deref() == Some(cdi_spec_mount_path(0).as_str())) + ); + assert!(mounts.iter().any(|mount| mount.target.as_deref() + == Some(cdi_spec_mount_path(1).as_str()) + && mount.source.as_deref() == Some(TEST_CDI_SPEC_DIR_ALT))); + assert!( + mounts + .iter() + .any(|mount| mount.target.as_deref() == Some("/unrelated")) + ); + assert!( + mounts + .iter() + .any(|mount| mount.typ == Some(MountTypeEnum::VOLUME)) + ); +} + +#[test] +fn cdi_projections_follow_live_daemon_directories_over_startup_snapshot() { + // Startup reported only the primary directory; the daemon now reports the + // alternate one. The context and the mounts must both follow the daemon. + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR]); + let devices = vec!["nvidia.com/gpu=0".to_string()]; + let mut cache = CdiSpecDirectoryCache::default(); + let info: SystemInfo = serde_json::from_value(serde_json::json!({ + "ID": "daemon-a", + "CDISpecDirs": [TEST_CDI_SPEC_DIR_ALT], + })) + .unwrap(); + cache.observe_daemon(&info); + + let context = cdi_context_for_spec_dirs(Some(&devices), &cache.spec_dirs) + .unwrap() + .unwrap(); + assert_eq!( + context.spec_dirs, + vec![CdiSpecDirectory::new( + cdi_spec_mount_path(0), + TEST_CDI_SPEC_DIR_ALT + )] + ); + + let mut body = build_container_create_body_with_gpu_devices( + &test_sandbox(), + &config, + &DockerSandboxDriverConfig::default(), + Some(&devices), + ) + .unwrap(); + retain_cdi_spec_projections(&mut body, &context).unwrap(); + let projections = + body.host_config + .unwrap() + .mounts + .unwrap() + .into_iter() + .filter(|mount| { + mount.target.as_deref().is_some_and(|target| { + target.starts_with(openshell_core::cdi::CDI_SPEC_DIR_BASE) + }) + }) + .collect::>(); + assert_eq!(projections.len(), 1); + assert_eq!( + projections[0].source.as_deref(), + Some(TEST_CDI_SPEC_DIR_ALT) + ); + assert_eq!(projections[0].read_only, Some(true)); +} + +#[test] +fn stale_cdi_probe_sweeps_remove_anonymous_volumes() { + let probe = auxiliary_container_remove_options(Some(LABEL_ISOLATION_ROLE_CDI_PROBE)); + assert!(probe.force); + assert!(probe.v); + let identity = auxiliary_container_remove_options(Some(LABEL_ISOLATION_ROLE_IDENTITY)); + assert!(identity.force); + assert!(!identity.v); +} + +#[test] +fn cdi_context_requires_daemon_reported_directories() { + let devices = vec!["nvidia.com/gpu=0".to_string()]; + assert_eq!( + cdi_context_for_spec_dirs(Some(&devices), &[]) + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + assert_eq!(cdi_context_for_spec_dirs(None, &[]).unwrap(), None); +} + +#[tokio::test] +async fn cdi_directory_cache_filters_missing_sources_and_expires() { + let config = runtime_config_with_cdi_spec_dirs(&["/missing/cdi", TEST_CDI_SPEC_DIR]); + let context = config + .gpu + .cdi_context(Some(&["nvidia.com/gpu=0".into()])) + .unwrap() + .unwrap(); + let mut cache = CdiSpecDirectoryCache::default(); + let now = Instant::now(); + let probes = std::cell::Cell::new(0); + let available = cache + .available_context(context.clone(), now, |dir| { + probes.set(probes.get() + 1); + std::future::ready(Ok(dir.source != "/missing/cdi")) + }) + .await + .unwrap(); + assert_eq!(probes.get(), 2); + assert_eq!( + available.spec_dirs, + vec![CdiSpecDirectory::new( + cdi_spec_mount_path(1), + TEST_CDI_SPEC_DIR + )] + ); + assert_eq!(available.selected_devices, context.selected_devices); + cache + .available_context(context.clone(), now + Duration::from_secs(1), |_| { + std::future::ready(Err(Status::internal( + "fresh cache entry unexpectedly probed", + ))) + }) + .await + .unwrap(); + let available = cache + .available_context(context, now + CDI_SPEC_DIRECTORY_CACHE_TTL, |_| { + probes.set(probes.get() + 1); + std::future::ready(Ok(true)) + }) + .await + .unwrap(); + assert_eq!(probes.get(), 4); + assert_eq!(available.spec_dirs.len(), 2); +} + +#[tokio::test] +async fn cdi_directory_cache_does_not_cache_probe_errors() { + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR]); + let context = config + .gpu + .cdi_context(Some(&["nvidia.com/gpu=0".into()])) + .unwrap() + .unwrap(); + let mut cache = CdiSpecDirectoryCache::default(); + let probes = std::cell::Cell::new(0); + for _ in 0..2 { + let result = cache + .available_context(context.clone(), Instant::now(), |_| { + probes.set(probes.get() + 1); + std::future::ready(Err(Status::unavailable("daemon failure"))) + }) + .await; + assert!(result.is_err()); + assert!(cache.entries.is_empty()); + } + assert_eq!(probes.get(), 2); + let result = cache + .available_context(context, Instant::now(), |_| std::future::ready(Ok(false))) + .await; + assert_eq!(result.unwrap_err().code(), tonic::Code::FailedPrecondition); +} + +#[test] +fn cdi_directory_cache_invalidates_on_daemon_discovery_changes() { + let mut cache = CdiSpecDirectoryCache::default(); + let mut value = serde_json::json!({"ID":"daemon-a", "CDISpecDirs":[TEST_CDI_SPEC_DIR], + "DiscoveredDevices":[{"ID":"nvidia.com/gpu=0", "Source":"cdi"}]}); + let info: SystemInfo = serde_json::from_value(value.clone()).unwrap(); + cache.observe_daemon(&info); + cache + .entries + .insert(TEST_CDI_SPEC_DIR.into(), (Instant::now(), true)); + cache.observe_daemon(&info); + assert_eq!(cache.entries.len(), 1); + for change in ["daemon", "directories", "inventory"] { + match change { + "daemon" => value["ID"] = serde_json::json!("daemon-b"), + "directories" => value["CDISpecDirs"] = serde_json::json!([TEST_CDI_SPEC_DIR_ALT]), + "inventory" => { + value["DiscoveredDevices"][0]["ID"] = serde_json::json!("nvidia.com/gpu=1"); + } + _ => unreachable!(), + } + cache.observe_daemon(&serde_json::from_value(value.clone()).unwrap()); + assert!(cache.entries.is_empty()); + cache + .entries + .insert(TEST_CDI_SPEC_DIR.into(), (Instant::now(), true)); + } +} + +async fn mock_cdi_probe_api(responses: Vec<(u16, String)>) -> (Docker, JoinHandle>) { + mock_cdi_probe_api_with_gate(responses, None).await +} + +async fn mock_cdi_probe_api_with_gate( + responses: Vec<(u16, String)>, + gate: Option<(Arc, Arc)>, +) -> (Docker, JoinHandle>) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + let mut requests = Vec::new(); + for (index, (status, body)) in responses.into_iter().enumerate() { + let (mut socket, _) = listener.accept().await.unwrap(); + openshell_core::net::set_tcp_nodelay_best_effort(&socket); + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + let count = socket.read(&mut buffer).await.unwrap(); + assert!(count > 0); + request.extend_from_slice(&buffer[..count]); + if let Some(end) = request.windows(4).position(|part| part == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&request[..end]); + let length = headers + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + if request.len() >= end + 4 + length { + break; + } + } + } + requests.push(String::from_utf8(request).unwrap()); + if index == 0 + && let Some((received, release)) = &gate + { + received.notify_one(); + release.notified().await; + } + let response = format!( + "HTTP/1.1 {status} Result\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(response.as_bytes()).await.unwrap(); + } + requests + }); + let docker = Docker::connect_with_http( + &format!("http://{address}"), + 5, + bollard::API_DEFAULT_VERSION, + ) + .unwrap(); + (docker, task) +} + +#[tokio::test] +async fn cdi_probe_uses_prepared_image_one_bind_and_removes_volumes_without_starting() { + let (docker, requests) = mock_cdi_probe_api(vec![ + (201, r#"{"Id":"probe-test","Warnings":[]}"#.into()), + (204, String::new()), + ]) + .await; + assert!( + probe_cdi_spec_directory( + &docker, + "sha256:prepared-workload", + HashMap::from([( + LABEL_ISOLATION_ROLE.to_string(), + LABEL_ISOLATION_ROLE_CDI_PROBE.to_string(), + )]), + CdiSpecDirectory::new(cdi_spec_mount_path(0), TEST_CDI_SPEC_DIR) + ) + .await + .unwrap() + ); + let requests = requests.await.unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].starts_with("POST ")); + assert!(requests[0].contains("/containers/create")); + let body: serde_json::Value = + serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body["Image"], "sha256:prepared-workload"); + assert_eq!( + body["Labels"][LABEL_ISOLATION_ROLE], + LABEL_ISOLATION_ROLE_CDI_PROBE + ); + assert_eq!(body["HostConfig"]["NetworkMode"], "none"); + let mounts = body["HostConfig"]["Mounts"].as_array().unwrap(); + assert_eq!(mounts.len(), 1); + assert_eq!(mounts[0]["Source"], TEST_CDI_SPEC_DIR); + assert_eq!(mounts[0]["ReadOnly"], true); + assert_eq!(mounts[0]["BindOptions"]["CreateMountpoint"], false); + assert_eq!(mounts[0]["BindOptions"]["Propagation"], "rprivate"); + assert!(body.get("Env").is_none_or(serde_json::Value::is_null)); + assert!(requests[1].starts_with("DELETE ")); + assert!(requests[1].contains("/containers/probe-test?")); + assert!(requests[1].contains("v=true")); +} + +#[tokio::test] +async fn cdi_probe_skips_only_its_missing_source_and_surfaces_other_failures() { + let missing = format!( + "invalid mount config for type \"bind\": bind source path does not exist: {TEST_CDI_SPEC_DIR}" + ); + let reworded = format!("bind source path does not exist: {TEST_CDI_SPEC_DIR}"); + for (code, message, expected_missing) in [ + (400, missing.as_str(), true), + (400, reworded.as_str(), true), + (403, "permission denied", false), + (500, "daemon failed", false), + ( + 400, + "invalid mount config for type \"bind\": bind source path does not exist: /unrelated", + false, + ), + ] { + let (docker, requests) = mock_cdi_probe_api(vec![( + code, + serde_json::json!({"message":message}).to_string(), + )]) + .await; + let result = probe_cdi_spec_directory( + &docker, + "image", + HashMap::new(), + CdiSpecDirectory::new(cdi_spec_mount_path(0), TEST_CDI_SPEC_DIR), + ) + .await; + if expected_missing { + assert!(!result.unwrap()); + } else { + assert!(result.is_err()); + } + assert_eq!(requests.await.unwrap().len(), 1); + } + let (docker, requests) = mock_cdi_probe_api(vec![ + (201, r#"{"Id":"probe-test","Warnings":[]}"#.into()), + (500, r#"{"message":"cleanup failed"}"#.into()), + ]) + .await; + // A probe that was created proves the directory exists, even if its + // removal fails; the leftover probe is removed with the sandbox. + assert!( + probe_cdi_spec_directory( + &docker, + "image", + HashMap::new(), + CdiSpecDirectory::new(cdi_spec_mount_path(0), TEST_CDI_SPEC_DIR), + ) + .await + .unwrap() + ); + assert_eq!(requests.await.unwrap().len(), 2); +} + +#[tokio::test] +async fn cdi_probe_cleans_up_after_provisioning_is_cancelled() { + let received = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let (docker, requests) = mock_cdi_probe_api_with_gate( + vec![ + (201, r#"{"Id":"probe-cancelled","Warnings":[]}"#.into()), + (204, String::new()), + ], + Some((received.clone(), release.clone())), + ) + .await; + let caller = tokio::spawn(async move { + probe_cdi_spec_directory( + &docker, + "image", + HashMap::new(), + CdiSpecDirectory::new(cdi_spec_mount_path(0), TEST_CDI_SPEC_DIR), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), received.notified()) + .await + .unwrap(); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + release.notify_one(); + let requests = tokio::time::timeout(Duration::from_secs(5), requests) + .await + .unwrap() + .unwrap(); + assert!(requests[1].starts_with("DELETE ")); + assert!(requests[1].contains("/containers/probe-cancelled?")); + assert!(requests[1].contains("v=true")); +} + +#[tokio::test] +#[ignore = "requires a local Docker daemon and the Ubuntu 24.04 test image"] +async fn live_docker_cdi_directory_probes_do_not_start_or_create_missing_sources() { + let directory = TempDir::new().unwrap(); + let docker = Docker::connect_with_local_defaults().unwrap(); + let image = "nvcr.io/nvidia/base/ubuntu:24.04"; + let existing = + CdiSpecDirectory::new(cdi_spec_mount_path(0), directory.path().to_string_lossy()); + let missing = directory.path().join("missing"); + assert!( + probe_cdi_spec_directory(&docker, image, HashMap::new(), existing) + .await + .unwrap() + ); + assert!( + !probe_cdi_spec_directory( + &docker, + image, + HashMap::new(), + CdiSpecDirectory::new(cdi_spec_mount_path(1), missing.to_string_lossy()) + ) + .await + .unwrap() + ); + assert!(!missing.exists()); +} + +#[test] +fn cdi_projections_reject_invalid_daemon_source_paths() { + for path in ["relative", "", " /etc/cdi", "/etc/cdi\0"] { + assert!( + cdi_spec_mount(&cdi_spec_mount_path(0), path).is_err(), + "{path}" + ); + } +} + fn test_workload_identity() -> ResolvedWorkloadIdentity { ResolvedWorkloadIdentity::new( 1234, @@ -442,6 +1399,7 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr CdiGpuInventory::default(), wsl_all_gpu_fallback_enabled, )), + cdi_spec_directory_cache: Arc::new(Mutex::new(CdiSpecDirectoryCache::default())), lifecycle_event_fences: DockerLifecycleEventFences::default(), control_processes: Arc::new(Mutex::new(HashMap::new())), runtime_failures: Arc::new(Mutex::new(HashMap::new())), @@ -1798,6 +2756,70 @@ fn container_creation_rejects_openshell_control_path_working_dir() { assert!(err.message().contains("OpenShell control path")); } +#[test] +fn container_creation_reserves_cdi_spec_projection_paths() { + let metadata = DockerImageMetadata { + id: "sha256:immutable".to_string(), + user: "1234:1235".to_string(), + working_dir: "/workspace".to_string(), + volumes: Vec::new(), + }; + // These mounts were previously dropped silently or failed as duplicates. + for mount in [ + serde_json::json!({"type": "tmpfs", "target": "/.openshell/cdi-specs/0"}), + serde_json::json!({"type": "volume", "source": "data", "target": "/.openshell/cdi-specs"}), + serde_json::json!({"type": "bind", "source": "/srv", "target": "/.openshell/cdi-specs/1"}), + serde_json::json!({"type": "tmpfs", "target": "/.openshell"}), + ] { + let driver_config: DockerSandboxDriverConfig = + serde_json::from_value(serde_json::json!({ "mounts": [mount.clone()] })).unwrap(); + let err = build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &driver_config, + None, + &metadata, + &test_workload_identity(), + ) + .unwrap_err(); + assert_eq!(err.code(), tonic::Code::FailedPrecondition, "{mount}"); + assert!( + err.message().contains("OpenShell control path"), + "{mount}: {err}" + ); + } + + let volume_metadata = DockerImageMetadata { + volumes: vec!["/.openshell/cdi-specs/0".to_string()], + ..metadata.clone() + }; + let err = build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &DockerSandboxDriverConfig::default(), + None, + &volume_metadata, + &test_workload_identity(), + ) + .unwrap_err(); + assert!(err.message().contains("OpenShell control path"), "{err}"); + + // Sibling paths are not reserved yet. + let sibling: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ + "mounts": [{"type": "tmpfs", "target": "/.openshell/cdi-specs-cache"}] + })) + .unwrap(); + build_container_create_body_for_image( + &test_sandbox(), + &runtime_config(), + &sibling, + None, + &metadata, + &test_workload_identity(), + ) + .unwrap(); +} + #[test] fn container_creation_rejects_image_volume_that_masks_working_dir() { let sandbox = test_sandbox(); @@ -2697,8 +3719,7 @@ fn validate_sandbox_auth_accepts_launch_authentication() { #[test] fn build_container_create_body_maps_default_gpu_to_selected_cdi_device() { - let mut config = runtime_config(); - config.gpu.cdi_supported = true; + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR]); let mut sandbox = test_sandbox(); sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); @@ -2723,6 +3744,22 @@ fn build_container_create_body_maps_default_gpu_to_selected_cdi_device() { request.device_ids.as_ref().unwrap(), &vec!["nvidia.com/gpu=1".to_string()] ); + let host = create_body.host_config.as_ref().unwrap(); + let mounts = host + .mounts + .as_ref() + .expect("GPU request should project CDI specs into the workload"); + let projection = mounts + .iter() + .find(|mount| mount.target.as_deref() == Some(cdi_spec_mount_path(0).as_str())) + .unwrap(); + assert_eq!(projection.source.as_deref(), Some(TEST_CDI_SPEC_DIR)); + assert_eq!(projection.read_only, Some(true)); + assert!( + host.binds + .as_ref() + .is_none_or(|binds| binds.iter().all(|bind| !bind.contains(TEST_CDI_SPEC_DIR))) + ); } #[test] @@ -2745,8 +3782,7 @@ fn build_container_create_body_omits_devices_without_resolved_default_cdi_device #[test] fn container_spec_accepts_non_nvidia_explicit_and_resolved_cdi_devices() { - let mut config = runtime_config(); - config.gpu.cdi_supported = true; + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR]); let mut sandbox = test_sandbox(); sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); for device in ["example.com/gpu=0", "intel.com/gpu=0", "amd.com/gpu=0"] { @@ -2830,8 +3866,7 @@ fn container_spec_rejects_host_paths_in_explicit_and_resolved_cdi_devices() { #[test] fn build_container_create_body_passes_explicit_cdi_device_id_through() { - let mut config = runtime_config(); - config.gpu.cdi_supported = true; + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR]); let mut sandbox = test_sandbox(); let spec = sandbox.spec.as_mut().unwrap(); spec.resource_requirements = Some(gpu_resources(None)); @@ -2901,8 +3936,7 @@ fn build_container_create_body_rejects_empty_cdi_devices() { #[test] fn driver_default_gpu_selection_consumes_distinct_devices_for_creates() { - let mut config = runtime_config(); - config.gpu.cdi_supported = true; + let config = runtime_config_with_cdi_spec_dirs(&[TEST_CDI_SPEC_DIR]); let driver = test_driver_with_config(config); driver.gpu_selector.refresh( CdiGpuInventory::new(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]), diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index b1a2559c20..a7457843c8 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -626,7 +626,11 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'sandbox_runtime_image = %s\n' "$(toml_string "${SANDBOX_RUNTIME_IMAGE}")" printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" printf '\n[openshell.drivers.docker.resource_admission]\n' - printf 'enabled = false\n' + if [ "${GPU_MODE}" = "1" ]; then + printf 'enabled = true\n' + else + printf 'enabled = false\n' + fi fi } > "${GATEWAY_CONFIG}" From db12abe128b466206943e6192549963c7953508d Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 14 Oct 2026 21:37:42 +0200 Subject: [PATCH 5/6] docs(gpu): document CDI-derived GPU sandbox policy Describe how Docker GPU sandboxes derive filesystem access from the selected CDI devices, which CDI mounts are rejected, the NVIDIA MPS pipe directory exception, missing additionalGids handling on Docker releases before 29.5, and SELinux requirements for the projected spec directories. Add cluster debugging entries for CDI projection, admission, probe and restart failures, and document the CDI context in the core, sandbox and Docker driver READMEs. Signed-off-by: Evan Lezar --- crates/openshell-core/README.md | 7 +++ crates/openshell-driver-docker/README.md | 37 ++++++++++- crates/openshell-sandbox/README.md | 51 +++++++++++++++ docs/how-it-works/policies/default-policy.mdx | 62 ++++++++++++++++--- docs/how-it-works/sandboxes/overview.mdx | 6 ++ docs/how-it-works/sandboxes/runtimes.mdx | 25 ++++++++ skills/debug-openshell-cluster/SKILL.md | 13 ++++ 7 files changed, 192 insertions(+), 9 deletions(-) diff --git a/crates/openshell-core/README.md b/crates/openshell-core/README.md index b4cdb05f5d..f377e7001e 100644 --- a/crates/openshell-core/README.md +++ b/crates/openshell-core/README.md @@ -64,3 +64,10 @@ Use `openshell_core::proto_struct` when crossing between `serde_json` values and `prost_types::{Struct, Value}`. Both conversion directions live in this module; JSON-to-protobuf conversion is fallible so callers cannot silently replace an unrepresentable number. + +## CDI context + +The shared `cdi` module defines the versioned context passed from compute drivers +to the workload, including selected devices and projected specification directories. +The CDI resolver and its dependency live in `openshell-sandbox`; see that crate's +README for resource validation and enforcement. diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index c4a551caa0..5044ec44c3 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -93,7 +93,8 @@ traverse every parent and write and enter the workdir; OpenShell does not change its ownership or mode. Image `VOLUME` declarations and user mounts must not cover the workdir, one of -its parents, or the reserved `/.openshell` runtime/channel tree. OpenShell asks +its parents, or the reserved `/.openshell/channel` and `/.openshell/cdi-specs` +paths. OpenShell asks the kernel to validate access under the final identity, so POSIX ACL and host LSM decisions remain authoritative. @@ -111,6 +112,7 @@ LSM decisions remain authoritative. | Private named volumes | One carries the authenticated sandbox/supervisor channel. The other is mounted only into the supervisor and contains its JWT and private gateway credentials. | | In-memory `/run/openshell-supervisor-ca` tmpfs | Holds only the public supervisor CA certificate and trust bundle without making all of `/run` writable. | | CDI GPU request | Assigns the exact validated CDI devices requested by driver config or count-based selection. | +| CDI workload projection | For GPU sandboxes only, embeds selected device metadata in the protected boundary config and mounts Docker daemon CDI spec directories read-only into the workload. | ## Stop, Start, and Delete @@ -124,6 +126,39 @@ Delete force-removes both containers, the driver-owned runtime volumes, and the host-private runtime descriptor. Missing or altered descriptor and channel resources fail closed; the driver does not run an older combined-supervisor layout. +## CDI GPU Metadata + +Docker remains the source of truth for GPU injection. The driver selects opaque +CDI device IDs from `driver_config.cdi_devices` or the daemon's discovered CDI +inventory, then passes the same IDs to Docker with a CDI `DeviceRequest`. + +For a GPU sandbox, the driver writes those IDs and projected CDI spec directory +paths to a versioned context in the protected workload boundary configuration. +It uses structured, read-only, non-recursive binds at `/.openshell/cdi-specs/`. +The driver checks source existence on the daemon host using one temporary, +stopped container per directory and the prepared workload image. Each probe has +one read-only, private bind with source creation disabled. Successful probes are +removed with anonymous image-volume cleanup, including after provisioning +cancellation. Confirmed missing directories are omitted; other errors fail +provisioning. Results are cached for 30 seconds and refreshed on demand; observed +changes to daemon identity, configured directories, or GPU inventory invalidate +the cache. The resulting context records only available projections. The actual +sandbox is created once, and selected devices must still resolve from the +projected specs. No probe starts a process, creates missing directories, or +relabels host spec directories. Administrators must make the existing directories readable +under the host's SELinux policy when SELinux is enabled. + +Whole spec directories are available to the trusted runtime, which resolves only +the selected devices. The mandatory Landlock baseline hides these projections +from agent processes, even when the authored policy grants a broad ancestor. +The sandbox runtime resolves the selected devices in that mount namespace before +it launches agent processes, derives Landlock paths from CDI `containerEdits`, +and warns when the runtime did not apply the spec's `additionalGids`. Containers with CDI context fail startup if their +selected devices cannot be resolved. Existing GPU containers without context +retain legacy baseline handling on restart. Admission checks persisted context +for existing containers and the creation context for new ones. Non-GPU sandboxes +receive no CDI context or spec mounts. + ## Driver Config Mounts The gateway forwards the `docker` block from `--driver-config-json`. Supported diff --git a/crates/openshell-sandbox/README.md b/crates/openshell-sandbox/README.md index 3ee57c085f..3d3502eed0 100644 --- a/crates/openshell-sandbox/README.md +++ b/crates/openshell-sandbox/README.md @@ -17,3 +17,54 @@ the kernel then tracks the lifetime through workload-owned descriptors. Native the last workload reference to a listener removes it without waiting for registry collection. Broker source descriptors always use close-on-exec independently of the workload's requested descriptor flags. + +## CDI policy provenance + +The workload resolves trusted CDI context into filesystem requirements. Library +mounts expand to their parent directory; the expanded path must pass the same +broad-root rejection as other CDI destinations. Specs are trusted host +administrator input. A shared-library parent grant also makes sibling files +readable, so administrators must choose library directories accordingly, including +when a spec places libraries under `/etc`, `/home`, or `/root`. These checks do +not turn administrator-authored specs into an untrusted-input interface. +CDI enrichment adds only spec-derived paths; it does not grant all of `/sys`. + +`WorkloadLaunchContext` retains derived filesystem requirements and required GIDs +separately from the authored policy. The initial process launcher and retained +exec executor share this context. Protobuf and boundary policy messages carry +no derived requirements or GIDs; the workload resolves them from trusted CDI context. +The sandbox processes deduplicated CDI targets sequentially: open a target, +validate its type and broad-root object identity, add matching Landlock rules +using that same descriptor, and close it before opening the next target. +Targets covered by authored ancestors are validated without adding a rule. +Only path requirements and root identities are retained; the number of open +CDI descriptors does not grow with the spec. Symlinks to legitimate library +directories remain supported. CDI validation and enforcement failures abort launch, including +when the authored Landlock compatibility is best effort. Authored paths retain +their configured opening behavior; absent authored paths can still be skipped. +Ruleset construction and enforcement remain mandatory when CDI filesystem targets +are present, so failures cannot silently discard CDI confinement. + +Projected specs reside under `/.openshell/cdi-specs`, which the mandatory baseline +excludes from agent access independently of authored grants. Fixture tests use +the production resolver with injected workload mount observations; real Linux +mount-state tests verify the OS inspection separately. + +Original CDI mount destinations remain separate from expanded library parents. +The workload classifies their effective read-only flags rather than trusting +spec option defaults, and checks original directory mounts for writable +submounts using one mount-table snapshot per pass. Ordinary writable file and +directory mounts are rejected, even when an exact or ancestor `read_write` +grant covers the destination. Dedicated authorization for CDI host writes is +not currently supported. The sequential Landlock pass rechecks mount access +on its opened descriptors and rejects persisted writable requirements too. +CDI does not add write access to `/proc`; existing thread-name mediation applies. +Required groups must already be supplied by the container runtime. The sandbox +checks membership before both the initial process launch and subsequent execs. + +CDI Unix socket mounts are recorded as exact socket destinations, separately from +ordinary file mounts. Their opened objects must remain Unix sockets at each +launch or exec. They do not derive filesystem grants, expand to parent +directories, or require writable filesystem grants, regardless of mount flags. +Communication follows the existing Unix IPC and inode permission checks; this +validation does not pin the endpoint for later pathname-based connections. diff --git a/docs/how-it-works/policies/default-policy.mdx b/docs/how-it-works/policies/default-policy.mdx index 3f148dcf70..d82696bba7 100644 --- a/docs/how-it-works/policies/default-policy.mdx +++ b/docs/how-it-works/policies/default-policy.mdx @@ -87,18 +87,58 @@ the sandbox's policy. ### GPU Sandboxes -On the Docker and VM compute drivers, a sandbox that requests a GPU receives -additional paths when the corresponding GPU device is present: +GPU sandboxes without a CDI context receive additional paths when the +corresponding GPU device is present: | Access | Paths | |---|---| | Read-only | `/run/nvidia-persistenced`, `/usr/lib/wsl` | -| Read-write | `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, `/dev/nvidia-modeset`, `/dev/dxg`, numbered `/dev/nvidia` device nodes, and `/proc` | - -CUDA writes thread names under `/proc` during initialization, so GPU enrichment -moves `/proc` from read-only to read-write. OpenShell adds each path only when -it exists in the workload. These paths apply at runtime and are not saved in -the sandbox's policy. +| Read-write | `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, `/dev/nvidia-modeset`, `/dev/dxg`, numbered `/dev/nvidia` device nodes | + +CUDA thread-name writes under `/proc` are handled by the existing process +open mediation. GPU enrichment preserves authored `/proc` access instead of +adding a blanket write grant. OpenShell adds each path +only when it exists in the workload. These paths apply at runtime and are not +saved in the sandbox's policy. + +Docker GPU sandboxes receive CDI-derived filesystem +requirements instead of the hard-coded GPU device and library baseline. The +workload-side sandbox runtime resolves the selected CDI device specs and adds +CDI device nodes as read-write paths; listing a device node in +`filesystem_policy.read_only` stops the launch. For read-only mounts, it adds the parent +directories of shared libraries whose filenames end in `.so` or a numeric +SONAME suffix with any number of dot-separated components, such as `.so.1`, +`.so.1.2`, or `.so.1.2.3`. It retains exact destinations for other read-only +mounts. OpenShell does not add a derived path when an existing read-only or +read-write parent already covers it. The container runtime supplies CDI `additionalGids` as supplemental groups. +Docker releases before 29.5 ignore them. OpenShell checks the workload's groups +before initial launch and subsequent execs and logs a warning naming any missing +group. The process still starts, but devices restricted to that group may be +inaccessible. + +Unix socket mounts are validated as sockets at their exact destinations at each +launch or exec. They do not add filesystem grants or parent directory access and +do not require `read_write` entries, regardless of mount flags. Communication +uses the existing Unix IPC and socket permission checks. + +Ordinary CDI file and directory mounts must be effectively read-only. OpenShell +rejects writable mounts even when `filesystem_policy.read_write` names their +exact destination or an ancestor. Explicit authorization for CDI host writes is +not currently supported. This restriction does not apply to device nodes or +Unix socket communication. + +The NVIDIA MPS pipe directory `/tmp/nvidia-mps` is the only writable directory +mount allowed. When CDI mounts it with `nosuid`, `nodev`, and `noexec`, OpenShell +grants read-write access to that exact directory so MPS clients can open its +pipes. OpenShell checks the directory and its mount flags at each launch or +exec. Listing it in `filesystem_policy.read_only` stops the launch. + +CDI paths such as `/`, `/dev`, `/proc`, `/sys`, `/run`, or `/usr` remain rejected +during sandbox startup, even when authored policy grants access to those roots. + +CDI does not add a blanket `/sys` grant. If an application needs sysfs paths +that the selected CDI spec does not supply, include those paths in your authored +filesystem policy. ### Protected Paths @@ -108,6 +148,12 @@ filesystem policy is applied on top of that baseline and can narrow access, but it cannot expose `/.openshell`. The `best_effort` compatibility setting does not disable this protection or allow a kernel without ABI v3. +CDI targets are validated strictly before each launch or exec. Authored paths +retain their configured opening behavior, so an absent authored path can still +be skipped in best-effort mode. When CDI filesystem targets are present, +ruleset construction and enforcement failures abort launch instead of falling +back to an unrestricted workload. + ## Inspect the Selected Policy To see which policy a sandbox uses, print its base and effective policies. diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index f10cccb561..024395252f 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -155,6 +155,12 @@ For Docker-backed sandboxes, GPU injection uses Docker CDI. If you enable Docker CDI after the gateway starts, restart the gateway so OpenShell can detect the updated Docker daemon capability. +For CDI-enabled Docker sandboxes, OpenShell derives filesystem access from the +selected CDI devices and validates those targets before each launch and exec. +Invalid targets stop the launch. Refer to +[GPU Sandboxes](/how-it-works/policies/default-policy#gpu-sandboxes) for the +paths OpenShell adds and the mounts it rejects. + Docker and Podman refresh the CDI inventory before validating or creating a default GPU request, so CDI devices added or removed after the driver starts can be reflected in later sandbox creates. On WSL2 all-only runtimes, the default diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index cc76cf98d2..3e9058255b 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -121,6 +121,31 @@ Common options in `[openshell.drivers.docker]` are `socket_path`, `grpc_endpoint Docker Desktop must have host networking enabled, and it cannot use Enhanced Container Isolation. Set `grpc_endpoint` when sandboxes cannot reach the gateway on host loopback. For GPU sandboxes, configure Docker CDI before starting the gateway. +For Docker GPU/CDI sandboxes, OpenShell uses Docker's selected CDI device IDs +and daemon-reported CDI spec directories to build a workload CDI context. The +driver embeds the context in the protected boundary configuration and +projects the spec directories read-only under the protected `/.openshell/cdi-specs` +hierarchy. User mounts and image volumes cannot target this hierarchy. These +structured binds neither create missing host directories nor relabel existing +directories. OpenShell skips spec directories that do not exist on the Docker +host. At least one directory must remain and every selected device must resolve from +its specs. On SELinux hosts, the host policy must allow containers to read the +directories; a denial fails the workload launch. Agent processes cannot read the +projections, even with a broad authored filesystem grant. The sandbox runtime derives filesystem requirements from CDI specs and +checks supplemental groups before launching agent processes. Non-GPU Docker +sandboxes do not receive the CDI context, spec mounts, or CDI-derived policy +changes. + +Existing Docker GPU containers without CDI context retain the legacy GPU +filesystem baseline on restart. Admission requires spec projections only when +the driver-owned boundary configuration contains CDI context, and validates +those projections against the directories recorded at creation. Missing or +unreadable boundary configuration blocks restart. To use CDI-derived access on +an older sandbox, recreate it. Ordinary CDI file and directory mounts must be +effectively read-only. Filesystem `read_write` grants do not authorize writable +CDI mounts; explicit authorization for CDI host writes is not currently +supported. Device nodes and Unix socket communication remain supported. + ### Docker Corporate Proxy Egress For proxy-required networks, the Docker driver accepts `https_proxy`, diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index f2203afdc5..8871071365 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -324,6 +324,19 @@ Common findings: - A GPU sandbox fails because Docker reports no discovered NVIDIA CDI devices: verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. Restart `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs, then restart or reload Docker and re-check `docker info`. - Docker or Podman rejects an explicit `cdi_devices` selector: use a CDI qualified name (`/=`), not a host-device path. Syntax validation accepts other vendors; it does not establish that the runtime has the device or that its GPU workload is supported. Consult the published GPU selection documentation for request requirements. +- A CDI-enabled Docker workload fails during filesystem validation: inspect the destination paths in the selected CDI specs and their targets inside the workload image. Shared-library parent directories must not alias broad roots such as `/`, `/usr`, or `/dev`; device destinations must resolve to character or block devices, and ordinary file and directory mounts must be effectively read-only; writable mounts are rejected even with exact or ancestor policy `read_write` grants. Check both initial launch and later exec failures. CDI target validation, ruleset construction, and enforcement failures are fatal even with best-effort Landlock compatibility; unrelated authored paths retain their configured opening behavior; correct the spec or image paths rather than relaxing the policy. + +- A Docker GPU sandbox fails restart with `CDI spec projection is missing`: its driver-owned boundary configuration contains CDI context, but the required projection is absent. Check the original projected sources and destinations; a later Docker CDI directory configuration change does not change the recorded expectations. Older GPU containers without CDI context retain the legacy filesystem baseline and do not require projections. Missing or unreadable boundary configuration blocks restart instead of selecting legacy handling. Recreate an older sandbox only when you want CDI-derived access. +- A Docker GPU sandbox fails to start with `bind source path does not exist` for a CDI spec directory: the directory recorded at creation is gone, for example `/var/run/cdi` after a reboot. Regenerate the specs (`nvidia-ctk cdi generate` or the `nvidia-cdi-refresh` service) and start the sandbox again, or recreate it. + +- Docker rejects a CDI spec-directory bind: inspect the directories reported in `docker info --format '{{json .CDISpecDirs}}'`. OpenShell probes each source with one temporary stopped container using the prepared workload image, removes the probe and its anonymous volumes, and omits confirmed missing sources before creating the actual sandbox once. Probe containers carry the `openshell.ai/isolation-role=cdi-probe` label; a probe that could not be removed is deleted with its sandbox or at the next gateway startup. Results are cached for 30 seconds and refreshed on demand; changed daemon identity, directories, or GPU inventory observed during refresh invalidate the cache. Probe errors are reported as `CdiSpecDirectoryProbeFailed`. A source that disappears after probing can still cause `ContainerCreateFailed`; this is not retried. At least one directory must remain, and the selected devices must resolve from its specs. Permission errors and unrelated bind failures remain fatal. Structured binds do not create missing directories or relabel existing ones. On SELinux hosts, ensure the host policy permits containers to read them without automatic relabeling; a denial surfaces when the workload launches. The projections are private to the trusted runtime under `/.openshell/cdi-specs`; the mandatory baseline prevents agent access even with broad authored grants. +- CDI spec projection admission fails: inspect the workload container's `HostConfig.DeviceRequests` and `Mounts`. Each CDI spec directory recorded in the driver-owned boundary configuration must appear at its assigned `/.openshell/cdi-specs/` destination as a read-only bind with private propagation and the expected host source. Missing, changed, duplicate, and additional unrelated bind mounts are rejected with admission enabled. Correct the projection or runtime configuration; do not disable resource admission for GPU workloads. +- CDI writable mount validation fails despite omitted options or `ro` in the spec: OpenShell checks effective mount flags inside the workload. Original directory destinations are also checked for writable submounts. Ordinary writable file and directory destinations are rejected; dedicated CDI host-write authorization is not currently supported; expanded shared-library parents are not treated as original directory mounts. +- CUDA topology discovery fails: CDI does not add blanket read access to `/sys`. Check the paths requested by the application and add the required sysfs paths to its authored filesystem policy when the selected CDI spec does not supply them. + +- CDI socket validation fails: verify the exact destination is a Unix socket inside the workload. Socket destinations (including GPU daemon endpoints) do not expand to their parents or require writable filesystem grants. A socket replaced by a regular file or directory fails launch or subsequent exec. +- GPU device access fails and the sandbox logs `CDI groups not applied by workload runtime`: compare the selected spec's `additionalGids` with the container process's actual supplemental groups (`id -G` or `/proc//status`). Docker releases before 29.5 ignore CDI `additionalGids`; upgrade older daemons and restart the sandbox. Numeric groups need no matching group name. OpenShell checks membership before initial launch and subsequent execs, logs the missing groups, and does not inject them. + #### Corporate upstream proxy Docker corporate proxy settings are operator-owned fields under From 33350a881f55011286e7c3a68d8007e43e50e0bf Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 16 Oct 2026 00:05:03 +0200 Subject: [PATCH 6/6] test(cdi): update CDI scenario expectations for derived policy Record the behavior of the named scenarios once filesystem access is derived from the selected CDI specs: read-only mounts and library parents are readable, device nodes are writable, and writable mounts, an unsafe library parent, and an NVIDIA MPS directory without noexec are rejected. Add scenarios for the MPS pipe directory with and without noexec. Signed-off-by: Evan Lezar --- TESTING.md | 33 ++++++------ .../roles/cdi_fixtures/vars/scenarios.yaml | 50 +++++++++++++------ tests/suites/features/cdi/tests/injection.rs | 2 + 3 files changed, 53 insertions(+), 32 deletions(-) diff --git a/TESTING.md b/TESTING.md index 96d29c001e..9b9fccb975 100644 --- a/TESTING.md +++ b/TESTING.md @@ -359,19 +359,21 @@ playbook restores the configuration and restarts the gateway even after test failure; the CDI fixtures remain part of the environment. Docker must support CDI. Branch and release feature-test matrices run this suite on Docker. -| Scenario | Injected resource | Asserted today | Expected with PR #4204 | -| --- | --- | --- | --- | -| `mount-under-read-only-baseline` | File at `/usr/share/tmachine-cdi/baseline-read-only-file`, under a directory the sandbox already makes read-only | Read succeeds, write denied. | Unchanged. | -| `read-only-file` | File at `/tmachine-cdi/read-only/file` | Read and write denied. | Read succeeds outside the baseline grants; write fails. | -| `read-only-directory` | Directory at `/tmachine-cdi/read-only/directory` | Reading its marker and creating files denied. | Reading its marker succeeds; write fails. | -| `library-parent` | Marker named `/tmachine-cdi/library/libfixture.so.1` | Read and write denied. | Resolver derives the library parent; the marker is not an actual shared library. | -| `broad-library-parent` | Marker named `/libtmachinefixture.so.1` | Read and write denied. | Launch rejects expansion to `/`. | -| `writable-file` | Writable file at `/tmachine-cdi/writable/file` | Read and write denied. | Launch rejects it even with exact or ancestor write grants. | -| `writable-directory` | Writable directory at `/tmachine-cdi/writable/directory` | Write denied. | Launch rejects it even with exact or ancestor write grants. | -| `omitted-access-file` | Bind without `ro` or `rw` | Read and write denied. | Effective writable access causes launch rejection. | -| `device-node` | Null character device at `/dev/tmachine-cdi-null` | Non-root, write denied. | Non-root read/write succeeds; an exact authored read-only entry fails launch. | -| `device-node-with-gid` | Group-restricted null device at `/dev/tmachine-cdi-group-null` | Non-root with the device's group, write denied. | The generated supplementary GID permits non-root access. | -| `unix-socket` | Echo socket at `/tmachine-cdi/socket/echo.sock` | Sibling file in the host directory is not visible. | Socket communication succeeds without deriving parent filesystem grants. | +| Scenario | Injected resource | Asserted | +| --- | --- | --- | +| `mount-under-read-only-baseline` | File at `/usr/share/tmachine-cdi/baseline-read-only-file`, under a directory the sandbox already makes read-only | Read succeeds, write denied. | +| `read-only-file` | File at `/tmachine-cdi/read-only/file` | Read succeeds from the derived grant; write denied. | +| `read-only-directory` | Directory at `/tmachine-cdi/read-only/directory` | Reading its marker succeeds; creating files denied. | +| `library-parent` | Marker named `/tmachine-cdi/library/libfixture.so.1` | The derived library-parent grant makes it readable; write denied. The marker is not an actual shared library. | +| `broad-library-parent` | Marker named `/libtmachinefixture.so.1` | Launch rejects expansion to `/`. | +| `writable-file` | Writable file at `/tmachine-cdi/writable/file` | Launch rejects the writable mount. | +| `writable-directory` | Writable directory at `/tmachine-cdi/writable/directory` | Launch rejects the writable mount. | +| `omitted-access-file` | Bind without `ro` or `rw` | Launch rejects the effectively writable mount. | +| `mps-pipe-directory` | Directory at `/tmp/nvidia-mps` mounted `nosuid`, `nodev`, `noexec` | Read and write succeed on its contents. | +| `mps-pipe-directory-without-noexec` | The same directory without `noexec` | Launch rejects the mount. | +| `device-node` | Null character device at `/dev/tmachine-cdi-null` | Non-root; write succeeds. | +| `device-node-with-gid` | Group-restricted null device at `/dev/tmachine-cdi-group-null` | Non-root with the device's group; write succeeds. | +| `unix-socket` | Echo socket at `/tmachine-cdi/socket/echo.sock` | Sibling file in the host directory is not visible. | Every scenario also supplies a spec-level environment marker and a read-only mount at `/usr/share/tmachine-cdi/common/spec-marker`, allowing checks of @@ -383,9 +385,8 @@ with positive cases. The role publishes each scenario's expectations as `scenarios.json` next to the fixtures. The `scenario_*` tests load them and assert the behavior recorded in -`vars/scenarios.yaml`, which is the sandbox's behavior today. A change that -alters a scenario's behavior, such as #4204 deriving grants from the spec, -updates those values in the same diff. To record the current behavior, run the +`vars/scenarios.yaml`. A change that alters a scenario's behavior updates those +values in the same diff. To record the sandbox's current behavior, run the tests with `OPENSHELL_CDI_CAPTURE=1` set for the test process; they then print what each scenario observed instead of asserting. diff --git a/tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml b/tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml index 1a2aac81d8..ebb238c4fa 100644 --- a/tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml +++ b/tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml @@ -79,7 +79,7 @@ cdi_scenarios: options: [bind, ro] expect: checks: - - {type: read, path: /tmachine-cdi/read-only/file, expect: denied} + - {type: contents, path: /tmachine-cdi/read-only/file, expect: tmachine-cdi-read-only-file} - {type: write, path: /tmachine-cdi/read-only/file, expect: denied} - name: read-only-directory @@ -93,7 +93,7 @@ cdi_scenarios: options: [bind, ro] expect: checks: - - {type: read, path: /tmachine-cdi/read-only/directory/marker, expect: denied} + - {type: contents, path: /tmachine-cdi/read-only/directory/marker, expect: tmachine-cdi-read-only-directory/marker} - {type: write, path: /tmachine-cdi/read-only/directory/marker, expect: denied} - {type: write, path: /tmachine-cdi/read-only/directory/new-file, expect: denied} @@ -110,7 +110,7 @@ cdi_scenarios: options: [bind, ro] expect: checks: - - {type: read, path: /tmachine-cdi/library/libfixture.so.1, expect: denied} + - {type: contents, path: /tmachine-cdi/library/libfixture.so.1, expect: tmachine-cdi-libfixture.so.1} - {type: write, path: /tmachine-cdi/library/libfixture.so.1, expect: denied} - name: broad-library-parent @@ -120,9 +120,7 @@ cdi_scenarios: containerPath: /libtmachinefixture.so.1 options: [bind, ro] expect: - checks: - - {type: read, path: /libtmachinefixture.so.1, expect: denied} - - {type: write, path: /libtmachinefixture.so.1, expect: denied} + rejected: unsafe - name: writable-file fixtures: @@ -133,9 +131,7 @@ cdi_scenarios: containerPath: /tmachine-cdi/writable/file options: [bind, rw] expect: - checks: - - {type: read, path: /tmachine-cdi/writable/file, expect: denied} - - {type: write, path: /tmachine-cdi/writable/file, expect: denied} + rejected: CDI writable mount - name: writable-directory fixtures: @@ -146,8 +142,7 @@ cdi_scenarios: containerPath: /tmachine-cdi/writable/directory options: [bind, rw] expect: - checks: - - {type: write, path: /tmachine-cdi/writable/directory/probe, expect: denied} + rejected: CDI writable mount - name: omitted-access-file edits: @@ -156,9 +151,7 @@ cdi_scenarios: containerPath: /tmachine-cdi/writable/omitted-access options: [bind] expect: - checks: - - {type: read, path: /tmachine-cdi/writable/omitted-access, expect: denied} - - {type: write, path: /tmachine-cdi/writable/omitted-access, expect: denied} + rejected: CDI writable mount - name: device-node edits: @@ -169,7 +162,32 @@ cdi_scenarios: expect: checks: - {type: non-root} - - {type: write, path: /dev/tmachine-cdi-null, expect: denied} + - {type: write, path: /dev/tmachine-cdi-null, expect: ok} + + # NVIDIA MPS clients open pipes under /tmp/nvidia-mps. The fixture uses a + # regular file in place of a pipe so the checks cannot block on open. + - name: mps-pipe-directory + fixtures: + - {path: mps-pipes, type: dir, mode: "0777"} + - {path: mps-pipes/control, type: file, content: "tmachine-cdi-mps-control\n", mode: "0666"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/mps-pipes" + containerPath: /tmp/nvidia-mps + options: [nosuid, nodev, rbind, rprivate, noexec] + expect: + checks: + - {type: contents, path: /tmp/nvidia-mps/control, expect: tmachine-cdi-mps-control} + - {type: write, path: /tmp/nvidia-mps/control, expect: ok} + + - name: mps-pipe-directory-without-noexec + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/mps-pipes" + containerPath: /tmp/nvidia-mps + options: [nosuid, nodev, rbind, rprivate] + expect: + rejected: CDI IPC directory # The device is mode 0660 with a group that only `additionalGids` grants. - name: device-node-with-gid @@ -188,7 +206,7 @@ cdi_scenarios: checks: - {type: non-root} - {type: group-of, path: /dev/tmachine-cdi-group-null} - - {type: write, path: /dev/tmachine-cdi-group-null, expect: denied} + - {type: write, path: /dev/tmachine-cdi-group-null, expect: ok} # The echo socket is served by the systemd units installed in scenarios.yaml. # Its sibling file checks that only the socket path is exposed. diff --git a/tests/suites/features/cdi/tests/injection.rs b/tests/suites/features/cdi/tests/injection.rs index 814e1920d5..c278f8603d 100644 --- a/tests/suites/features/cdi/tests/injection.rs +++ b/tests/suites/features/cdi/tests/injection.rs @@ -408,6 +408,8 @@ cdi_scenarios! { scenario_writable_file => "writable-file", scenario_writable_directory => "writable-directory", scenario_omitted_access_file => "omitted-access-file", + scenario_mps_pipe_directory => "mps-pipe-directory", + scenario_mps_pipe_directory_without_noexec => "mps-pipe-directory-without-noexec", scenario_device_node => "device-node", scenario_device_node_with_gid => "device-node-with-gid", scenario_unix_socket => "unix-socket",