diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 90b9a38409..a7a3235dc7 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -491,6 +491,15 @@ proxy UID must remain at least `1000` and must not match the workload UID because the pod egress fence exempts its traffic. The pod `fsGroup` is set to `sandbox_gid` in both modes. +If the agent command starts with a bootstrap before `openshell-sandbox`, compare +its exact image digest with `[openshell.drivers.kubernetes.process_bootstraps]` +(Helm `supervisor.processBootstraps`). +Only operator-allowlisted images receive UID 0 and CHOWN/SETUID/SETGID/SETPCAP at +startup. Verify that the bootstrap drops these before supervisor startup, while +RuntimeDefault seccomp and `allowPrivilegeEscalation: false` remain enabled. +Never add a mutable tag or caller-selected bootstrap to recover a failing sandbox. +Images outside the allowlist retain the existing non-root startup. + In sidecar topology only the network sidecar should mount the gateway bootstrap credentials (`openshell-sa-token` and `openshell-client-tls`). The process container should not receive `OPENSHELL_ENDPOINT`, gateway TLS env vars, the diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 2a36073486..be9163e68a 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -487,3 +487,18 @@ namespace. When runtime infrastructure changes, validate the relevant sandbox e2e path and update the matching driver README if a maintainer-facing constraint changes. + +### Operator-owned process bootstrap + +The Kubernetes sidecar topology can prefix the process supervisor with a bootstrap +selected from `process_bootstraps` by the workload's exact immutable image reference. +The allowlist belongs to gateway configuration, never the sandbox request or image +environment. Empty configuration preserves non-root startup for every image. + +A matching image starts with UID 0 and only CHOWN, SETUID, SETGID and SETPCAP, with +RuntimeDefault seccomp and privilege escalation disabled. The operator must review +the image and bootstrap as trusted startup code: it must sanitize the environment, +provision identities, drop all capabilities and switch to the configured sandbox +identity before starting the process supervisor. The driver does not add a container, +change network-sidecar authority, or expose gateway credentials to the process +container. This grant must be reviewed again for each new image digest. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 7c9d339403..af7c3afce0 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -326,6 +326,9 @@ pub struct KubernetesComputeConfig { pub topology: SupervisorTopology, /// Sidecar-only settings used when `topology = "sidecar"`. pub sidecar: KubernetesSidecarConfig, + /// Operator-owned bootstrap commands keyed by exact immutable workload image. + /// Only sidecar workloads matching this allowlist start with identity setup capabilities. + pub process_bootstraps: std::collections::HashMap>, /// Corporate HTTP forward proxy used by the network supervisor for /// policy-approved TLS CONNECT egress. pub https_proxy: Option, @@ -454,6 +457,7 @@ impl Default for KubernetesComputeConfig { supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), + process_bootstraps: std::collections::HashMap::new(), https_proxy: None, no_proxy: None, proxy_auth_secret_name: None, @@ -506,6 +510,27 @@ impl KubernetesComputeConfig { ) } + pub fn validate_process_bootstraps(&self) -> Result<(), String> { + for (image, command) in &self.process_bootstraps { + let digest = image.rsplit_once("@sha256:").map(|(_, value)| value); + if !digest.is_some_and(|value| { + value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) + }) || command.is_empty() + || !command[0].starts_with('/') + || command.len() > 16 + || command + .iter() + .any(|arg| arg.contains('\0') || arg.len() > 4096) + { + return Err( + "process_bootstraps requires exact SHA256 images and bounded absolute commands" + .into(), + ); + } + } + Ok(()) + } + pub fn validate_proxy_uid(&self) -> Result<(), String> { self.sidecar.validate_proxy_uid() } @@ -1937,4 +1962,28 @@ mod tests { assert!(al.read().contains("ns2")); assert!(al.remove("ns1")); } + #[test] + fn process_bootstrap_configuration_requires_immutable_images_and_absolute_commands() { + let digest = format!("registry/agent@sha256:{}", "a".repeat(64)); + for (image, command, valid) in [ + ( + digest.as_str(), + vec!["/usr/local/bin/bootstrap".to_string()], + true, + ), + ( + "registry/agent:latest", + vec!["/usr/local/bin/bootstrap".to_string()], + false, + ), + (digest.as_str(), vec!["bootstrap".to_string()], false), + (digest.as_str(), vec![], false), + ] { + let config = KubernetesComputeConfig { + process_bootstraps: std::collections::HashMap::from([(image.to_string(), command)]), + ..KubernetesComputeConfig::default() + }; + assert_eq!(config.validate_process_bootstraps().is_ok(), valid); + } + } } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index c19ace4df4..ab15bfa092 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -739,6 +739,9 @@ impl KubernetesComputeDriver { config .validate_sandbox_identity_config() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_process_bootstraps() + .map_err(KubernetesDriverError::InvalidArgument)?; config .validate_proxy_uid() .map_err(KubernetesDriverError::Precondition)?; @@ -1682,6 +1685,7 @@ impl KubernetesComputeDriver { supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, supervisor_sideload_method: self.config.supervisor_sideload_method, topology: self.config.topology, + process_bootstraps: Some(&self.config.process_bootstraps), proxy_uid: self.config.sidecar.proxy_uid, process_binary_aware_network_policy: self .config @@ -3298,28 +3302,58 @@ fn apply_supervisor_sidecar_topology( ]), ); + let bootstrap = container + .get("image") + .and_then(|image| image.as_str()) + .and_then(|image| params.process_bootstraps.and_then(|items| items.get(image))); + if let Some(bootstrap) = bootstrap { + let mut command: Vec = + bootstrap.iter().map(|s| serde_json::json!(s)).collect(); + command.push(serde_json::json!("--")); + command.extend( + container["command"] + .as_array() + .expect("supervisor command") + .iter() + .cloned(), + ); + container.insert("command".to_string(), serde_json::json!(command)); + } + let has_bootstrap = bootstrap.is_some(); + let security_context = container .entry("securityContext") .or_insert_with(|| serde_json::json!({})); if let Some(sc) = security_context.as_object_mut() { + if has_bootstrap { + sc.insert( + "seccompProfile".to_string(), + serde_json::json!({"type": "RuntimeDefault"}), + ); + } sc.insert( "runAsUser".to_string(), - serde_json::json!(params.sandbox_uid), + serde_json::json!(if has_bootstrap { 0 } else { params.sandbox_uid }), ); sc.insert( "runAsGroup".to_string(), serde_json::json!(params.sandbox_gid), ); - sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); + sc.insert( + "runAsNonRoot".to_string(), + serde_json::json!(!has_bootstrap), + ); sc.insert( "allowPrivilegeEscalation".to_string(), serde_json::json!(false), ); sc.insert( "capabilities".to_string(), - serde_json::json!({ - "drop": ["ALL"] - }), + if has_bootstrap { + serde_json::json!({"drop": ["ALL"], "add": ["CHOWN", "SETUID", "SETGID", "SETPCAP"]}) + } else { + serde_json::json!({"drop": ["ALL"]}) + }, ); } @@ -3552,6 +3586,7 @@ struct SandboxPodParams<'a> { supervisor_image_pull_policy: &'a str, supervisor_sideload_method: SupervisorSideloadMethod, topology: SupervisorTopology, + process_bootstraps: Option<&'a std::collections::HashMap>>, proxy_uid: u32, process_binary_aware_network_policy: bool, https_proxy: Option<&'a str>, @@ -3594,6 +3629,7 @@ impl Default for SandboxPodParams<'_> { supervisor_image_pull_policy: "", supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), + process_bootstraps: None, proxy_uid: DEFAULT_PROXY_UID, process_binary_aware_network_policy: true, https_proxy: None, @@ -6322,6 +6358,70 @@ mod tests { ); } + #[test] + fn process_bootstrap_requires_the_operator_allowlisted_image() { + let image = format!("registry/agent@sha256:{}", "a".repeat(64)); + let bootstraps = std::collections::HashMap::from([( + image.clone(), + vec!["/usr/local/bin/bootstrap".to_string()], + )]); + for selected in [image.as_str(), "registry/agent:latest"] { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + sandbox_uid: 1000, + sandbox_gid: 1000, + process_bootstraps: Some(&bootstraps), + ..SandboxPodParams::default() + }; + let pod = sandbox_template_to_k8s( + &SandboxTemplate { + image: selected.to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + let agent = pod["spec"]["containers"] + .as_array() + .unwrap() + .iter() + .find(|container| container["name"] == "agent") + .unwrap(); + assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false); + assert_eq!( + agent["securityContext"]["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); + if selected == image { + assert_eq!(agent["command"][0], "/usr/local/bin/bootstrap"); + assert_eq!( + agent["securityContext"]["seccompProfile"]["type"], + "RuntimeDefault" + ); + assert_eq!(agent["command"][1], "--"); + assert_eq!(agent["securityContext"]["runAsUser"], 0); + assert_eq!( + agent["securityContext"]["capabilities"]["add"], + serde_json::json!(["CHOWN", "SETUID", "SETGID", "SETPCAP"]) + ); + } else { + assert_eq!(agent["securityContext"]["runAsUser"], 1000); + assert_eq!(agent["securityContext"]["runAsNonRoot"], true); + assert!( + agent["securityContext"]["capabilities"] + .get("add") + .is_none() + ); + assert_eq!( + agent["command"][0], + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + ); + } + } + } + #[test] fn sidecar_topology_renders_process_agent_and_network_sidecar() { let params = SandboxPodParams { diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 0577f93d27..985a4ed474 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -257,6 +257,7 @@ async fn main() -> Result<()> { let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { + process_bootstraps: std::collections::HashMap::new(), workspace_mode: args.workspace_mode, gateway_id: args.gateway_id, namespace: args.sandbox_namespace, diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 3fe11b6a2c..9c328825a6 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -293,6 +293,7 @@ discovery endpoint or its TLS CA. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | +| supervisor.processBootstraps | object | `{}` | Operator-owned absolute bootstrap commands keyed by exact immutable workload image. Only matching sidecar process containers receive temporary identity-setup capabilities. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | | supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index eaa0b02496..ed8d62169d 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -211,6 +211,13 @@ data: supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} {{- end }} + {{- if .Values.supervisor.processBootstraps }} + [openshell.drivers.kubernetes.process_bootstraps] + {{- range $image, $command := .Values.supervisor.processBootstraps }} + {{ $image | quote }} = {{ $command | toJson }} + {{- end }} + {{- end }} + [openshell.drivers.kubernetes.managed_ssh_ingress] enabled = {{ .Values.networkPolicy.enabled }} gateway_namespace = {{ .Release.Namespace | quote }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 5d213669ad..1da36bdfe7 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -660,3 +660,23 @@ tests: asserts: - failedTemplate: errorMessage: "certManager.serverIssuerRef.name is set but certManager.enabled is false \u2014 the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." + + - it: renders only operator configured bootstrap commands with quoted digest keys + template: templates/gateway-config.yaml + set: + supervisor.processBootstraps: + "registry.example/agent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": ["/usr/local/bin/bootstrap"] + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '\[openshell\.drivers\.kubernetes\.process_bootstraps\]' + - matchRegex: + path: data["gateway.toml"] + pattern: '"registry.example/agent@sha256:a{64}" = \["/usr/local/bin/bootstrap"\]' + + - it: leaves bootstrap privileges disabled by default + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'process_bootstraps' diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 9f2e6d2be6..835247aa2f 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -49,6 +49,9 @@ supervisor: # "sidecar" runs network enforcement in a dedicated sidecar and the process # supervisor as a low-capability wrapper in the agent container. topology: "combined" + # -- Operator-owned absolute bootstrap commands keyed by exact immutable workload image. + # Only matching sidecar process containers receive temporary identity-setup capabilities. + processBootstraps: {} sidecar: # -- UID for relaxed long-running network sidecars in sidecar topology. # Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index a4b85f5844..f77a64f9c5 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -556,6 +556,18 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # (hot-reloaded on change, e.g. via ConfigMap volume mount). # operator_namespace_file = "/etc/openshell/workspace-namespaces.json" +# Optional operator-owned bootstrap allowlist for sidecar process containers. +# Helm value: supervisor.processBootstraps. +# Keys must be exact immutable image references. Mutable tags never match. +# Each command is an absolute executable plus at most 15 arguments; the driver +# appends "--" and the normal process-supervisor command. Default: empty. +# Only matching images start as UID 0 with CHOWN, SETUID, SETGID, SETPCAP. +# The reviewed bootstrap MUST discard those capabilities and switch to the +# configured sandbox UID/GID before launching the supervisor or any workload code. +# RuntimeDefault seccomp and allowPrivilegeEscalation=false remain mandatory. +# [openshell.drivers.kubernetes.process_bootstraps] +# "registry.example/agent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" = ["/usr/local/bin/bootstrap"] + [openshell.drivers.kubernetes.managed_ssh_ingress] enabled = true gateway_namespace = "openshell" diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index 757a21298b..529a76e15b 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -254,7 +254,7 @@ build_component_for_arch() { if [[ -n "$build_rustflags" ]]; then export RUSTFLAGS="$build_rustflags" fi - CARGO_INCREMENTAL=0 mise x -- "${cargo_env[@]}" "${cargo_subcommand[@]}" "${args[@]}" + CARGO_INCREMENTAL=0 mise x -- ${cargo_env[@]+"${cargo_env[@]}"} "${cargo_subcommand[@]}" "${args[@]}" ) binary_path="${ROOT}/target/${target}/release/${binary}"