diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index a7a3235dc7..4c72179f58 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -491,14 +491,14 @@ proxy UID must remain at least `1000` and must not match the workload UID because the pod egress fence exempts its traffic. The pod `fsGroup` is set to `sandbox_gid` in both modes. -If the agent command starts with a bootstrap before `openshell-sandbox`, compare -its exact image digest with `[openshell.drivers.kubernetes.process_bootstraps]` -(Helm `supervisor.processBootstraps`). -Only operator-allowlisted images receive UID 0 and CHOWN/SETUID/SETGID/SETPCAP at -startup. Verify that the bootstrap drops these before supervisor startup, while -RuntimeDefault seccomp and `allowPrivilegeEscalation: false` remain enabled. -Never add a mutable tag or caller-selected bootstrap to recover a failing sandbox. -Images outside the allowlist retain the existing non-root startup. +If the agent command starts with a bootstrap before `openshell-sandbox`, inspect +operator configuration. `process_bootstrap` (Helm `supervisor.processBootstrap`) +applies to every sidecar process container independently of its image and requires +trusted image admission. It cannot be combined with the legacy `process_bootstraps` +image map or combined topology. The disabled default retains non-root startup. +Verify that bootstrap drops UID 0 and CHOWN/SETUID/SETGID/SETPCAP before supervisor +startup, with RuntimeDefault seccomp and `allowPrivilegeEscalation: false` enabled. +Never accept a caller-selected bootstrap to recover a failing sandbox. In sidecar topology only the network sidecar should mount the gateway bootstrap credentials (`openshell-sa-token` and `openshell-client-tls`). The process diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index be9163e68a..9212eba25b 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -490,15 +490,17 @@ update the matching driver README if a maintainer-facing constraint changes. ### Operator-owned process bootstrap -The Kubernetes sidecar topology can prefix the process supervisor with a bootstrap -selected from `process_bootstraps` by the workload's exact immutable image reference. -The allowlist belongs to gateway configuration, never the sandbox request or image -environment. Empty configuration preserves non-root startup for every image. - -A matching image starts with UID 0 and only CHOWN, SETUID, SETGID and SETPCAP, with -RuntimeDefault seccomp and privilege escalation disabled. The operator must review -the image and bootstrap as trusted startup code: it must sanitize the environment, -provision identities, drop all capabilities and switch to the configured sandbox -identity before starting the process supervisor. The driver does not add a container, -change network-sidecar authority, or expose gateway credentials to the process -container. This grant must be reviewed again for each new image digest. +The Kubernetes sidecar topology can prefix the process supervisor with an +operator-owned `process_bootstrap` command. It applies to every process container +without reading the workload image. This mode requires trusted image admission, +rejects combined topology and cannot be combined with the legacy `process_bootstraps` +image map. Empty configuration preserves non-root startup. Workload requests and +image environment cannot supply or override either operator setting. + +Bootstrap starts with UID 0 and only CHOWN, SETUID, SETGID and SETPCAP, with +RuntimeDefault seccomp and privilege escalation disabled. The trusted bootstrap +must sanitize the environment, provision identities, drop all capabilities and +switch to the configured sandbox identity before starting the process supervisor. +The driver does not add a container, change network-sidecar authority or expose +gateway credentials to the process container. Review admitted images through the +owning platform's release-verification boundary. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index af7c3afce0..9b1ac8a880 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -326,6 +326,9 @@ pub struct KubernetesComputeConfig { pub topology: SupervisorTopology, /// Sidecar-only settings used when `topology = "sidecar"`. pub sidecar: KubernetesSidecarConfig, + /// Operator-owned bootstrap command for every sidecar process container. + /// Disabled by default; workload input cannot select or override it. + pub process_bootstrap: Vec, /// Operator-owned bootstrap commands keyed by exact immutable workload image. /// Only sidecar workloads matching this allowlist start with identity setup capabilities. pub process_bootstraps: std::collections::HashMap>, @@ -435,6 +438,19 @@ pub const ANNOTATION_SCC_UID_RANGE: &str = "openshift.io/sa.scc.uid-range"; /// Format: `/` (e.g. `1000000000/10000`). pub const ANNOTATION_SCC_SUPPLEMENTAL_GROUPS: &str = "openshift.io/sa.scc.supplemental-groups"; +fn validate_process_bootstrap_command(command: &[String]) -> Result<(), String> { + if command.is_empty() + || !command[0].starts_with('/') + || command.len() > 16 + || command + .iter() + .any(|arg| arg.contains('\0') || arg.len() > 4096) + { + return Err("process bootstrap requires a bounded absolute command".into()); + } + Ok(()) +} + impl Default for KubernetesComputeConfig { fn default() -> Self { Self { @@ -457,6 +473,7 @@ impl Default for KubernetesComputeConfig { supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), + process_bootstrap: Vec::new(), process_bootstraps: std::collections::HashMap::new(), https_proxy: None, no_proxy: None, @@ -511,16 +528,17 @@ impl KubernetesComputeConfig { } pub fn validate_process_bootstraps(&self) -> Result<(), String> { + if !self.process_bootstrap.is_empty() { + if self.topology != SupervisorTopology::Sidecar || !self.process_bootstraps.is_empty() { + return Err("process_bootstrap requires sidecar topology and cannot be combined with process_bootstraps".into()); + } + validate_process_bootstrap_command(&self.process_bootstrap)?; + } for (image, command) in &self.process_bootstraps { let digest = image.rsplit_once("@sha256:").map(|(_, value)| value); if !digest.is_some_and(|value| { value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit()) - }) || command.is_empty() - || !command[0].starts_with('/') - || command.len() > 16 - || command - .iter() - .any(|arg| arg.contains('\0') || arg.len() > 4096) + }) || validate_process_bootstrap_command(command).is_err() { return Err( "process_bootstraps requires exact SHA256 images and bounded absolute commands" @@ -1962,6 +1980,35 @@ mod tests { assert!(al.read().contains("ns2")); assert!(al.remove("ns1")); } + #[test] + fn operator_process_bootstrap_requires_an_unambiguous_sidecar_command() { + let command = vec!["/usr/local/bin/bootstrap".to_string()]; + let mut config = KubernetesComputeConfig { + topology: SupervisorTopology::Sidecar, + process_bootstrap: command.clone(), + ..KubernetesComputeConfig::default() + }; + assert!(config.validate_process_bootstraps().is_ok()); + config.topology = SupervisorTopology::Combined; + assert!(config.validate_process_bootstraps().is_err()); + config.topology = SupervisorTopology::Sidecar; + config + .process_bootstraps + .insert(format!("registry/agent@sha256:{}", "a".repeat(64)), command); + assert!(config.validate_process_bootstraps().is_err()); + config.process_bootstraps.clear(); + for invalid in [ + vec!["relative".to_string()], + vec!["/bin/bootstrap\0".to_string()], + vec!["/bin/bootstrap".to_string(); 17], + ] { + config.process_bootstrap = invalid; + assert!(config.validate_process_bootstraps().is_err()); + } + config.process_bootstrap.clear(); + assert!(config.validate_process_bootstraps().is_ok()); + } + #[test] fn process_bootstrap_configuration_requires_immutable_images_and_absolute_commands() { let digest = format!("registry/agent@sha256:{}", "a".repeat(64)); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index ab15bfa092..c63611aab5 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1685,6 +1685,7 @@ impl KubernetesComputeDriver { supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, supervisor_sideload_method: self.config.supervisor_sideload_method, topology: self.config.topology, + process_bootstrap: &self.config.process_bootstrap, process_bootstraps: Some(&self.config.process_bootstraps), proxy_uid: self.config.sidecar.proxy_uid, process_binary_aware_network_policy: self @@ -3302,10 +3303,15 @@ fn apply_supervisor_sidecar_topology( ]), ); - let bootstrap = container - .get("image") - .and_then(|image| image.as_str()) - .and_then(|image| params.process_bootstraps.and_then(|items| items.get(image))); + let bootstrap = if params.process_bootstrap.is_empty() { + container + .get("image") + .and_then(|image| image.as_str()) + .and_then(|image| params.process_bootstraps.and_then(|items| items.get(image))) + .map(Vec::as_slice) + } else { + Some(params.process_bootstrap) + }; if let Some(bootstrap) = bootstrap { let mut command: Vec = bootstrap.iter().map(|s| serde_json::json!(s)).collect(); @@ -3586,6 +3592,7 @@ struct SandboxPodParams<'a> { supervisor_image_pull_policy: &'a str, supervisor_sideload_method: SupervisorSideloadMethod, topology: SupervisorTopology, + process_bootstrap: &'a [String], process_bootstraps: Option<&'a std::collections::HashMap>>, proxy_uid: u32, process_binary_aware_network_policy: bool, @@ -3629,6 +3636,7 @@ impl Default for SandboxPodParams<'_> { supervisor_image_pull_policy: "", supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), + process_bootstrap: &[], process_bootstraps: None, proxy_uid: DEFAULT_PROXY_UID, process_binary_aware_network_policy: true, @@ -6358,6 +6366,55 @@ mod tests { ); } + #[test] + fn operator_process_bootstrap_is_independent_of_the_workload_image() { + let command = vec!["/usr/local/bin/bootstrap".to_string()]; + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + process_bootstrap: &command, + sandbox_uid: 1000, + sandbox_gid: 1000, + ..SandboxPodParams::default() + }; + for image in [ + "registry/assistant:release-a", + "registry/assistant:release-b", + ] { + let pod = sandbox_template_to_k8s( + &SandboxTemplate { + image: image.to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + let agent = pod["spec"]["containers"] + .as_array() + .unwrap() + .iter() + .find(|container| container["name"] == "agent") + .unwrap(); + assert_eq!(agent["command"][0], "/usr/local/bin/bootstrap"); + assert_eq!(agent["command"][1], "--"); + assert_eq!(agent["securityContext"]["runAsUser"], 0); + assert_eq!( + agent["securityContext"]["seccompProfile"]["type"], + "RuntimeDefault" + ); + assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false); + assert_eq!( + agent["securityContext"]["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); + assert_eq!( + agent["securityContext"]["capabilities"]["add"], + serde_json::json!(["CHOWN", "SETUID", "SETGID", "SETPCAP"]) + ); + } + } + #[test] fn process_bootstrap_requires_the_operator_allowlisted_image() { let image = format!("registry/agent@sha256:{}", "a".repeat(64)); diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 985a4ed474..0d1a13d8c6 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -257,6 +257,7 @@ async fn main() -> Result<()> { let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { + process_bootstrap: Vec::new(), process_bootstraps: std::collections::HashMap::new(), workspace_mode: args.workspace_mode, gateway_id: args.gateway_id, diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 9c328825a6..873bb8a8c4 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -293,6 +293,7 @@ discovery endpoint or its TLS CA. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | +| supervisor.processBootstrap | list | `[]` | Operator-owned absolute bootstrap command for every sidecar process container. Disabled by default; requires trusted workload admission and cannot be combined with processBootstraps. | | supervisor.processBootstraps | object | `{}` | Operator-owned absolute bootstrap commands keyed by exact immutable workload image. Only matching sidecar process containers receive temporary identity-setup capabilities. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | | supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index ed8d62169d..6f1bf5dd9c 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -211,6 +211,16 @@ data: supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} {{- end }} + {{- if .Values.supervisor.processBootstrap }} + {{- if ne .Values.supervisor.topology "sidecar" }} + {{- fail "supervisor.processBootstrap requires sidecar topology" }} + {{- end }} + {{- if .Values.supervisor.processBootstraps }} + {{- fail "supervisor.processBootstrap cannot be combined with processBootstraps" }} + {{- end }} + process_bootstrap = {{ .Values.supervisor.processBootstrap | toJson }} + {{- end }} + {{- if .Values.supervisor.processBootstraps }} [openshell.drivers.kubernetes.process_bootstraps] {{- range $image, $command := .Values.supervisor.processBootstraps }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 1da36bdfe7..ae96eda71f 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -677,6 +677,38 @@ tests: - it: leaves bootstrap privileges disabled by default template: templates/gateway-config.yaml asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'process_bootstrap' + + - it: renders an operator process bootstrap without an image map + template: templates/gateway-config.yaml + set: + supervisor.topology: sidecar + supervisor.processBootstrap: ["/usr/local/bin/bootstrap"] + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'process_bootstrap = \["/usr/local/bin/bootstrap"\]' - notMatchRegex: path: data["gateway.toml"] pattern: 'process_bootstraps' + + - it: rejects operator process bootstrap in combined topology + template: templates/statefulset.yaml + set: + supervisor.processBootstrap: ["/usr/local/bin/bootstrap"] + asserts: + - failedTemplate: + errorMessage: "supervisor.processBootstrap requires sidecar topology" + + - it: rejects simultaneous operator bootstrap and image map + template: templates/statefulset.yaml + set: + supervisor.topology: sidecar + supervisor.processBootstrap: ["/usr/local/bin/bootstrap"] + supervisor.processBootstraps: + "registry.example/agent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": ["/usr/local/bin/bootstrap"] + asserts: + - failedTemplate: + errorMessage: "supervisor.processBootstrap cannot be combined with processBootstraps" diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 835247aa2f..600b3a4a80 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -49,6 +49,8 @@ supervisor: # "sidecar" runs network enforcement in a dedicated sidecar and the process # supervisor as a low-capability wrapper in the agent container. topology: "combined" + # -- Operator-owned absolute bootstrap command for every sidecar process container. Disabled by default; requires trusted workload admission and cannot be combined with processBootstraps. + processBootstrap: [] # -- Operator-owned absolute bootstrap commands keyed by exact immutable workload image. # Only matching sidecar process containers receive temporary identity-setup capabilities. processBootstraps: {} diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index f77a64f9c5..f26e975623 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -556,6 +556,15 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # (hot-reloaded on change, e.g. via ConfigMap volume mount). # operator_namespace_file = "/etc/openshell/workspace-namespaces.json" +# Optional operator-owned bootstrap for every sidecar process container. +# Helm value: supervisor.processBootstrap. Default: disabled. +# Configure only where admission accepts trusted, reviewed workload images. +# Requires topology="sidecar" and cannot be combined with process_bootstraps. +# The absolute command has at most 15 arguments. The driver appends "--" and +# the process supervisor command. It starts with UID 0 and the identity-setup +# capabilities below, which MUST be discarded before workload execution. +# process_bootstrap = ["/usr/local/bin/bootstrap"] + # Optional operator-owned bootstrap allowlist for sidecar process containers. # Helm value: supervisor.processBootstraps. # Keys must be exact immutable image references. Mutable tags never match.