feat: allow operator-pinned process bootstraps (task 33.03.24)

.agents/skills/debug-openshell-cluster/SKILL.md

@@ -491,6 +491,15 @@ proxy UID must remain at least `1000` and must not match the workload UID
 because the pod egress fence exempts its traffic. The pod `fsGroup` is set to
 `sandbox_gid` in both modes.
 
+If the agent command starts with a bootstrap before `openshell-sandbox`, compare
+its exact image digest with `[openshell.drivers.kubernetes.process_bootstraps]`
+(Helm `supervisor.processBootstraps`).
+Only operator-allowlisted images receive UID 0 and CHOWN/SETUID/SETGID/SETPCAP at
+startup. Verify that the bootstrap drops these before supervisor startup, while
+RuntimeDefault seccomp and `allowPrivilegeEscalation: false` remain enabled.
+Never add a mutable tag or caller-selected bootstrap to recover a failing sandbox.
+Images outside the allowlist retain the existing non-root startup.
+
 In sidecar topology only the network sidecar should mount the gateway bootstrap
 credentials (`openshell-sa-token` and `openshell-client-tls`). The process
 container should not receive `OPENSHELL_ENDPOINT`, gateway TLS env vars, the

architecture/compute-runtimes.md

@@ -487,3 +487,18 @@ namespace.
 
 When runtime infrastructure changes, validate the relevant sandbox e2e path and
 update the matching driver README if a maintainer-facing constraint changes.
+
+### Operator-owned process bootstrap
+
+The Kubernetes sidecar topology can prefix the process supervisor with a bootstrap
+selected from `process_bootstraps` by the workload's exact immutable image reference.
+The allowlist belongs to gateway configuration, never the sandbox request or image
+environment. Empty configuration preserves non-root startup for every image.
+
+A matching image starts with UID 0 and only CHOWN, SETUID, SETGID and SETPCAP, with
+RuntimeDefault seccomp and privilege escalation disabled. The operator must review
+the image and bootstrap as trusted startup code: it must sanitize the environment,
+provision identities, drop all capabilities and switch to the configured sandbox
+identity before starting the process supervisor. The driver does not add a container,
+change network-sidecar authority, or expose gateway credentials to the process
+container. This grant must be reviewed again for each new image digest.

crates/openshell-driver-kubernetes/src/config.rs

@@ -326,6 +326,9 @@ pub struct KubernetesComputeConfig {
     pub topology: SupervisorTopology,
     /// Sidecar-only settings used when `topology = "sidecar"`.
     pub sidecar: KubernetesSidecarConfig,
+    /// Operator-owned bootstrap commands keyed by exact immutable workload image.
+    /// Only sidecar workloads matching this allowlist start with identity setup capabilities.
+    pub process_bootstraps: std::collections::HashMap<String, Vec<String>>,
     /// Corporate HTTP forward proxy used by the network supervisor for
     /// policy-approved TLS CONNECT egress.
     pub https_proxy: Option<String>,
@@ -454,6 +457,7 @@ impl Default for KubernetesComputeConfig {
             supervisor_sideload_method: SupervisorSideloadMethod::default(),
             topology: SupervisorTopology::default(),
             sidecar: KubernetesSidecarConfig::default(),
+            process_bootstraps: std::collections::HashMap::new(),
             https_proxy: None,
             no_proxy: None,
             proxy_auth_secret_name: None,
@@ -506,6 +510,27 @@ impl KubernetesComputeConfig {
         )
     }
 
+    pub fn validate_process_bootstraps(&self) -> Result<(), String> {
+        for (image, command) in &self.process_bootstraps {
+            let digest = image.rsplit_once("@sha256:").map(|(_, value)| value);
+            if !digest.is_some_and(|value| {
+                value.len() == 64 && value.bytes().all(|b| b.is_ascii_hexdigit())
+            }) || command.is_empty()
+                || !command[0].starts_with('/')
+                || command.len() > 16
+                || command
+                    .iter()
+                    .any(|arg| arg.contains('\0') || arg.len() > 4096)
+            {
+                return Err(
+                    "process_bootstraps requires exact SHA256 images and bounded absolute commands"
+                        .into(),
+                );
+            }
+        }
+        Ok(())
+    }
+
     pub fn validate_proxy_uid(&self) -> Result<(), String> {
         self.sidecar.validate_proxy_uid()
     }
@@ -1937,4 +1962,28 @@ mod tests {
         assert!(al.read().contains("ns2"));
         assert!(al.remove("ns1"));
     }
+    #[test]
+    fn process_bootstrap_configuration_requires_immutable_images_and_absolute_commands() {
+        let digest = format!("registry/agent@sha256:{}", "a".repeat(64));
+        for (image, command, valid) in [
+            (
+                digest.as_str(),
+                vec!["/usr/local/bin/bootstrap".to_string()],
+                true,
+            ),
+            (
+                "registry/agent:latest",
+                vec!["/usr/local/bin/bootstrap".to_string()],
+                false,
+            ),
+            (digest.as_str(), vec!["bootstrap".to_string()], false),
+            (digest.as_str(), vec![], false),
+        ] {
+            let config = KubernetesComputeConfig {
+                process_bootstraps: std::collections::HashMap::from([(image.to_string(), command)]),
+                ..KubernetesComputeConfig::default()
+            };
+            assert_eq!(config.validate_process_bootstraps().is_ok(), valid);
+        }
+    }
 }

crates/openshell-driver-kubernetes/src/driver.rs

@@ -739,6 +739,9 @@ impl KubernetesComputeDriver {
         config
             .validate_sandbox_identity_config()
             .map_err(KubernetesDriverError::Precondition)?;
+        config
+            .validate_process_bootstraps()
+            .map_err(KubernetesDriverError::InvalidArgument)?;
         config
             .validate_proxy_uid()
             .map_err(KubernetesDriverError::Precondition)?;
@@ -1682,6 +1685,7 @@ impl KubernetesComputeDriver {
             supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy,
             supervisor_sideload_method: self.config.supervisor_sideload_method,
             topology: self.config.topology,
+            process_bootstraps: Some(&self.config.process_bootstraps),
             proxy_uid: self.config.sidecar.proxy_uid,
             process_binary_aware_network_policy: self
                 .config
@@ -3298,28 +3302,58 @@ fn apply_supervisor_sidecar_topology(
             ]),
         );
 
+        let bootstrap = container
+            .get("image")
+            .and_then(|image| image.as_str())
+            .and_then(|image| params.process_bootstraps.and_then(|items| items.get(image)));
+        if let Some(bootstrap) = bootstrap {
+            let mut command: Vec<serde_json::Value> =
+                bootstrap.iter().map(|s| serde_json::json!(s)).collect();
+            command.push(serde_json::json!("--"));
+            command.extend(
+                container["command"]
+                    .as_array()
+                    .expect("supervisor command")
+                    .iter()
+                    .cloned(),
+            );
+            container.insert("command".to_string(), serde_json::json!(command));
+        }
+        let has_bootstrap = bootstrap.is_some();
+
         let security_context = container
             .entry("securityContext")
             .or_insert_with(|| serde_json::json!({}));
         if let Some(sc) = security_context.as_object_mut() {
+            if has_bootstrap {
+                sc.insert(
+                    "seccompProfile".to_string(),
+                    serde_json::json!({"type": "RuntimeDefault"}),
+                );
+            }
             sc.insert(
                 "runAsUser".to_string(),
-                serde_json::json!(params.sandbox_uid),
+                serde_json::json!(if has_bootstrap { 0 } else { params.sandbox_uid }),
             );
             sc.insert(
                 "runAsGroup".to_string(),
                 serde_json::json!(params.sandbox_gid),
             );
-            sc.insert("runAsNonRoot".to_string(), serde_json::json!(true));
+            sc.insert(
+                "runAsNonRoot".to_string(),
+                serde_json::json!(!has_bootstrap),
+            );
             sc.insert(
                 "allowPrivilegeEscalation".to_string(),
                 serde_json::json!(false),
             );
             sc.insert(
                 "capabilities".to_string(),
-                serde_json::json!({
-                    "drop": ["ALL"]
-                }),
+                if has_bootstrap {
+                    serde_json::json!({"drop": ["ALL"], "add": ["CHOWN", "SETUID", "SETGID", "SETPCAP"]})
+                } else {
+                    serde_json::json!({"drop": ["ALL"]})
+                },
             );
         }
 
@@ -3552,6 +3586,7 @@ struct SandboxPodParams<'a> {
     supervisor_image_pull_policy: &'a str,
     supervisor_sideload_method: SupervisorSideloadMethod,
     topology: SupervisorTopology,
+    process_bootstraps: Option<&'a std::collections::HashMap<String, Vec<String>>>,
     proxy_uid: u32,
     process_binary_aware_network_policy: bool,
     https_proxy: Option<&'a str>,
@@ -3594,6 +3629,7 @@ impl Default for SandboxPodParams<'_> {
             supervisor_image_pull_policy: "",
             supervisor_sideload_method: SupervisorSideloadMethod::default(),
             topology: SupervisorTopology::default(),
+            process_bootstraps: None,
             proxy_uid: DEFAULT_PROXY_UID,
             process_binary_aware_network_policy: true,
             https_proxy: None,
@@ -6322,6 +6358,70 @@ mod tests {
         );
     }
 
+    #[test]
+    fn process_bootstrap_requires_the_operator_allowlisted_image() {
+        let image = format!("registry/agent@sha256:{}", "a".repeat(64));
+        let bootstraps = std::collections::HashMap::from([(
+            image.clone(),
+            vec!["/usr/local/bin/bootstrap".to_string()],
+        )]);
+        for selected in [image.as_str(), "registry/agent:latest"] {
+            let params = SandboxPodParams {
+                topology: SupervisorTopology::Sidecar,
+                sandbox_uid: 1000,
+                sandbox_gid: 1000,
+                process_bootstraps: Some(&bootstraps),
+                ..SandboxPodParams::default()
+            };
+            let pod = sandbox_template_to_k8s(
+                &SandboxTemplate {
+                    image: selected.to_string(),
+                    ..SandboxTemplate::default()
+                },
+                false,
+                &std::collections::HashMap::new(),
+                false,
+                &params,
+            );
+            let agent = pod["spec"]["containers"]
+                .as_array()
+                .unwrap()
+                .iter()
+                .find(|container| container["name"] == "agent")
+                .unwrap();
+            assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false);
+            assert_eq!(
+                agent["securityContext"]["capabilities"]["drop"],
+                serde_json::json!(["ALL"])
+            );
+            if selected == image {
+                assert_eq!(agent["command"][0], "/usr/local/bin/bootstrap");
+                assert_eq!(
+                    agent["securityContext"]["seccompProfile"]["type"],
+                    "RuntimeDefault"
+                );
+                assert_eq!(agent["command"][1], "--");
+                assert_eq!(agent["securityContext"]["runAsUser"], 0);
+                assert_eq!(
+                    agent["securityContext"]["capabilities"]["add"],
+                    serde_json::json!(["CHOWN", "SETUID", "SETGID", "SETPCAP"])
+                );
+            } else {
+                assert_eq!(agent["securityContext"]["runAsUser"], 1000);
+                assert_eq!(agent["securityContext"]["runAsNonRoot"], true);
+                assert!(
+                    agent["securityContext"]["capabilities"]
+                        .get("add")
+                        .is_none()
+                );
+                assert_eq!(
+                    agent["command"][0],
+                    format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox")
+                );
+            }
+        }
+    }
+
     #[test]
     fn sidecar_topology_renders_process_agent_and_network_sidecar() {
         let params = SandboxPodParams {

crates/openshell-driver-kubernetes/src/main.rs

@@ -257,6 +257,7 @@ async fn main() -> Result<()> {
     let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
     let driver = KubernetesComputeDriver::new(
         KubernetesComputeConfig {
+            process_bootstraps: std::collections::HashMap::new(),
             workspace_mode: args.workspace_mode,
             gateway_id: args.gateway_id,
             namespace: args.sandbox_namespace,

deploy/helm/openshell/README.md

@@ -293,6 +293,7 @@ discovery endpoint or its TLS CA.
 | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. |
 | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. |
 | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. |
+| supervisor.processBootstraps | object | `{}` | Operator-owned absolute bootstrap commands keyed by exact immutable workload image. Only matching sidecar process containers receive temporary identity-setup capabilities. |
 | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. |
 | supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. |
 | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version:   K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36)   K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. |

deploy/helm/openshell/templates/gateway-config.yaml

@@ -211,6 +211,13 @@ data:
     supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }}
     {{- end }}
 
+    {{- if .Values.supervisor.processBootstraps }}
+    [openshell.drivers.kubernetes.process_bootstraps]
+    {{- range $image, $command := .Values.supervisor.processBootstraps }}
+    {{ $image | quote }} = {{ $command | toJson }}
+    {{- end }}
+    {{- end }}
+
     [openshell.drivers.kubernetes.managed_ssh_ingress]
     enabled                       = {{ .Values.networkPolicy.enabled }}
     gateway_namespace             = {{ .Release.Namespace | quote }}

deploy/helm/openshell/tests/gateway_config_test.yaml

@@ -660,3 +660,23 @@ tests:
     asserts:
       - failedTemplate:
           errorMessage: "certManager.serverIssuerRef.name is set but certManager.enabled is false \u2014 the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name."
+
+  - it: renders only operator configured bootstrap commands with quoted digest keys
+    template: templates/gateway-config.yaml
+    set:
+      supervisor.processBootstraps:
+        "registry.example/agent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": ["/usr/local/bin/bootstrap"]
+    asserts:
+      - matchRegex:
+          path: data["gateway.toml"]
+          pattern: '\[openshell\.drivers\.kubernetes\.process_bootstraps\]'
+      - matchRegex:
+          path: data["gateway.toml"]
+          pattern: '"registry.example/agent@sha256:a{64}" = \["/usr/local/bin/bootstrap"\]'
+
+  - it: leaves bootstrap privileges disabled by default
+    template: templates/gateway-config.yaml
+    asserts:
+      - notMatchRegex:
+          path: data["gateway.toml"]
+          pattern: 'process_bootstraps'

deploy/helm/openshell/values.yaml

@@ -49,6 +49,9 @@ supervisor:
   # "sidecar" runs network enforcement in a dedicated sidecar and the process
   # supervisor as a low-capability wrapper in the agent container.
   topology: "combined"
+  # -- Operator-owned absolute bootstrap commands keyed by exact immutable workload image.
+  # Only matching sidecar process containers receive temporary identity-setup capabilities.
+  processBootstraps: {}
   sidecar:
     # -- UID for relaxed long-running network sidecars in sidecar topology.
     # Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants

docs/reference/gateway-config.mdx

@@ -556,6 +556,18 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc
 # (hot-reloaded on change, e.g. via ConfigMap volume mount).
 # operator_namespace_file      = "/etc/openshell/workspace-namespaces.json"
 
+# Optional operator-owned bootstrap allowlist for sidecar process containers.
+# Helm value: supervisor.processBootstraps.
+# Keys must be exact immutable image references. Mutable tags never match.
+# Each command is an absolute executable plus at most 15 arguments; the driver
+# appends "--" and the normal process-supervisor command. Default: empty.
+# Only matching images start as UID 0 with CHOWN, SETUID, SETGID, SETPCAP.
+# The reviewed bootstrap MUST discard those capabilities and switch to the
+# configured sandbox UID/GID before launching the supervisor or any workload code.
+# RuntimeDefault seccomp and allowPrivilegeEscalation=false remain mandatory.
+# [openshell.drivers.kubernetes.process_bootstraps]
+# "registry.example/agent@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" = ["/usr/local/bin/bootstrap"]
+
 [openshell.drivers.kubernetes.managed_ssh_ingress]
 enabled = true
 gateway_namespace = "openshell"

tasks/scripts/stage-prebuilt-binaries.sh

@@ -254,7 +254,7 @@ build_component_for_arch() {
     if [[ -n "$build_rustflags" ]]; then
       export RUSTFLAGS="$build_rustflags"
     fi
-    CARGO_INCREMENTAL=0 mise x -- "${cargo_env[@]}" "${cargo_subcommand[@]}" "${args[@]}"
+    CARGO_INCREMENTAL=0 mise x -- ${cargo_env[@]+"${cargo_env[@]}"} "${cargo_subcommand[@]}" "${args[@]}"
   )
 
   binary_path="${ROOT}/target/${target}/release/${binary}"