fix(podman): provision runtime state and rotate restart channel identity

Signed-off-by: Drew Newberry <anewberry@nvidia.com>

Author: drew

crates/openshell-driver-podman/README.md

@@ -36,12 +36,18 @@ separate. The channel volume uses shared SELinux relabeling (`:z`).
 
 Before starting either container, the driver uploads volume-relative archives
 directly to the channel and workspace volume destinations. A rootfs upload on a
-stopped Podman container does not populate nested named volumes. Restart restores
-only the channel bootstrap into the existing channel volume, preserving the
-workspace. The workload starts before the supervisor so its user namespace exists
+stopped Podman container does not populate nested named volumes. Each restart
+rotates the generation, session, bootstrap token, and mutual-TLS credentials,
+then stages the matching channel bootstrap and supervisor topology while both
+containers are stopped. Failed staging leaves both stopped; retry rotates again.
+Restart preserves the workspace. The workload starts before the supervisor so its user namespace exists
 when the supervisor joins it; a stopped supervisor resolves that namespace again
 on its next start.
 
+Both containers have an identity-owned, 64 MiB `/run` tmpfs. This lets the
+non-root sandbox install `/run/openshell-proxy-ca` without capabilities or an
+image-provided writable runtime directory.
+
 The runtime must pass the sandbox's unprivileged enforcement probe, including
 nested seccomp notification and Landlock. Unsupported runtime defaults fail
 closed; do not switch to an unconfined profile or add capabilities.

crates/openshell-driver-podman/src/container.rs

@@ -1448,6 +1448,11 @@ pub fn build_isolation_specs(
     workload
         .mounts
         .retain(|mount| !trusted_mount(&mount.destination));
+    // The capability-free sandbox installs interception CA material below
+    // /run before launching the agent. Image-owned /run is commonly root-only.
+    workload
+        .mounts
+        .push(identity_owned_tmpfs("/run", input.identity));
     workload.volumes.push(NamedVolume {
         name: channel.clone(),
         dest: crate::isolation::CHANNEL_ROOT.into(),
@@ -1494,20 +1499,9 @@ pub fn build_isolation_specs(
             && mount.destination != openshell_core::container_paths::NETNS_MOUNT_ROOT
     });
     for destination in ["/run", "/var/log", "/tmp"] {
-        supervisor.mounts.push(Mount {
-            kind: "tmpfs".into(),
-            source: "tmpfs".into(),
-            destination: destination.into(),
-            options: vec![
-                "rw".into(),
-                "nosuid".into(),
-                "nodev".into(),
-                format!("uid={}", input.identity.uid),
-                format!("gid={}", input.identity.gid),
-                "mode=0700".into(),
-                "size=64m".into(),
-            ],
-        });
+        supervisor
+            .mounts
+            .push(identity_owned_tmpfs(destination, input.identity));
     }
     for secret in &mut supervisor.secrets {
         secret.uid = input.identity.uid;
@@ -1528,6 +1522,26 @@ pub fn build_isolation_specs(
     })
 }
 
+fn identity_owned_tmpfs(
+    destination: &str,
+    identity: &openshell_isolation_interface::contract::ResolvedWorkloadIdentity,
+) -> Mount {
+    Mount {
+        kind: "tmpfs".into(),
+        source: "tmpfs".into(),
+        destination: destination.into(),
+        options: vec![
+            "rw".into(),
+            "nosuid".into(),
+            "nodev".into(),
+            format!("uid={}", identity.uid),
+            format!("gid={}", identity.gid),
+            "mode=0700".into(),
+            "size=64m".into(),
+        ],
+    }
+}
+
 fn trusted_mount(destination: &str) -> bool {
     matches!(
         destination,
@@ -1684,6 +1698,27 @@ mod tests {
             assert!(spec.cap_add.is_empty());
             assert!(spec.seccomp_profile_path.is_empty());
             assert!(spec.no_new_privileges);
+            let runtime_mounts: Vec<_> = spec
+                .mounts
+                .iter()
+                .filter(|mount| mount.destination == "/run")
+                .collect();
+            assert_eq!(runtime_mounts.len(), 1);
+            let runtime = runtime_mounts[0];
+            assert_eq!(runtime.kind, "tmpfs");
+            assert_eq!(runtime.source, "tmpfs");
+            assert_eq!(
+                runtime.options,
+                vec![
+                    "rw",
+                    "nosuid",
+                    "nodev",
+                    "uid=1000",
+                    "gid=1001",
+                    "mode=0700",
+                    "size=64m"
+                ]
+            );
         }
         assert_eq!(specs.workload.netns.nsmode, "none");
         assert!(specs.workload.networks.is_empty());

crates/openshell-driver-podman/src/driver.rs

@@ -1318,8 +1318,19 @@ impl PodmanComputeDriver {
                 .await?;
             let bundle =
                 extract_first_tar_entry(&archive).map_err(ComputeDriverError::Precondition)?;
+            let archives =
+                crate::isolation::rotate_bootstrap_archives(sandbox_id, &container_id, &bundle)?;
+            // Update both endpoints while stopped. A failed upload leaves no
+            // running peer with a mismatched generation; retry rotates again.
             self.client
-                .copy_to_container(&container_id, crate::isolation::CHANNEL_ROOT, bundle)
+                .copy_to_container(
+                    &container_id,
+                    crate::isolation::CHANNEL_ROOT,
+                    archives.channel,
+                )
+                .await?;
+            self.client
+                .copy_to_container(&supervisor, "/", archives.supervisor)
                 .await?;
             self.client.verify_isolation_fence(&container_id).await?;
             self.client.start_container(&container_id).await?;
@@ -1901,10 +1912,16 @@ mod tests {
                 .filter(|request| request.starts_with("PUT "))
                 .cloned()
                 .collect::<Vec<_>>(),
-            vec![format!(
-                "PUT {}",
-                api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel")
-            )]
+            vec![
+                format!(
+                    "PUT {}",
+                    api_path("/libpod/containers/ctr-1/archive?path=%2F.openshell%2Fchannel")
+                ),
+                format!(
+                    "PUT {}",
+                    api_path("/libpod/containers/openshell-supervisor-sandbox-1/archive?path=%2F")
+                ),
+            ]
         );
         assert!(
             !restart_requests
@@ -1932,6 +1949,67 @@ mod tests {
         let _ = fs::remove_file(start_socket);
     }
 
+    #[tokio::test]
+    async fn restart_upload_failures_leave_both_peers_stopped() {
+        #[derive(serde::Serialize)]
+        #[serde(rename_all = "PascalCase")]
+        struct Summary {
+            id: &'static str,
+            state: &'static str,
+        }
+        #[derive(serde::Serialize)]
+        #[serde(rename_all = "PascalCase")]
+        struct State {
+            status: &'static str,
+            running: bool,
+        }
+        #[derive(serde::Serialize)]
+        #[serde(rename_all = "PascalCase")]
+        struct Inspect {
+            id: &'static str,
+            name: &'static str,
+            state: State,
+        }
+        for upload_index in [3, 4] {
+            let mut responses = vec![
+                StubResponse::new(
+                    StatusCode::OK,
+                    serde_json::to_vec(&[Summary {
+                        id: "ctr-1",
+                        state: "stopped",
+                    }])
+                    .unwrap(),
+                ),
+                StubResponse::new(
+                    StatusCode::OK,
+                    serde_json::to_vec(&Inspect {
+                        id: "ctr-1",
+                        name: "sandbox",
+                        state: State {
+                            status: "exited",
+                            running: false,
+                        },
+                    })
+                    .unwrap(),
+                ),
+            ];
+            let mut restart = restart_responses();
+            restart[upload_index] =
+                StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "upload failed");
+            responses.extend(restart.into_iter().take(upload_index + 1));
+            let (path, requests, server) = spawn_podman_stub("restart-upload-failure", responses);
+            assert!(test_driver(path).start_sandbox("sandbox-1").await.is_err());
+            server.await.unwrap();
+            assert!(
+                requests
+                    .lock()
+                    .unwrap()
+                    .iter()
+                    .all(|request| !request.ends_with("/start"))
+            );
+        }
+    }
+
     #[tokio::test]
     async fn stop_waits_for_the_container_to_leave_stopping_state() {
         let (socket, requests, handle) = spawn_podman_stub(
@@ -3150,6 +3228,12 @@ mod tests {
             ),
             StubResponse::new(StatusCode::OK, archive.into_inner().unwrap()),
             StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()),
+            StubResponse::new(StatusCode::OK, "").with_archive_members(&[
+                ".openshell",
+                ".openshell/supervisor",
+                ".openshell/supervisor/topology.payload",
+                ".openshell/supervisor/sandbox-bundle.tar",
+            ]),
             fence_response(),
             StubResponse::new(StatusCode::NO_CONTENT, ""), // workload start
             StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor start

crates/openshell-driver-podman/src/isolation.rs

@@ -226,6 +226,52 @@ pub fn bootstrap_archives(
     })
 }
 
+/// Retain only stable launch inputs from the supervisor-private restart copy.
+/// Every container restart establishes a new authenticated listener generation.
+pub fn rotate_bootstrap_archives(
+    sandbox_id: &str,
+    container_id: &str,
+    previous_channel: &[u8],
+) -> Result<BootstrapArchives, ComputeDriverError> {
+    let mut archive = tar::Archive::new(previous_channel);
+    for entry in archive.entries().map_err(invalid)? {
+        let entry = entry.map_err(invalid)?;
+        if entry.path().map_err(invalid)?.as_ref() != std::path::Path::new("sandbox/bootstrap.json")
+        {
+            continue;
+        }
+        if !entry.header().entry_type().is_file() {
+            return Err(invalid("restart bootstrap must be a regular file"));
+        }
+        let previous: BoundaryConfig = serde_json::from_reader(entry).map_err(invalid)?;
+        if previous.boundary_id != sandbox_id
+            || previous
+                .resource_claims
+                .get("podman.container_id")
+                .map(String::as_str)
+                != Some(container_id)
+            || !matches!(&previous.driver_fence, DriverFenceEvidence::Podman { container_id: bound, .. } if bound == container_id)
+        {
+            return Err(invalid(
+                "restart bootstrap does not belong to this Podman workload",
+            ));
+        }
+        previous
+            .driver_fence
+            .validate_for_backend("podman")
+            .map_err(invalid)?;
+        return bootstrap_archives(
+            sandbox_id,
+            container_id,
+            &previous.workload_identity,
+            previous.child_env,
+        );
+    }
+    Err(invalid(
+        "supervisor restart archive is missing sandbox/bootstrap.json",
+    ))
+}
+
 struct Archive<'a> {
     builder: tar::Builder<Vec<u8>>,
     identity: &'a ResolvedWorkloadIdentity,
@@ -379,4 +425,69 @@ mod tests {
             &archives.channel
         );
     }
+
+    #[test]
+    fn restart_rotates_every_authentication_value_while_preserving_launch_inputs() {
+        let identity = ResolvedWorkloadIdentity::new(
+            1000,
+            1001,
+            vec![],
+            "image".into(),
+            "sha256:image".into(),
+        )
+        .unwrap();
+        let original = bootstrap_archives(
+            "sandbox",
+            "container",
+            &identity,
+            HashMap::from([("PATH".into(), "/usr/bin".into())]),
+        )
+        .unwrap();
+        let renewed = rotate_bootstrap_archives("sandbox", "container", &original.channel).unwrap();
+        let original_files = files(&original.channel);
+        let renewed_files = files(&renewed.channel);
+        let supervisor_files = files(&renewed.supervisor);
+        let old: BoundaryConfig =
+            serde_json::from_slice(&original_files[&PathBuf::from("sandbox/bootstrap.json")])
+                .unwrap();
+        let new: BoundaryConfig =
+            serde_json::from_slice(&renewed_files[&PathBuf::from("sandbox/bootstrap.json")])
+                .unwrap();
+        let supervisor: BoundaryTopology = serde_json::from_slice(
+            &supervisor_files[&PathBuf::from(TOPOLOGY_PATH.trim_start_matches('/'))],
+        )
+        .unwrap();
+        assert_ne!(new.generation, old.generation);
+        assert_ne!(new.session_epoch, old.session_epoch);
+        assert_ne!(new.bootstrap_token, old.bootstrap_token);
+        assert_eq!(new.generation, supervisor.generation);
+        assert_eq!(new.session_epoch, supervisor.session_epoch);
+        assert_eq!(new.bootstrap_token, supervisor.bootstrap_token);
+        assert_eq!(new.workload_identity, old.workload_identity);
+        assert_eq!(new.resource_claims, old.resource_claims);
+        assert_eq!(new.child_env, old.child_env);
+        for path in [
+            "sandbox/server.crt",
+            "sandbox/server.key",
+            "sandbox/client-ca.crt",
+        ] {
+            assert_ne!(
+                renewed_files[&PathBuf::from(path)],
+                original_files[&PathBuf::from(path)]
+            );
+        }
+        let BoundaryTransport::Unix { tls, .. } = supervisor.transport else {
+            panic!("expected Unix channel")
+        };
+        assert_eq!(
+            tls.ca_certificate_pem.as_bytes(),
+            &renewed_files[&PathBuf::from("sandbox/client-ca.crt")]
+        );
+        assert_eq!(
+            supervisor_files[&PathBuf::from(RESTART_BUNDLE_PATH.trim_start_matches('/'))],
+            renewed.channel
+        );
+        assert!(rotate_bootstrap_archives("other", "container", &original.channel).is_err());
+        assert!(rotate_bootstrap_archives("sandbox", "other", &original.channel).is_err());
+    }
 }