From cbfbe2f790be207593daa2fbb0b5237715cdada5 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 18:25:37 -0700 Subject: [PATCH 01/25] feat(sandbox-backend): route runtime connections to shared supervisor sessions A supervisor shared by several sandboxes needs one listener for every session's runtime-initiated Sandbox Protocol connections. The runtime now sends a fixed, bounded routing preface (magic, version, length, and an opaque route handle) before its pinned TLS handshake when its bootstrap names a route. RuntimeRouter treats the handle only as a hint that selects which generation-pinned TLS identity to demand, closes unknown, removed, and replaced handles before any handshake, bounds unauthenticated handshakes globally and per route, and queues an authenticated connection only for its own session, so no peer can displace a live channel. A routed session waits up to the bootstrap grace for its first connection, because it is registered before its workload Pod is scheduled. The session_control module defines the trusted contract between a compute driver and a shared supervisor: registering one generation's runtime descriptor and gateway-issued credentials, reading its status, and removing it, over mutually authenticated TLS 1.3. Dedicated descriptors and bootstrap listeners keep their encodings. Signed-off-by: Drew Newberry --- .../src/isolation.rs | 1 + crates/openshell-sandbox-backend/README.md | 14 + .../src/boundary_protocol.rs | 14 + crates/openshell-sandbox-backend/src/lib.rs | 4 +- .../openshell-sandbox-backend/src/runtime.rs | 50 +- .../src/runtime_routed.rs | 492 ++++++++++++++++++ .../src/session_control.rs | 269 ++++++++++ .../openshell-sandbox/src/boundary_server.rs | 27 +- 8 files changed, 865 insertions(+), 6 deletions(-) create mode 100644 crates/openshell-sandbox-backend/src/runtime_routed.rs create mode 100644 crates/openshell-sandbox-backend/src/session_control.rs diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs index c9da7f3e3a..45ea8a5baa 100644 --- a/crates/openshell-driver-kubernetes/src/isolation.rs +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -400,6 +400,7 @@ impl KubernetesSandboxRuntimeBoundarySpec { authority: control_authority.clone(), addresses: vec![supervisor_address], tls: self.sandbox_tls, + route: None, }, SandboxTransport::RuntimeInitiatedTcp { authority: control_authority, diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index a63d52ebde..6a5df84128 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -124,6 +124,20 @@ open, so reconnects and credential rotation never wait for a new dial. The new variants leave existing encodings unchanged. Binaries that predate them reject runtime-initiated descriptors instead of falling back to another mode. +`SandboxTransport::RuntimeInitiatedRouted` serves the sessions of a supervisor +shared by many sandboxes from one listener (`runtime_routed::RuntimeRouter`). +The runtime first sends a fixed preface: `OSRP`, version `1`, a length byte, and +an opaque route handle of 16 to 64 lowercase letters, digits, or hyphens. The +handle is untrusted; it only selects which session's generation-pinned TLS +identity the supervisor then demands. Unknown, removed, and replaced handles are +closed before any handshake. Handshakes are bounded globally and per route, and +an authenticated connection only fills its own session's spare queue. A routed +session waits up to five minutes for its first connection, since it is +registered before its workload Pod is scheduled, and the usual connect window +afterwards. `session_control` defines the registration, status, and removal +contract between a compute driver and the shared supervisor, including its +mutually authenticated TLS configuration. + Runtime-initiated transport supports TCP, protected Unix sockets, and host vsock listeners. These share bounded TLS handshakes, the authenticated spare queue, and the same generation-pinned runtime identity check. The legacy diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index cf772b7118..93a1efabe1 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -269,6 +269,13 @@ pub enum SandboxTransport { guest_cid: u32, port: u32, }, + /// The runtime dials a supervisor shared by several sessions and names + /// this session's opaque route handle before the pinned TLS handshake. + /// Only a host that owns the shared listener can serve this transport. + RuntimeInitiatedRouted { + authority: String, + route: String, + }, } impl SandboxTransport { @@ -294,6 +301,9 @@ impl SandboxTransport { Self::Vsock { guest_cid, port } | Self::RuntimeInitiatedVsock { guest_cid, port } => { *guest_cid >= 3 && *guest_cid != u32::MAX && *port != 0 && *port != u32::MAX } + Self::RuntimeInitiatedRouted { authority, route } => { + !authority.is_empty() && crate::runtime_routed::validate_route_handle(route).is_ok() + } }; if valid { Ok(()) @@ -444,6 +454,10 @@ pub enum BoundaryListener { /// Driver-resolved supervisor endpoints, tried in order. addresses: Vec, tls: SandboxTlsServerConfig, + /// Route handle sent before TLS to a supervisor shared by several + /// sessions. Absent for a dedicated supervisor. + #[serde(default, skip_serializing_if = "Option::is_none")] + route: Option, }, /// Dial the supervisor over the driver's protected shared Unix socket. RuntimeInitiatedUnix { diff --git a/crates/openshell-sandbox-backend/src/lib.rs b/crates/openshell-sandbox-backend/src/lib.rs index e38e216d02..afb5e6561a 100644 --- a/crates/openshell-sandbox-backend/src/lib.rs +++ b/crates/openshell-sandbox-backend/src/lib.rs @@ -18,7 +18,9 @@ pub mod mediation; pub mod protobuf; mod runtime; mod runtime_initiated; +pub mod runtime_routed; pub mod sandbox_auth; +pub mod session_control; pub mod stream_adapter; pub use openshell_core::proto::sandbox::backend::v1 as backend_proto; @@ -27,7 +29,7 @@ pub use runtime::{ ConfirmedBoundary, ExecSession, NetworkMediationSource, OpenShellRuntimeBackend, ProcessAttachment, ReadyBoundary, RunningBoundary, }; -pub use runtime::{BoundaryTransport, BoundaryTransportConnector}; +pub use runtime::{BoundaryTransport, BoundaryTransportConnector, validate_runtime_coordinates}; /// Stable isolation backend name implemented by `openshell-sandbox`. pub const BACKEND_NAME: &str = openshell_core::isolation_registration::BUILTIN_BACKEND_NAME; diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index bc687e1b80..112e098f6c 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -50,6 +50,10 @@ const ATTACH_REQUEST_TIMEOUT: Duration = Duration::from_mins(5); /// callers retry whole calls above this; past boot, exhausting this window /// means the remote boundary (or its launcher) is gone rather than still starting. const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +/// Bound on a routed session's first connection. The session is registered +/// before its workload Pod is scheduled and its image pulled; the driver +/// reaps a bootstrap that is still incomplete after the same interval. +const ROUTED_FIRST_CONNECT_TIMEOUT: Duration = Duration::from_mins(5); /// Bounds the liveness check that decides whether a stream failure lost the /// whole connection. const CONNECTION_PROBE_TIMEOUT: Duration = Duration::from_secs(5); @@ -96,6 +100,8 @@ enum TransportSource { Connector(Arc), /// Accept streams the sandbox runtime opens to this supervisor. RuntimeInitiated(Arc), + /// Accept streams routed to this session by a shared listener. + Routed(Arc), } impl BoundaryTransport { @@ -109,10 +115,34 @@ impl BoundaryTransport { | SandboxTransport::RuntimeInitiatedVsock { .. } => TransportSource::RuntimeInitiated( crate::runtime_initiated::RuntimeInitiatedListener::bind(descriptor)?, ), + SandboxTransport::RuntimeInitiatedRouted { .. } => { + return Err(BackendError::Descriptor( + "a routed runtime-initiated transport requires a shared runtime listener" + .to_string(), + )); + } _ => TransportSource::Dial, })) } + /// Accept the connections a shared listener routes to one session. The + /// registration's handle must be the descriptor's route. + pub fn routed( + descriptor: &SandboxRuntimeDescriptor, + registration: Arc, + ) -> Result { + match &descriptor.transport { + SandboxTransport::RuntimeInitiatedRouted { route, .. } + if route == registration.handle() => + { + Ok(Self(TransportSource::Routed(registration))) + } + _ => Err(BackendError::Descriptor( + "runtime descriptor does not use this session's route".to_string(), + )), + } + } + /// Establish raw streams through a backend-owned connector. #[must_use] pub fn connector(connector: Arc) -> Self { @@ -137,6 +167,20 @@ impl BoundaryTransport { ) })?; } + TransportSource::Routed(registration) => { + let window = if registration.has_connected() { + CONNECT_RETRY_TIMEOUT + } else { + ROUTED_FIRST_CONNECT_TIMEOUT + }; + return tokio::time::timeout(window, registration.accept(&descriptor.tls)) + .await + .map_err(|_| { + BackendError::Unavailable( + "timed out waiting for the sandbox runtime to connect".to_string(), + ) + })?; + } }; let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; loop { @@ -1964,7 +2008,8 @@ impl BoundaryClient { let connector_transport = transport.clone(); let authority = match &runtime_descriptor.transport { SandboxTransport::Tcp { authority, .. } - | SandboxTransport::RuntimeInitiatedTcp { authority, .. } => authority.as_str(), + | SandboxTransport::RuntimeInitiatedTcp { authority, .. } + | SandboxTransport::RuntimeInitiatedRouted { authority, .. } => authority.as_str(), SandboxTransport::Unix { .. } | SandboxTransport::Vsock { .. } | SandboxTransport::RuntimeInitiatedUnix { .. } @@ -2134,7 +2179,8 @@ async fn connect_boundary_using_connector( SandboxTransport::Vsock { guest_cid, port } => connect_host_vsock(*guest_cid, *port)?, SandboxTransport::RuntimeInitiatedTcp { .. } | SandboxTransport::RuntimeInitiatedUnix { .. } - | SandboxTransport::RuntimeInitiatedVsock { .. } => { + | SandboxTransport::RuntimeInitiatedVsock { .. } + | SandboxTransport::RuntimeInitiatedRouted { .. } => { return Err(BackendError::Descriptor( "a runtime-initiated transport cannot be dialed".to_string(), )); diff --git a/crates/openshell-sandbox-backend/src/runtime_routed.rs b/crates/openshell-sandbox-backend/src/runtime_routed.rs new file mode 100644 index 0000000000..1444f72da4 --- /dev/null +++ b/crates/openshell-sandbox-backend/src/runtime_routed.rs @@ -0,0 +1,492 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One supervisor listener for many sessions' runtime-initiated connections. +//! +//! A runtime that dials a shared supervisor first sends a fixed, bounded +//! routing preface naming an opaque route handle. The handle is only a hint: +//! it selects which session's generation-pinned TLS identity the supervisor +//! expects, and the supervisor then completes the same pinned TLS client +//! handshake as for a dedicated listener. A peer that names another session's +//! handle cannot present that session's TLS identity, so it is rejected before +//! any request is carried. Every Sandbox Protocol request still carries the +//! session's bearer token. Unknown, removed, and replaced handles are closed +//! without a handshake, and authenticated connections only fill the target +//! session's bounded spare queue, so no peer can displace a live channel. + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use openshell_isolation_interface::contract::{BackendError, BoundaryDuplexStream}; +use tokio::io::{AsyncRead, AsyncReadExt as _}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}; + +use crate::boundary_protocol::SandboxTlsClientConfig; + +/// First bytes of a routing preface. +pub const ROUTE_PREFACE_MAGIC: [u8; 4] = *b"OSRP"; +/// Routing preface layout version. +pub const ROUTE_PREFACE_VERSION: u8 = 1; +/// Longest accepted route handle, in bytes. +pub const MAX_ROUTE_HANDLE_LEN: usize = 64; +const MIN_ROUTE_HANDLE_LEN: usize = 16; + +/// Unauthenticated sockets the listener handshakes at once, across sessions. +const MAX_PENDING_HANDSHAKES: usize = 64; +/// Unauthenticated sockets naming one handle; one handle cannot exhaust the +/// listener for every other session. +const MAX_PENDING_HANDSHAKES_PER_ROUTE: usize = 4; +/// Bound on reading the preface and completing TLS for one socket. +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); +/// Authenticated connections queued for a session but not yet claimed. +const MAX_QUEUED_CONNECTIONS: usize = 2; + +/// Validate an opaque route handle: 16 to 64 lowercase ASCII letters, digits, +/// or hyphens. +pub fn validate_route_handle(handle: &str) -> Result<(), BackendError> { + let valid = (MIN_ROUTE_HANDLE_LEN..=MAX_ROUTE_HANDLE_LEN).contains(&handle.len()) + && handle + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-'); + if valid { + Ok(()) + } else { + Err(BackendError::Descriptor( + "runtime route handle must be 16 to 64 lowercase letters, digits, or hyphens" + .to_string(), + )) + } +} + +/// Generate a fresh opaque route handle for one runtime generation. +#[must_use] +pub fn new_route_handle() -> String { + format!("rt-{}", uuid::Uuid::new_v4().simple()) +} + +/// Encode the routing preface a runtime sends before its TLS handshake. +pub fn encode_route_preface(handle: &str) -> Result, BackendError> { + validate_route_handle(handle)?; + let length = u8::try_from(handle.len()) + .map_err(|_| BackendError::Descriptor("runtime route handle is too long".to_string()))?; + let mut preface = Vec::with_capacity(ROUTE_PREFACE_MAGIC.len() + 2 + handle.len()); + preface.extend_from_slice(&ROUTE_PREFACE_MAGIC); + preface.push(ROUTE_PREFACE_VERSION); + preface.push(length); + preface.extend_from_slice(handle.as_bytes()); + Ok(preface) +} + +/// Read and validate one routing preface. Reads only the declared bytes. +async fn read_route_preface(stream: &mut S) -> std::io::Result { + let invalid = |message: &str| std::io::Error::new(std::io::ErrorKind::InvalidData, message); + let mut header = [0_u8; 6]; + stream.read_exact(&mut header).await?; + if header[..4] != ROUTE_PREFACE_MAGIC { + return Err(invalid("missing runtime routing preface")); + } + if header[4] != ROUTE_PREFACE_VERSION { + return Err(invalid("unsupported runtime routing preface version")); + } + let length = usize::from(header[5]); + if length > MAX_ROUTE_HANDLE_LEN { + return Err(invalid("runtime route handle is too long")); + } + let mut handle = vec![0_u8; length]; + stream.read_exact(&mut handle).await?; + let handle = String::from_utf8(handle).map_err(|_| invalid("invalid runtime route handle"))?; + validate_route_handle(&handle).map_err(|error| invalid(&error.to_string()))?; + Ok(handle) +} + +struct Route { + /// Distinguishes a re-registered handle from the registration it replaced. + id: u64, + tls: SandboxTlsClientConfig, + spares: mpsc::Sender, + pending: Arc, +} + +#[derive(Default)] +struct Routes { + next_id: u64, + by_handle: HashMap, +} + +/// Shared listener that routes runtime connections to registered sessions. +pub struct RuntimeRouter { + routes: Arc>, + local_addr: SocketAddr, + accept_task: tokio::task::AbortHandle, +} + +impl std::fmt::Debug for RuntimeRouter { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RuntimeRouter") + .field("local_addr", &self.local_addr) + .finish_non_exhaustive() + } +} + +impl RuntimeRouter { + /// Bind `listen` and start routing authenticated runtime connections. + pub fn bind(listen: SocketAddr) -> Result, BackendError> { + let runtime = tokio::runtime::Handle::try_current().map_err(|_| { + BackendError::Unavailable("runtime router requires a Tokio runtime".into()) + })?; + let _entered = runtime.enter(); + let listener = bind_tcp(listen).map_err(|error| { + BackendError::Unavailable(format!("runtime router listener: {error}")) + })?; + let local_addr = listener.local_addr().map_err(|error| { + BackendError::Unavailable(format!("runtime router listener: {error}")) + })?; + let routes = Arc::new(Mutex::new(Routes::default())); + let accept_task = runtime + .spawn(accept_loop(listener, routes.clone())) + .abort_handle(); + tracing::info!(%local_addr, "Routed runtime-initiated Sandbox Protocol listener ready"); + Ok(Arc::new(Self { + routes, + local_addr, + accept_task, + })) + } + + #[must_use] + pub fn local_addr(&self) -> SocketAddr { + self.local_addr + } + + /// Accept connections for `handle`, authenticated with `tls`. A handle + /// that is already registered is refused; the existing session keeps it. + pub fn register( + &self, + handle: &str, + tls: &SandboxTlsClientConfig, + ) -> Result, BackendError> { + validate_route_handle(handle)?; + crate::runtime::validate_client_tls(tls)?; + let mut routes = lock(&self.routes); + if routes.by_handle.contains_key(handle) { + return Err(BackendError::Denied( + "runtime route handle is already registered".to_string(), + )); + } + routes.next_id += 1; + let id = routes.next_id; + let (sender, spares) = mpsc::channel(MAX_QUEUED_CONNECTIONS); + routes.by_handle.insert( + handle.to_string(), + Route { + id, + tls: tls.clone(), + spares: sender, + pending: Arc::new(Semaphore::new(MAX_PENDING_HANDSHAKES_PER_ROUTE)), + }, + ); + Ok(Arc::new(RouteRegistration { + routes: Arc::downgrade(&self.routes), + handle: handle.to_string(), + id, + tls: tls.clone(), + spares: tokio::sync::Mutex::new(spares), + connected: AtomicBool::new(false), + })) + } +} + +impl Drop for RuntimeRouter { + fn drop(&mut self) { + self.accept_task.abort(); + } +} + +/// One session's claim on a route handle. Dropping it unregisters the handle +/// and closes connections queued for it. +pub struct RouteRegistration { + routes: std::sync::Weak>, + handle: String, + id: u64, + tls: SandboxTlsClientConfig, + spares: tokio::sync::Mutex>, + /// The runtime has delivered at least one authenticated connection. + connected: AtomicBool, +} + +impl std::fmt::Debug for RouteRegistration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RouteRegistration") + .field("handle", &self.handle) + .finish_non_exhaustive() + } +} + +impl RouteRegistration { + #[must_use] + pub fn handle(&self) -> &str { + &self.handle + } + + /// Wait for one authenticated connection for `descriptor`'s session. + pub(crate) async fn accept( + &self, + tls: &SandboxTlsClientConfig, + ) -> Result { + if *tls != self.tls { + return Err(BackendError::Descriptor( + "runtime route is registered for a different sandbox session".to_string(), + )); + } + let stream = self + .spares + .lock() + .await + .recv() + .await + .ok_or_else(|| BackendError::Unavailable("runtime router stopped".to_string()))?; + self.connected.store(true, Ordering::Relaxed); + Ok(stream) + } + + /// Whether the runtime has connected for this session before. + pub(crate) fn has_connected(&self) -> bool { + self.connected.load(Ordering::Relaxed) + } +} + +impl Drop for RouteRegistration { + fn drop(&mut self) { + if let Some(routes) = self.routes.upgrade() { + let mut routes = lock(&routes); + if routes + .by_handle + .get(&self.handle) + .is_some_and(|route| route.id == self.id) + { + routes.by_handle.remove(&self.handle); + } + } + } +} + +fn lock(routes: &Mutex) -> std::sync::MutexGuard<'_, Routes> { + routes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn bind_tcp(address: SocketAddr) -> std::io::Result { + let socket = socket2::Socket::new( + socket2::Domain::for_address(address), + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + socket.set_reuse_address(true)?; + socket.set_nonblocking(true)?; + socket.bind(&address.into())?; + socket.listen(128)?; + tokio::net::TcpListener::from_std(socket.into()) +} + +/// Route each accepted socket within global and per-handle handshake bounds. +async fn accept_loop(listener: tokio::net::TcpListener, routes: Arc>) { + let pending = Arc::new(Semaphore::new(MAX_PENDING_HANDSHAKES)); + // Owned here so aborting the listener also cancels pending handshakes. + let mut handshakes = tokio::task::JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let stream = match accepted { + Ok((stream, _)) => stream, + Err(error) => { + // Descriptor exhaustion and similar errors are transient. + tracing::warn!(%error, "routed runtime accept failed"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let Ok(permit) = pending.clone().try_acquire_owned() else { + tracing::debug!("routed runtime handshake limit reached; closing connection"); + continue; + }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + crate::runtime::enable_boundary_tcp_keepalive(&stream); + let routes = routes.clone(); + handshakes.spawn(async move { + if let Err(error) = route_connection(stream, &routes, permit).await { + tracing::debug!(%error, "rejected routed runtime connection"); + } + }); + } + Some(_) = handshakes.join_next(), if !handshakes.is_empty() => {} + } + } +} + +async fn route_connection( + mut stream: tokio::net::TcpStream, + routes: &Mutex, + _listener_permit: OwnedSemaphorePermit, +) -> Result<(), String> { + let deadline = tokio::time::Instant::now() + HANDSHAKE_TIMEOUT; + let handle = tokio::time::timeout_at(deadline, read_route_preface(&mut stream)) + .await + .map_err(|_| "routing preface timed out".to_string())? + .map_err(|error| format!("routing preface: {error}"))?; + // The handle only selects which pinned identity to expect. + let (tls, spares, route_pending) = { + let routes = lock(routes); + let route = routes + .by_handle + .get(&handle) + .ok_or_else(|| "unknown or retired runtime route handle".to_string())?; + ( + route.tls.clone(), + route.spares.clone(), + route.pending.clone(), + ) + }; + let Ok(_route_permit) = route_pending.try_acquire_owned() else { + return Err("runtime route handshake limit reached".to_string()); + }; + let stream: BoundaryDuplexStream = Box::new(stream); + let authenticated = tokio::time::timeout_at( + deadline, + crate::runtime::authenticate_boundary_stream(&tls, stream), + ) + .await + .map_err(|_| "runtime TLS handshake timed out".to_string())? + .map_err(|error| error.to_string())?; + // A sender captured before its registration was removed or replaced has + // no receiver, so the stream cannot reach another generation. + spares + .try_send(authenticated) + .map_err(|_| "runtime route has no free spare slot".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::boundary_protocol::{SandboxTlsMaterial, generate_sandbox_tls_material}; + use tokio::io::AsyncWriteExt as _; + + /// A runtime: dial the router, name `route`, and serve `identity`'s TLS. + async fn runtime_dial( + router: SocketAddr, + route: &str, + identity: &SandboxTlsMaterial, + ) -> std::io::Result> { + let mut stream = tokio::net::TcpStream::connect(router).await?; + stream + .write_all(&encode_route_preface(route).unwrap()) + .await?; + let mut config = rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::aws_lc_rs::default_provider(), + )) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_no_client_auth() + .with_single_cert( + rustls_pemfile::certs(&mut identity.certificate_chain_pem.as_bytes()) + .collect::, _>>() + .unwrap(), + rustls_pemfile::private_key(&mut identity.private_key_pem.as_bytes()) + .unwrap() + .unwrap(), + ) + .unwrap(); + config.alpn_protocols = vec![b"h2".to_vec()]; + tokio_rustls::TlsAcceptor::from(Arc::new(config)) + .accept(stream) + .await + } + + fn client_tls(identity: &SandboxTlsMaterial) -> SandboxTlsClientConfig { + SandboxTlsClientConfig { + server_name: identity.server_name.clone(), + trust_anchor_pem: identity.trust_anchor_pem.clone(), + } + } + + async fn delivered(registration: &RouteRegistration, tls: &SandboxTlsClientConfig) -> bool { + tokio::time::timeout(Duration::from_millis(300), registration.accept(tls)) + .await + .is_ok_and(|accepted| accepted.is_ok()) + } + + #[tokio::test] + async fn routes_only_authenticated_runtimes_to_their_own_live_session() { + let router = RuntimeRouter::bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let address = router.local_addr(); + let alpha = generate_sandbox_tls_material(openshell_core::SandboxSessionId::new()).unwrap(); + let beta = generate_sandbox_tls_material(openshell_core::SandboxSessionId::new()).unwrap(); + let (alpha_route, beta_route) = (new_route_handle(), new_route_handle()); + let alpha_registration = router.register(&alpha_route, &client_tls(&alpha)).unwrap(); + let beta_registration = router.register(&beta_route, &client_tls(&beta)).unwrap(); + assert!( + router.register(&beta_route, &client_tls(&alpha)).is_err(), + "a live route cannot be claimed again" + ); + + // A runtime holding the named session's identity is delivered to it. + let alpha_tls = client_tls(&alpha); + let (runtime, accepted) = tokio::join!( + runtime_dial(address, &alpha_route, &alpha), + alpha_registration.accept(&alpha_tls) + ); + assert!(runtime.is_ok() && accepted.is_ok()); + + // Naming another session without its identity fails TLS, and + // nothing reaches either session. + assert!(runtime_dial(address, &beta_route, &alpha).await.is_err()); + assert!(!delivered(&beta_registration, &client_tls(&beta)).await); + assert!(!delivered(&alpha_registration, &client_tls(&alpha)).await); + + // Unknown and retired routes are closed before any TLS handshake. + assert!( + runtime_dial(address, &new_route_handle(), &beta) + .await + .is_err() + ); + drop(alpha_registration); + assert!(runtime_dial(address, &alpha_route, &alpha).await.is_err()); + + // The other session is unaffected and keeps accepting its runtime. + let beta_tls = client_tls(&beta); + let (runtime, accepted) = tokio::join!( + runtime_dial(address, &beta_route, &beta), + beta_registration.accept(&beta_tls) + ); + assert!(runtime.is_ok() && accepted.is_ok()); + + // A retired route can be registered again by a new owner. + assert!(router.register(&alpha_route, &client_tls(&alpha)).is_ok()); + } + + #[tokio::test] + async fn preface_round_trips_and_rejects_malformed_input() { + let handle = new_route_handle(); + let encoded = encode_route_preface(&handle).unwrap(); + assert_eq!( + read_route_preface(&mut encoded.as_slice()).await.unwrap(), + handle + ); + let mut wrong_version = encoded.clone(); + wrong_version[4] = 2; + for malformed in [ + b"TLS\x16\x03\x01".to_vec(), + wrong_version, + [b"OSRP\x01\x41".as_slice(), &[b'a'; 65]].concat(), + [b"OSRP\x01\x10".as_slice(), b"UPPERCASE-HANDLE"].concat(), + ] { + assert!(read_route_preface(&mut malformed.as_slice()).await.is_err()); + } + assert!(encode_route_preface("short").is_err()); + } +} diff --git a/crates/openshell-sandbox-backend/src/session_control.rs b/crates/openshell-sandbox-backend/src/session_control.rs new file mode 100644 index 0000000000..3756244548 --- /dev/null +++ b/crates/openshell-sandbox-backend/src/session_control.rs @@ -0,0 +1,269 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Contract between a compute driver and a supervisor shared by sessions. +//! +//! The driver registers one runtime generation with trusted launch material +//! the gateway issued for it, reads that session's status, and removes it. +//! The exchange runs over mutually authenticated TLS with a credential +//! reserved for session admission; it is never available to a sandbox. A +//! registration grants the supervisor exactly the per-generation gateway and +//! Sandbox Protocol credentials it carries; it confers no other identity. + +use std::sync::Arc; + +use openshell_isolation_interface::contract::BackendError; +use serde::{Deserialize, Serialize}; + +use crate::boundary_protocol::{SandboxRuntimeDescriptor, SandboxTransport}; + +/// Collection path of the session-control API; a session is `{prefix}/{route}`. +pub const SESSIONS_PATH: &str = "/v1/sessions"; +/// Largest registration body a supervisor accepts. +pub const MAX_REGISTRATION_BYTES: usize = 256 * 1024; + +/// PEM private key that never appears in `Debug` output. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct SecretPem(String); + +impl SecretPem { + #[must_use] + pub fn new(pem: String) -> Self { + Self(pem) + } + + #[must_use] + pub fn expose_secret(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Debug for SecretPem { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("SecretPem([REDACTED])") + } +} + +/// Proxy interception CA issued for one generation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProxyCaMaterial { + pub certificate_pem: String, + pub private_key_pem: SecretPem, +} + +/// W3C trace context of the operation that created the generation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RegistrationTraceContext { + pub traceparent: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tracestate: Option, +} + +/// Trusted launch material for one runtime generation on a shared supervisor. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SessionRegistration { + /// Route handle the runtime names; also the session's key on the host. + pub route: String, + pub sandbox_id: String, + pub sandbox_name: String, + /// Supervisor-side launch descriptor using the routed transport. + pub runtime_descriptor: SandboxRuntimeDescriptor, + /// Gateway-issued credentials for exactly this generation. + pub auth: openshell_core::jwt::SupervisorAuthBundle, + /// Encoded `MainProcessConfig` for the canonical process. + pub main_process_spec: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workdir: Option, + /// Workload image recorded in the session's OCSF events. + #[serde(default)] + pub container_image: String, + pub proxy_ca: ProxyCaMaterial, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trace_context: Option, +} + +impl std::fmt::Debug for SessionRegistration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SessionRegistration") + .field("route", &self.route) + .field("sandbox_id", &self.sandbox_id) + .field("sandbox_name", &self.sandbox_name) + .field("runtime_descriptor", &self.runtime_descriptor) + .field("auth", &self.auth) + .field("workdir", &self.workdir) + .field("container_image", &self.container_image) + .finish_non_exhaustive() + } +} + +impl SessionRegistration { + /// Check that every part of the registration names the same sandbox, + /// generation, launch session, and route. + pub fn validate(&self) -> Result<(), BackendError> { + crate::runtime_routed::validate_route_handle(&self.route)?; + let descriptor = &self.runtime_descriptor; + match &descriptor.transport { + SandboxTransport::RuntimeInitiatedRouted { route, .. } if *route == self.route => {} + _ => { + return Err(BackendError::Descriptor( + "shared supervisor sessions require this route's routed transport".to_string(), + )); + } + } + crate::runtime::validate_runtime_coordinates(descriptor)?; + if self.sandbox_id.is_empty() || descriptor.boundary_id != self.sandbox_id { + return Err(BackendError::Descriptor( + "runtime descriptor does not match the registered sandbox".to_string(), + )); + } + if descriptor.generation != self.auth.runtime_generation.as_str() + || descriptor.session_id != self.auth.session_id + { + return Err(BackendError::Descriptor( + "launch credentials do not match the runtime generation".to_string(), + )); + } + self.auth + .validate() + .map_err(|error| BackendError::Descriptor(error.to_string()))?; + openshell_core::sandbox_env::MainProcessConfig::decode(&self.main_process_spec) + .map_err(BackendError::Descriptor)?; + if self.proxy_ca.certificate_pem.is_empty() + || self.proxy_ca.private_key_pem.expose_secret().is_empty() + { + return Err(BackendError::Descriptor( + "proxy CA material is required".to_string(), + )); + } + Ok(()) + } +} + +/// Lifecycle phase of a hosted session. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case")] +pub enum SessionPhaseReport { + Starting, + Supervising, + Stopping, + Exited { exit_code: i32 }, + Failed { error: String }, +} + +impl SessionPhaseReport { + /// Supervision has ended and cannot resume for this generation. + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Exited { .. } | Self::Failed { .. }) + } +} + +/// Status of one hosted session. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SessionStatusReport { + pub route: String, + pub sandbox_id: String, + #[serde(flatten)] + pub phase: SessionPhaseReport, + /// The gateway currently accepts this session's supervision. + pub ready: bool, +} + +/// Every session a shared supervisor hosts. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SessionList { + pub sessions: Vec, +} + +fn pem_certificates( + pem: &[u8], + what: &str, +) -> Result>, BackendError> { + let certificates = rustls_pemfile::certs(&mut &*pem) + .collect::, _>>() + .map_err(|error| BackendError::Descriptor(format!("parse {what}: {error}")))?; + if certificates.is_empty() { + return Err(BackendError::Descriptor(format!( + "{what} contains no certificates" + ))); + } + Ok(certificates) +} + +fn pem_private_key(pem: &[u8]) -> Result, BackendError> { + rustls_pemfile::private_key(&mut &*pem) + .map_err(|error| { + BackendError::Descriptor(format!("parse session-control private key: {error}")) + })? + .ok_or_else(|| { + BackendError::Descriptor("session-control private key PEM is empty".to_string()) + }) +} + +fn root_store(ca_pem: &[u8]) -> Result { + let mut roots = rustls::RootCertStore::empty(); + for certificate in pem_certificates(ca_pem, "session-control CA")? { + roots.add(certificate).map_err(|error| { + BackendError::Descriptor(format!("load session-control CA: {error}")) + })?; + } + Ok(roots) +} + +/// Server TLS for a shared supervisor: TLS 1.3, and only clients presenting a +/// certificate issued by the session-control CA. +pub fn server_tls_config( + certificate_pem: &[u8], + private_key_pem: &[u8], + client_ca_pem: &[u8], +) -> Result, BackendError> { + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let verifier = rustls::server::WebPkiClientVerifier::builder_with_provider( + Arc::new(root_store(client_ca_pem)?), + provider.clone(), + ) + .build() + .map_err(|error| { + BackendError::Descriptor(format!("build session-control client verifier: {error}")) + })?; + let config = rustls::ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&rustls::version::TLS13]) + .map_err(|error| BackendError::Descriptor(error.to_string()))? + .with_client_cert_verifier(verifier) + .with_single_cert( + pem_certificates(certificate_pem, "session-control certificate")?, + pem_private_key(private_key_pem)?, + ) + .map_err(|error| { + BackendError::Descriptor(format!("load session-control certificate: {error}")) + })?; + Ok(Arc::new(config)) +} + +/// Client TLS for a compute driver: TLS 1.3, trusting only the +/// session-control CA, and presenting the admission client certificate. +pub fn client_tls_config( + certificate_pem: &[u8], + private_key_pem: &[u8], + server_ca_pem: &[u8], +) -> Result, BackendError> { + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let config = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&rustls::version::TLS13]) + .map_err(|error| BackendError::Descriptor(error.to_string()))? + .with_root_certificates(root_store(server_ca_pem)?) + .with_client_auth_cert( + pem_certificates(certificate_pem, "session-control client certificate")?, + pem_private_key(private_key_pem)?, + ) + .map_err(|error| { + BackendError::Descriptor(format!("load session-control client certificate: {error}")) + })?; + Ok(Arc::new(config)) +} diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 578522e4c3..56a974b9a1 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -382,7 +382,14 @@ mod linux { authority, addresses, tls, - } => validate_runtime_initiated_targets(authority, addresses, tls)?, + route, + } => { + validate_runtime_initiated_targets(authority, addresses, tls)?; + if let Some(route) = route { + openshell_sandbox_backend::runtime_routed::validate_route_handle(route) + .map_err(|error| error.to_string())?; + } + } BoundaryListenerConfig::Unix { .. } | BoundaryListenerConfig::RuntimeInitiatedUnix { .. } | BoundaryListenerConfig::RuntimeInitiatedVsock { .. } @@ -742,16 +749,30 @@ mod linux { } /// Dial only driver-provided supervisor addresses from the bootstrap. + /// + /// A shared supervisor first receives this generation's route handle so + /// it can expect the right pinned identity; this runtime then serves TLS + /// exactly as it does for a dedicated supervisor. async fn dial_supervisor(config: &BoundaryListenerConfig) -> io::Result { match config { - BoundaryListenerConfig::RuntimeInitiatedTlsTcp { addresses, .. } => { - let stream = + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { + addresses, route, .. + } => { + let mut stream = openshell_core::net::connect_tcp_nodelay_best_effort(addresses).await?; let keepalive = socket2::TcpKeepalive::new() .with_time(Duration::from_secs(10)) .with_interval(Duration::from_secs(5)) .with_retries(3); let _ = socket2::SockRef::from(&stream).set_tcp_keepalive(&keepalive); + if let Some(route) = route { + let preface = + openshell_sandbox_backend::runtime_routed::encode_route_preface(route) + .map_err(|error| { + io::Error::new(io::ErrorKind::InvalidInput, error.to_string()) + })?; + stream.write_all(&preface).await?; + } Ok(PlainControlStream::Tcp(stream.into_std()?)) } BoundaryListenerConfig::RuntimeInitiatedUnix { socket_path, .. } => { From 5591b1eeaa9c41ea3b5afa9af652868fa024b7cb Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 18:26:26 -0700 Subject: [PATCH 02/25] feat(supervisor): host gateway-registered sessions in one process openshell-supervisor experimental-host replaces the idle bootstrap process with a session host. A mutually authenticated session-control API accepts registrations only from the admission certificate holder, the gateway. Each registration starts a SessionHost session that acts only with the gateway credentials issued for its generation, keeps its proxy CA, trust directory, and SSH socket under a private directory, and receives its runtime connections through the routed listener. Status reports each session's own readiness; the health socket keeps reporting process health only. Removing a session runs on a task the host owns, so it completes when the caller disconnects, concurrent removers share its result, and the route stays reserved until the session has ended and its files are gone. An identical registration is accepted again and a different one for a registered route is refused. Sessions get no upstream proxy, extra CA bundle, or provider workload identity; the Kubernetes driver refuses configurations that need them. Signed-off-by: Drew Newberry --- Cargo.lock | 2 + crates/openshell-supervisor/Cargo.toml | 14 +- crates/openshell-supervisor/README.md | 43 +- .../openshell-supervisor/src/backend_setup.rs | 54 ++ crates/openshell-supervisor/src/lib.rs | 1 + crates/openshell-supervisor/src/main.rs | 40 +- crates/openshell-supervisor/src/shared.rs | 159 +++- .../openshell-supervisor/src/shared_host.rs | 681 ++++++++++++++++++ 8 files changed, 914 insertions(+), 80 deletions(-) create mode 100644 crates/openshell-supervisor/src/shared_host.rs diff --git a/Cargo.lock b/Cargo.lock index 025ba026d9..42b642600d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5121,11 +5121,13 @@ dependencies = [ "rustls", "serde", "serde_json", + "sha2 0.10.9", "socket2", "temp-env", "tempfile", "thiserror 2.0.20", "tokio", + "tokio-rustls", "tokio-tungstenite 0.26.2", "tonic", "tonic-prost", diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index a42c370404..f343147f87 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -28,7 +28,13 @@ openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-mid openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } openshell-supervisor-process = { path = "../openshell-supervisor-process" } +bytes = { workspace = true } clap = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } miette = { workspace = true } nix = { workspace = true } opentelemetry = { workspace = true } @@ -38,10 +44,12 @@ rustls = { workspace = true } rustix = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } socket2 = { workspace = true } tempfile = "3" thiserror = { workspace = true } tokio = { workspace = true } +tokio-rustls = { workspace = true } tonic = { workspace = true, features = ["channel", "tls-native-roots"] } tracing = { workspace = true } tracing-appender = { workspace = true } @@ -58,13 +66,7 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } tonic-prost = { workspace = true } -bytes = { workspace = true } -http = { workspace = true } -http-body-util = { workspace = true } -hyper = { workspace = true } -hyper-util = { workspace = true } tokio = { workspace = true, features = ["test-util"] } -futures = { workspace = true } temp-env = "0.3" tokio-tungstenite = { workspace = true } diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 12c08cebb0..8a10177189 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -97,15 +97,34 @@ belong to the sandbox or isolation backend. Signals use `BoundarySignal`. These portable control-plane foundations do not qualify a platform isolation backend or enable a Windows workload runtime. -## Shared host - -`openshell-supervisor --mode=shared --gateway-endpoint https://gateway:8080 --address 0.0.0.0:5501` -starts a shared-host placeholder for the Kubernetes gateway supervisor topology. -The listener at `--address` accepts and closes connections, including kubelet -`tcpSocket` probes, and reports process health only. `--gateway-endpoint` selects -the gateway the supervisor will dial for session control. The outbound connection, -control RPCs, sandbox sessions, gateway authentication, and tenant routing are not -implemented yet. Omitting `--mode` keeps the existing dedicated behavior. -`supervisor_topology = "gateway"` deploys this placeholder; sandboxes -continue using dedicated supervisors. SIGTERM and SIGINT close the listener. -Shared mode is Unix-only. +## Shared supervisor host + +`openshell-supervisor experimental-host` hosts the sessions of many sandboxes in +one process for the Kubernetes gateway supervisor topology. It is Unix-only. + +- **Admission.** A mutually authenticated TLS 1.3 listener + (`--session-control-listen`, credentials in `--session-control-tls-dir`) + serves `PUT`, `GET`, and `DELETE /v1/sessions/{route}` and `GET /v1/sessions` + as JSON. Only a client holding the admission certificate, which Helm gives + only the gateway, can register sessions. A registration carries one + generation's runtime descriptor and gateway-issued credentials; the host + never acts with any other identity and has no Kubernetes identity. An + identical registration is accepted again; a different one for a registered + route returns `409`. +- **Sessions.** `shared_host::SharedHost` starts each registration as a + `SessionHost` session with private files under `--session-root` (proxy CA, + trust directory, SSH socket). Removal runs on a task the host owns, so it + completes when the caller disconnects, and the route stays reserved until the + session has ended and its files are deleted. +- **Runtime connections.** One routed listener (`--runtime-listen`) accepts every + session's runtime-initiated Sandbox Protocol connections; see + `openshell-sandbox-backend` for the routing preface. +- **Health.** `--health-socket` serves the existing `health --socket` check and + reports process health only. Session readiness is reported per session over + session control. SIGTERM and SIGINT stop every session, which ends their + workloads' supervision. + +Sessions get no upstream proxy, extra CA bundle, OCSF OTLP export, or provider +workload identity: the Kubernetes driver refuses the topology when an upstream +proxy or provider SPIFFE identity is configured, so a workload never silently +loses them. diff --git a/crates/openshell-supervisor/src/backend_setup.rs b/crates/openshell-supervisor/src/backend_setup.rs index fe11068b52..72b4c30129 100644 --- a/crates/openshell-supervisor/src/backend_setup.rs +++ b/crates/openshell-supervisor/src/backend_setup.rs @@ -242,6 +242,60 @@ impl BackendSetup for OpenShellBackendSetup { } } +/// The built-in Sandbox Protocol for one session of a shared supervisor. +/// +/// The descriptor must use the routed transport for exactly `route`; +/// decoding claims that route on the shared listener for as long as the +/// session's backend exists. +pub struct RoutedBackendSetup { + router: Arc, + route: String, +} + +impl RoutedBackendSetup { + #[must_use] + pub fn new( + router: Arc, + route: String, + ) -> Self { + Self { router, route } + } +} + +impl BackendSetup for RoutedBackendSetup { + fn backend_name(&self) -> &str { + openshell_sandbox_backend::BACKEND_NAME + } + + fn decode( + &self, + payload: &[u8], + ) -> std::result::Result<(LaunchIdentity, Box), BackendError> { + let descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor = + serde_json::from_slice(payload).map_err(|error| { + BackendError::Descriptor(format!("decode sandbox runtime descriptor: {error}")) + })?; + openshell_sandbox_backend::validate_runtime_coordinates(&descriptor)?; + let registration = self.router.register(&self.route, &descriptor.tls)?; + let transport = + openshell_sandbox_backend::BoundaryTransport::routed(&descriptor, registration)?; + let identity = LaunchIdentity { + sandbox_id: descriptor.boundary_id.clone(), + generation: descriptor.generation.clone(), + session_id: descriptor.session_id, + workload_identity: descriptor.workload_identity.clone(), + vm_policy_identity: None, + }; + Ok(( + identity, + Box::new(OpenShellLaunch { + descriptor, + transport, + }), + )) + } +} + /// Launch data plus the transport chosen for it. Discovery and the attached /// backend use clones of one transport, so a runtime-initiated descriptor binds /// its control listener once for the supervisor's lifetime. diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index bd402b5d38..7aba2f703e 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -25,6 +25,7 @@ mod endpoint_status; mod mechanistic_mapper; mod provider_readiness; pub mod session_host; +pub mod shared_host; use config_runtime::{agent_proposals_enabled_from_settings, run_policy_poll_loop}; use miette::{IntoDiagnostic, Result, WrapErr}; diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index daafbc364b..694f1eeeaf 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -24,15 +24,6 @@ const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; const HEALTH_SUBCOMMAND: &str = "health"; const SERVICE_NAME: &str = "openshell-supervisor"; -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] -enum SupervisorMode { - /// Supervise a single sandbox generation. - #[default] - Dedicated, - /// Host sandbox supervision sessions in a shared process. - Shared, -} - #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] enum SupervisorRole { /// Attach an Isolation Backend and supervise one sandbox generation. @@ -56,18 +47,6 @@ struct HealthArgs { #[command(about = "OpenShell policy and workload supervisor")] #[allow(clippy::struct_excessive_bools)] struct Args { - /// Supervisor deployment mode. - #[arg(long, value_enum, default_value_t)] - mode: SupervisorMode, - - /// Gateway endpoint for the shared supervisor's outbound connection. - #[arg(long, required_if_eq("mode", "shared"))] - gateway_endpoint: Option, - - /// Shared supervisor listen address, also used for TCP health probes. - #[arg(long, required_if_eq("mode", "shared"))] - address: Option, - /// Supervisor responsibility to run. #[arg(long, value_enum, default_value_t)] role: SupervisorRole, @@ -387,22 +366,11 @@ fn main() -> Result<()> { return openshell_supervisor::check_control_readiness(&args.socket); } - let args = Args::parse(); - if args.mode == SupervisorMode::Shared { - let address = args - .address - .ok_or_else(|| miette::miette!("--address is required for --mode=shared"))?; - let gateway_endpoint = args - .gateway_endpoint - .as_deref() - .ok_or_else(|| miette::miette!("--gateway-endpoint is required for --mode=shared"))?; - return shared::run(address, gateway_endpoint); - } - if args.gateway_endpoint.is_some() || args.address.is_some() { - return Err(miette::miette!( - "--gateway-endpoint and --address require --mode=shared" - )); + if raw_args.get(1).map(String::as_str) == Some("experimental-host") { + return shared::run(&raw_args[1..]); } + + let args = Args::parse(); validate_role_arguments(&args)?; arm_parent_liveness(args.parent_liveness_stdin)?; validate_main_exit_marker(args.main_exit_marker.as_deref())?; diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index bc67386fd4..d4e4530640 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -1,26 +1,89 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Shared-host placeholder: no sandbox admission, credentials, or session RPCs. +//! Supervisor shared by the sandboxes of one gateway (experimental). -#[cfg(unix)] -use miette::IntoDiagnostic; -use miette::Result; +use clap::Parser; +use miette::{IntoDiagnostic, Result, WrapErr}; use std::net::SocketAddr; +use std::path::PathBuf; +use std::time::Duration; + +/// Time a removed or shut-down session may take to stop its workload. +const SESSION_REMOVAL_GRACE: Duration = Duration::from_secs(20); + +#[derive(Debug, Parser)] +#[command(name = "openshell-supervisor experimental-host")] +#[command(about = "Supervise the sandboxes a gateway registers with this process")] +struct Args { + /// Private socket reporting process health, not session readiness. + #[arg(long)] + health_socket: PathBuf, + + /// Gateway gRPC endpoint; each session authenticates with its own credentials. + #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] + openshell_endpoint: String, + + /// Mutually authenticated session-control listener used by the gateway. + #[arg(long)] + session_control_listen: SocketAddr, + + /// Directory with the session-control `tls.crt`, `tls.key`, and client `ca.crt`. + #[arg(long)] + session_control_tls_dir: PathBuf, + + /// Listener for routed runtime-initiated Sandbox Protocol connections. + #[arg(long)] + runtime_listen: SocketAddr, + + /// Private directory for per-session files. + #[arg(long)] + session_root: PathBuf, -// The gateway endpoint is reserved for the shared supervisor's outbound session. -// This plumbing stage only starts the local listener. -pub fn run(address: SocketAddr, _gateway_endpoint: &str) -> Result<()> { - run_platform(address) + #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] + log_level: String, +} + +pub fn run(raw_args: &[String]) -> Result<()> { + let args = Args::parse_from(raw_args); + run_platform(args) } #[cfg(unix)] -fn run_platform(address: SocketAddr) -> Result<()> { - let runtime = tokio::runtime::Builder::new_current_thread() +fn run_platform(args: Args) -> Result<()> { + use std::sync::Arc; + use std::sync::atomic::AtomicBool; + use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _}; + + let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .into_diagnostic()?; runtime.block_on(async { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let console_filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(&args.log_level)); + // Process-wide OCSF JSON output stays off; each session's own + // settings apply to the events its threads emit. + let (logging_layers, _logging_guards, _) = crate::logging::layers( + std::io::stderr(), + std::path::Path::new("/var/log"), + console_filter, + Arc::new(AtomicBool::new(false)), + Arc::new(std::sync::Mutex::new(String::new())), + ); + tracing_subscriber::registry() + .with(logging_layers) + .with(openshell_supervisor_process::log_push::LogPushLayer::thread_bound()) + .with( + openshell_ocsf::OcsfShorthandLayer::new( + openshell_supervisor::session_host::SessionDenialWriter, + ) + .with_non_ocsf(false) + .with_filter(tracing_subscriber::filter::LevelFilter::INFO), + ) + .init(); + // Install shutdown handlers before publishing health. let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) @@ -28,31 +91,75 @@ fn run_platform(address: SocketAddr) -> Result<()> { let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()) .into_diagnostic()?; - let listener = tokio::net::TcpListener::bind(address) + let tls_file = |name: &str| { + let path = args.session_control_tls_dir.join(name); + std::fs::read(&path) + .into_diagnostic() + .wrap_err_with(|| format!("read session-control TLS file {}", path.display())) + }; + let control_tls = openshell_sandbox_backend::session_control::server_tls_config( + &tls_file("tls.crt")?, + &tls_file("tls.key")?, + &tls_file("ca.crt")?, + ) + .map_err(|error| miette::miette!("{error}"))?; + let host = openshell_supervisor::shared_host::SharedHost::start( + openshell_supervisor::shared_host::SharedHostConfig { + gateway_endpoint: args.openshell_endpoint.clone(), + session_root: args.session_root.clone(), + runtime_listen: args.runtime_listen, + removal_grace: SESSION_REMOVAL_GRACE, + }, + )?; + let control = tokio::net::TcpListener::bind(args.session_control_listen) .await - .into_diagnostic()?; - eprintln!( - "Shared supervisor started; gateway connection and sandbox sessions are not implemented" + .into_diagnostic() + .wrap_err("bind session-control listener")?; + + // The private emptyDir survives a container restart after an unclean exit. + match std::fs::remove_file(&args.health_socket) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error).into_diagnostic(), + } + let health = tokio::net::UnixListener::bind(&args.health_socket).into_diagnostic()?; + tracing::info!( + runtime = %host.runtime_addr(), + control = %args.session_control_listen, + "Shared supervisor ready for sessions" ); - loop { + let (stop, stopped) = tokio::sync::watch::channel(false); + let serve = tokio::spawn(openshell_supervisor::shared_host::serve_session_control( + host.clone(), + control, + control_tls, + async move { + let mut stopped = stopped; + let _ = stopped.wait_for(|stop| *stop).await; + }, + )); + let result = loop { tokio::select! { - // This placeholder only accepts and closes connections, including - // health probes. Session control will use an outbound gateway stream. - result = listener.accept() => { + result = health.accept() => { if let Err(error) = result { - eprintln!("shared supervisor listener accept failed: {error}"); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + break Err(error).into_diagnostic(); } } - _ = terminate.recv() => break, - _ = interrupt.recv() => break, + _ = terminate.recv() => break Ok(()), + _ = interrupt.recv() => break Ok(()), } - } - Ok(()) + }; + drop(health); + let _ = std::fs::remove_file(&args.health_socket); + stop.send_replace(true); + let _ = serve.await; + // Sandboxes lose supervision with this process and fail closed. + host.remove_all().await; + result }) } #[cfg(not(unix))] -fn run_platform(_address: SocketAddr) -> Result<()> { - Err(miette::miette!("--mode=shared requires Unix")) +fn run_platform(_args: Args) -> Result<()> { + Err(miette::miette!("experimental-host requires Unix")) } diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs new file mode 100644 index 0000000000..536e557be9 --- /dev/null +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -0,0 +1,681 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor shared by the sandboxes of one gateway. +//! +//! A compute driver registers each runtime generation over the +//! session-control API, which accepts only mutually authenticated TLS clients +//! holding the admission certificate. A registration carries exactly the +//! gateway-issued credentials of that generation; the host never acts with +//! any other identity. Each registration becomes a [`SessionHost`] session +//! whose runtime connections arrive through one routed listener. Removing a +//! registration stops only that session. + +use std::collections::HashMap; +use std::future::Future; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use bytes::Bytes; +use http_body_util::{BodyExt as _, Full, Limited}; +use miette::{IntoDiagnostic, Result, WrapErr}; +use openshell_sandbox_backend::runtime_routed::RuntimeRouter; +use openshell_sandbox_backend::session_control::{ + MAX_REGISTRATION_BYTES, SESSIONS_PATH, SessionList, SessionPhaseReport, SessionRegistration, + SessionStatusReport, +}; +use sha2::{Digest as _, Sha256}; +use tokio::sync::watch; + +use crate::session_host::{HostedSessionSpec, SessionHost, SessionPhase, SessionStatus}; + +/// Concurrent session-control connections; the only client is the gateway, +/// whose removals hold a connection while a workload stops. +const MAX_CONTROL_CONNECTIONS: usize = 64; +/// Bound on a TLS handshake and on reading one request. +const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(10); + +/// Process-level inputs of a shared supervisor. +pub struct SharedHostConfig { + /// Gateway gRPC endpoint every session uses with its own credentials. + pub gateway_endpoint: String, + /// Private directory for per-session files. + pub session_root: PathBuf, + /// Listener for runtime-initiated Sandbox Protocol connections. + pub runtime_listen: SocketAddr, + /// Time a removed session may take to stop its workload. + pub removal_grace: Duration, +} + +struct Registered { + /// Distinguishes this registration from a later one of the same route. + serial: u64, + digest: [u8; 32], + sandbox_id: String, + /// Final status of the removal in progress, shared by every remover. + removal: Option>>, +} + +type Registrations = Mutex>; + +fn lock(registrations: &Registrations) -> MutexGuard<'_, HashMap> { + registrations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// Registered sessions, their runtime router, and their private files. +pub struct SharedHost { + host: SessionHost, + router: Arc, + registrations: Registrations, + next_serial: AtomicU64, + config: SharedHostConfig, +} + +/// Outcome of a registration request. +#[derive(Debug, PartialEq, Eq)] +pub enum RegisterOutcome { + Created, + /// The identical registration already exists. + Unchanged, +} + +/// A rejected session-control request. +#[derive(Debug, thiserror::Error)] +pub enum ControlError { + #[error("{0}")] + Invalid(String), + #[error("route is already registered with different launch material")] + Conflict, + #[error("route is still being removed")] + Removing, + #[error("{0}")] + Unavailable(String), +} + +impl SharedHost { + /// Bind the routed runtime listener and prepare the session root. + pub fn start(config: SharedHostConfig) -> Result> { + prepare_private_dir(&config.session_root)?; + let router = RuntimeRouter::bind(config.runtime_listen) + .map_err(|error| miette::miette!("{error}"))?; + Ok(Arc::new(Self { + host: SessionHost::new(), + router, + registrations: Mutex::default(), + next_serial: AtomicU64::new(0), + config, + })) + } + + /// Address of the routed runtime listener. + #[must_use] + pub fn runtime_addr(&self) -> SocketAddr { + self.router.local_addr() + } + + /// Start a session for `registration`, or confirm an identical one. + pub fn register( + &self, + registration: SessionRegistration, + ) -> Result { + registration + .validate() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let digest = registration_digest(®istration)?; + let route = registration.route.clone(); + let sandbox_id = registration.sandbox_id.clone(); + self.admit(&route, digest, sandbox_id, |session_dir| { + let spec = self.session_spec(registration, session_dir)?; + self.host + .start_supervisor_session(spec) + .map_err(|error| ControlError::Unavailable(error.to_string())) + }) + } + + /// Reserve `route` and run `start` with the session's private directory. + fn admit( + &self, + route: &str, + digest: [u8; 32], + sandbox_id: String, + start: impl FnOnce(&Path) -> Result<(), ControlError>, + ) -> Result { + let mut registrations = lock(&self.registrations); + if let Some(existing) = registrations.get(route) { + return if existing.removal.is_some() { + Err(ControlError::Removing) + } else if existing.digest == digest { + Ok(RegisterOutcome::Unchanged) + } else { + Err(ControlError::Conflict) + }; + } + let session_dir = self.config.session_root.join(route); + if let Err(error) = start(&session_dir) { + let _ = std::fs::remove_dir_all(&session_dir); + return Err(error); + } + registrations.insert( + route.to_string(), + Registered { + serial: self.next_serial.fetch_add(1, Ordering::Relaxed), + digest, + sandbox_id, + removal: None, + }, + ); + Ok(RegisterOutcome::Created) + } + + /// Status of one registered session. + #[must_use] + pub fn status(&self, route: &str) -> Option { + let sandbox_id = lock(&self.registrations).get(route)?.sandbox_id.clone(); + let status = self.host.status(route)?; + Some(status_report(route, sandbox_id, &status)) + } + + /// Status of every registered session. + #[must_use] + pub fn list(&self) -> SessionList { + let routes = lock(&self.registrations) + .keys() + .cloned() + .collect::>(); + SessionList { + sessions: routes + .iter() + .filter_map(|route| self.status(route)) + .collect(), + } + } + + /// Stop one session, delete its private files, and forget it. + /// + /// Removal runs on a task the host owns, so it completes even when the + /// caller disconnects, and concurrent callers share its result. The + /// route stays reserved until the session has ended and its files are + /// gone, so a new registration of the route can never meet them. + pub async fn remove(self: &Arc, route: &str) -> Option { + let mut removed = self.begin_removal(route)?; + removed + .wait_for(Option::is_some) + .await + .ok() + .and_then(|report| report.clone()) + } + + fn begin_removal( + self: &Arc, + route: &str, + ) -> Option>> { + let mut registrations = lock(&self.registrations); + let registered = registrations.get_mut(route)?; + if let Some(removed) = ®istered.removal { + return Some(removed.clone()); + } + let (report, removed) = watch::channel(None); + registered.removal = Some(removed.clone()); + let serial = registered.serial; + let sandbox_id = registered.sandbox_id.clone(); + let shared = Arc::clone(self); + let route = route.to_string(); + tokio::spawn(async move { + let status = shared + .host + .remove(&route, shared.config.removal_grace) + .await + .unwrap_or_else(|| SessionStatus { + phase: SessionPhase::Failed("session was not running".to_string()), + ready: false, + }); + let session_dir = shared.config.session_root.join(&route); + let _ = + tokio::task::spawn_blocking(move || match std::fs::remove_dir_all(&session_dir) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + tracing::warn!(%error, "could not delete removed session files"); + } + _ => {} + }) + .await; + { + let mut registrations = lock(&shared.registrations); + if registrations + .get(&route) + .is_some_and(|registered| registered.serial == serial) + { + registrations.remove(&route); + } + } + report.send_replace(Some(status_report(&route, sandbox_id, &status))); + }); + Some(removed) + } + + /// Stop every session, for process shutdown. + pub async fn remove_all(self: &Arc) { + let routes = lock(&self.registrations) + .keys() + .cloned() + .collect::>(); + futures::future::join_all(routes.iter().map(|route| self.remove(route))).await; + } + + fn session_spec( + &self, + registration: SessionRegistration, + session_dir: &Path, + ) -> Result { + let failure = |error: miette::Report| ControlError::Unavailable(error.to_string()); + let main_process = + openshell_core::sandbox_env::MainProcessConfig::decode(®istration.main_process_spec) + .map_err(ControlError::Invalid)?; + let descriptor = registration + .runtime_descriptor + .backend_descriptor() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + if session_dir.exists() { + std::fs::remove_dir_all(session_dir) + .into_diagnostic() + .map_err(failure)?; + } + prepare_private_dir(session_dir).map_err(failure)?; + let ca_certificate = session_dir.join("proxy-ca.crt"); + let ca_private_key = session_dir.join("proxy-ca.key"); + write_private_file( + &ca_certificate, + registration.proxy_ca.certificate_pem.as_bytes(), + ) + .map_err(failure)?; + write_private_file( + &ca_private_key, + registration + .proxy_ca + .private_key_pem + .expose_secret() + .as_bytes(), + ) + .map_err(failure)?; + let trust_dir = session_dir.join("trust"); + prepare_private_dir(&trust_dir).map_err(failure)?; + let ssh_socket = session_dir.join("ssh.sock"); + Ok(HostedSessionSpec { + key: registration.route.clone(), + backend_setup: Box::new(crate::backend_setup::RoutedBackendSetup::new( + self.router.clone(), + registration.route, + )), + config: crate::SessionConfig { + command: main_process.command, + workdir: registration.workdir, + timeout_secs: 0, + interactive: main_process.tty, + await_main_process_attachment: main_process.await_main_process_attachment, + sandbox_id: Some(registration.sandbox_id), + sandbox: Some(registration.sandbox_name), + openshell_endpoint: Some(self.config.gateway_endpoint.clone()), + policy_rules: None, + policy_data: None, + ssh_socket_path: Some(ssh_socket.to_string_lossy().into_owned()), + shared_ssh_socket: false, + ocsf_json: openshell_ocsf::OcsfJsonSettings::default(), + // The Kubernetes driver refuses the gateway topology when an + // upstream proxy or provider workload identity is configured, + // and it configures no extra CA for dedicated supervisors. + upstream_proxy_args: + openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: descriptor, + auth_bundle: registration.auth, + gateway_credentials: None, + admitted_isolation_backend: Some( + openshell_sandbox_backend::BACKEND_NAME.to_string(), + ), + main_exit_marker: None, + trace_context: registration + .trace_context + .map(|context| crate::TraceContext { + traceparent: context.traceparent, + tracestate: context.tracestate, + }), + // One process cannot hold a per-sandbox workload identity, so + // providers that need one fail closed. + provider_workload_api_socket: None, + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles { + ca_certificate: Some(ca_certificate), + ca_private_key: Some(ca_private_key), + directory: Some(trust_dir), + }, + denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), + }, + container_image: registration.container_image, + }) + } +} + +fn registration_digest(registration: &SessionRegistration) -> Result<[u8; 32], ControlError> { + let encoded = serde_json::to_vec(registration) + .map_err(|error| ControlError::Invalid(format!("encode registration: {error}")))?; + Ok(Sha256::digest(&encoded).into()) +} + +fn status_report(route: &str, sandbox_id: String, status: &SessionStatus) -> SessionStatusReport { + SessionStatusReport { + route: route.to_string(), + sandbox_id, + phase: match &status.phase { + SessionPhase::Starting => SessionPhaseReport::Starting, + SessionPhase::Supervising => SessionPhaseReport::Supervising, + SessionPhase::Stopping => SessionPhaseReport::Stopping, + SessionPhase::Exited(code) => SessionPhaseReport::Exited { exit_code: *code }, + SessionPhase::Failed(error) => SessionPhaseReport::Failed { + error: error.clone(), + }, + }, + ready: status.ready, + } +} + +#[cfg(unix)] +fn prepare_private_dir(path: &Path) -> Result<()> { + use std::os::unix::fs::{DirBuilderExt as _, PermissionsExt as _}; + + match std::fs::DirBuilder::new().mode(0o700).create(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let metadata = std::fs::symlink_metadata(path).into_diagnostic()?; + if !metadata.is_dir() { + return Err(miette::miette!( + "session path is not a directory: {}", + path.display() + )); + } + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) + .into_diagnostic()?; + } + Err(error) => { + return Err(error) + .into_diagnostic() + .wrap_err_with(|| format!("create private directory {}", path.display())); + } + } + Ok(()) +} + +#[cfg(not(unix))] +fn prepare_private_dir(_path: &Path) -> Result<()> { + Err(miette::miette!("shared supervision requires a Unix host")) +} + +#[cfg(unix)] +fn write_private_file(path: &Path, contents: &[u8]) -> Result<()> { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .into_diagnostic() + .wrap_err_with(|| format!("create {}", path.display()))?; + file.write_all(contents).into_diagnostic()?; + file.sync_all().into_diagnostic() +} + +#[cfg(not(unix))] +fn write_private_file(_path: &Path, _contents: &[u8]) -> Result<()> { + Err(miette::miette!("shared supervision requires a Unix host")) +} + +type ControlResponse = http::Response>; + +fn json_response(status: http::StatusCode, body: &impl serde::Serialize) -> ControlResponse { + let body = serde_json::to_vec(body).unwrap_or_default(); + http::Response::builder() + .status(status) + .header(http::header::CONTENT_TYPE, "application/json") + .body(Full::new(Bytes::from(body))) + .unwrap_or_default() +} + +fn error_response(status: http::StatusCode, message: &str) -> ControlResponse { + json_response(status, &serde_json::json!({ "error": message })) +} + +async fn handle_control( + host: Arc, + request: http::Request, +) -> ControlResponse { + let path = request.uri().path().to_string(); + let route = path + .strip_prefix(SESSIONS_PATH) + .and_then(|rest| rest.strip_prefix('/')) + .filter(|route| !route.is_empty() && !route.contains('/')); + match (request.method().clone(), route) { + (http::Method::GET, None) if path == SESSIONS_PATH => { + json_response(http::StatusCode::OK, &host.list()) + } + (http::Method::GET, Some(route)) => host.status(route).map_or_else( + || error_response(http::StatusCode::NOT_FOUND, "session is not registered"), + |status| json_response(http::StatusCode::OK, &status), + ), + (http::Method::PUT, Some(route)) => { + let route = route.to_string(); + let body = match tokio::time::timeout( + CONTROL_IO_TIMEOUT, + Limited::new(request.into_body(), MAX_REGISTRATION_BYTES).collect(), + ) + .await + { + Ok(Ok(body)) => body.to_bytes(), + _ => { + return error_response( + http::StatusCode::PAYLOAD_TOO_LARGE, + "registration body is invalid or too large", + ); + } + }; + let registration = match serde_json::from_slice::(&body) { + Ok(registration) if registration.route == route => registration, + Ok(_) => { + return error_response( + http::StatusCode::BAD_REQUEST, + "registration route does not match the request path", + ); + } + // Decoder errors can quote field values, including credentials. + Err(_) => { + return error_response( + http::StatusCode::BAD_REQUEST, + "registration body is not a valid session registration", + ); + } + }; + match host.register(registration) { + Ok(outcome) => { + let status = if outcome == RegisterOutcome::Created { + http::StatusCode::CREATED + } else { + http::StatusCode::OK + }; + host.status(&route).map_or_else( + || error_response(http::StatusCode::CONFLICT, "session was removed"), + |report| json_response(status, &report), + ) + } + Err(error @ ControlError::Invalid(_)) => { + error_response(http::StatusCode::BAD_REQUEST, &error.to_string()) + } + Err(error @ (ControlError::Conflict | ControlError::Removing)) => { + error_response(http::StatusCode::CONFLICT, &error.to_string()) + } + Err(error @ ControlError::Unavailable(_)) => { + error_response(http::StatusCode::SERVICE_UNAVAILABLE, &error.to_string()) + } + } + } + (http::Method::DELETE, Some(route)) => host.remove(route).await.map_or_else( + || error_response(http::StatusCode::NOT_FOUND, "session is not registered"), + |status| json_response(http::StatusCode::OK, &status), + ), + _ => error_response(http::StatusCode::NOT_FOUND, "unknown session-control route"), + } +} + +/// Serve the session-control API until `shutdown` completes. Every +/// connection must complete mutually authenticated TLS before any request. +pub async fn serve_session_control( + host: Arc, + listener: tokio::net::TcpListener, + tls: Arc, + shutdown: impl Future, +) { + let acceptor = tokio_rustls::TlsAcceptor::from(tls); + let slots = Arc::new(tokio::sync::Semaphore::new(MAX_CONTROL_CONNECTIONS)); + let mut connections = tokio::task::JoinSet::new(); + tokio::pin!(shutdown); + loop { + tokio::select! { + () = &mut shutdown => break, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + }; + let Ok(slot) = slots.clone().try_acquire_owned() else { + continue; + }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + let acceptor = acceptor.clone(); + let host = host.clone(); + connections.spawn(async move { + let _slot = slot; + let Ok(Ok(stream)) = + tokio::time::timeout(CONTROL_IO_TIMEOUT, acceptor.accept(stream)).await + else { + tracing::debug!("rejected session-control connection without admission TLS"); + return; + }; + let service = hyper::service::service_fn(move |request| { + let host = host.clone(); + async move { + Ok::<_, std::convert::Infallible>(handle_control(host, request).await) + } + }); + let _ = hyper::server::conn::http1::Builder::new() + .timer(hyper_util::rt::TokioTimer::new()) + .header_read_timeout(CONTROL_IO_TIMEOUT) + .serve_connection(hyper_util::rt::TokioIo::new(stream), service) + .await; + }); + } + Some(_) = connections.join_next(), if !connections.is_empty() => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::session_host::{SessionBinding, SessionBody}; + + fn shared_host(root: &Path) -> Arc { + SharedHost::start(SharedHostConfig { + gateway_endpoint: "http://127.0.0.1:9".to_string(), + session_root: root.to_path_buf(), + runtime_listen: "127.0.0.1:0".parse().unwrap(), + removal_grace: Duration::from_secs(10), + }) + .unwrap() + } + + /// Admit a session that keeps a private file and takes a while to stop. + fn admit(host: &SharedHost, route: &str, launch: u8) -> Result { + host.admit(route, [launch; 32], format!("sandbox-{launch}"), |dir| { + let unavailable = |error: miette::Report| ControlError::Unavailable(error.to_string()); + prepare_private_dir(dir).map_err(unavailable)?; + write_private_file(&dir.join("launch"), &[launch]).map_err(unavailable)?; + let body: SessionBody = Box::new(|controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + controls.shutdown.await; + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(0) + }) + }); + let binding = SessionBinding { + event_context: Arc::new(crate::session_event_context(None, None, String::new())), + ocsf_json: openshell_ocsf::OcsfJsonSettings::default(), + log_sink: None, + denial_log: openshell_supervisor_network::policy_local::SessionDenialLog::default(), + }; + host.host + .start(route, binding, None, body) + .map_err(|error| ControlError::Unavailable(error.to_string())) + }) + } + + async fn wait_ready(host: &SharedHost, route: &str) { + tokio::time::timeout(Duration::from_secs(10), async { + while !host.status(route).is_some_and(|status| status.ready) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("session becomes ready"); + } + + #[tokio::test(flavor = "multi_thread")] + async fn a_route_stays_reserved_until_its_single_removal_finishes() { + let root = tempfile::tempdir().unwrap(); + let host = shared_host(root.path()); + let route = "route-a"; + assert_eq!(admit(&host, route, 1).unwrap(), RegisterOutcome::Created); + assert_eq!(admit(&host, route, 1).unwrap(), RegisterOutcome::Unchanged); + assert!(matches!( + admit(&host, route, 2), + Err(ControlError::Conflict) + )); + wait_ready(&host, route).await; + + // A disconnected caller still removes the session. + assert!( + tokio::time::timeout(Duration::from_millis(20), host.remove(route)) + .await + .is_err() + ); + assert!(matches!( + admit(&host, route, 1), + Err(ControlError::Removing) + )); + let removers = [(); 2].map(|()| { + let host = host.clone(); + tokio::spawn(async move { host.remove(route).await }) + }); + for remover in removers { + let report = remover.await.unwrap().expect("route was registered"); + assert_eq!(report.sandbox_id, "sandbox-1"); + assert_eq!(report.phase, SessionPhaseReport::Exited { exit_code: 0 }); + } + assert!(host.status(route).is_none()); + assert!(!root.path().join(route).exists()); + + // Nothing from the removed session reaches the route's next owner. + assert_eq!(admit(&host, route, 3).unwrap(), RegisterOutcome::Created); + wait_ready(&host, route).await; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + std::fs::read(root.path().join(route).join("launch")).unwrap(), + [3] + ); + assert_eq!(host.status(route).unwrap().sandbox_id, "sandbox-3"); + host.remove_all().await; + assert!(host.list().sessions.is_empty()); + } +} From 1e096ebedeef96623af2386d5bbdb8f1cc505c78 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 18:27:11 -0700 Subject: [PATCH 03/25] feat(server): issue a TLS-only private PKI from generate-certs generate-certs --tls-only creates just a server and a client TLS Secret from a fresh CA, with the same create-if-missing contract and a recovery hint for partial state. The Helm certgen hook uses it for private PKIs that must not carry a sandbox-JWT signing key. Signed-off-by: Drew Newberry --- crates/openshell-server/src/certgen.rs | 77 +++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/crates/openshell-server/src/certgen.rs b/crates/openshell-server/src/certgen.rs index 764ceba98c..537b15aef1 100644 --- a/crates/openshell-server/src/certgen.rs +++ b/crates/openshell-server/src/certgen.rs @@ -12,6 +12,9 @@ //! - **Kubernetes JWT-only mode** (`--jwt-only`): create only the //! sandbox-JWT signing Secret. Used when another controller, such as //! cert-manager, owns the TLS Secrets. +//! - **Kubernetes TLS-only mode** (`--tls-only`): create only the server and +//! client TLS Secrets of a private PKI with no sandbox-JWT signing key, +//! such as the shared supervisor's session-control PKI. //! - **Local mode** (`--output-dir `): write PEMs to the local package //! filesystem layout. Used by systemd units' `ExecStartPre`. Also copies //! client materials to @@ -38,6 +41,7 @@ use tracing::{info, warn}; use tracing_subscriber::EnvFilter; #[derive(Args, Debug)] +#[allow(clippy::struct_excessive_bools)] pub struct CertgenArgs { /// Write PEMs to a filesystem directory instead of Kubernetes Secrets. /// When set, the kube-related flags are not required. @@ -61,7 +65,7 @@ pub struct CertgenArgs { /// Name of the sandbox-JWT signing-key Secret (`Opaque`) to create. /// Holds `signing.pem`, `public.pem`, and `kid` keys. Mounted on the /// gateway pod (only) so it can mint and validate per-sandbox JWTs. - #[arg(long, required_unless_present = "output_dir")] + #[arg(long, required_unless_present_any = ["output_dir", "tls_only"])] jwt_secret_name: Option, /// Create only the sandbox-JWT signing-key Secret in Kubernetes mode. @@ -69,6 +73,14 @@ pub struct CertgenArgs { #[arg(long, conflicts_with = "output_dir")] jwt_only: bool, + /// Create only the server and client TLS Secrets in Kubernetes mode, for + /// a private PKI that issues no sandbox-JWT signing key. + #[arg( + long, + conflicts_with_all = ["output_dir", "jwt_only", "jwt_secret_name", "backend_ca_configmap_name"] + )] + tls_only: bool, + /// Extra Subject Alternative Name for the server certificate. Repeatable. /// Auto-detected as an IP address or DNS name. #[arg(long = "server-san", value_name = "SAN")] @@ -224,6 +236,34 @@ async fn run_kubernetes(args: &CertgenArgs, bundle: &PkiBundle) -> Result<()> { .wrap_err_with(|| format!("failed to read secret {client_name}"))? .is_some(); + if args.tls_only { + match (server_exists, client_exists) { + (true, true) => info!( + namespace = %namespace, + server = %server_name, + client = %client_name, + "TLS secrets already exist, skipping." + ), + (false, false) => { + create_tls_secrets(&api, server_name, client_name, bundle).await?; + info!( + namespace = %namespace, + server = %server_name, + client = %client_name, + "TLS secrets created." + ); + } + _ => { + return Err(miette::miette!( + "partial PKI state in namespace {namespace}: only one of \ + {server_name} / {client_name} exists. Recover with: \ + kubectl delete secret -n {namespace} {server_name} {client_name}", + )); + } + } + return Ok(()); + } + let jwt_name = args .jwt_secret_name .as_deref() @@ -986,6 +1026,41 @@ mod tests { use openshell_bootstrap::pki::generate_pki; use std::path::Path; + #[derive(clap::Parser)] + struct Cli { + #[command(flatten)] + args: super::CertgenArgs, + } + + #[test] + fn tls_only_mode_needs_no_jwt_secret_and_refuses_one() { + use clap::Parser as _; + let tls_only = [ + "generate-certs", + "--namespace=gateway", + "--tls-only", + "--server-secret-name=server", + "--client-secret-name=client", + ]; + assert!(Cli::try_parse_from(tls_only).unwrap().args.tls_only); + for conflicting in [ + "--jwt-secret-name=jwt", + "--jwt-only", + "--output-dir=/tmp/pki", + ] { + let mut args = tls_only.to_vec(); + args.push(conflicting); + assert!(Cli::try_parse_from(args).is_err(), "{conflicting}"); + } + let missing_client = [ + "generate-certs", + "--namespace=gateway", + "--tls-only", + "--server-secret-name=server", + ]; + assert!(Cli::try_parse_from(missing_client).is_err()); + } + // ── Kubernetes-mode decision ── #[test] From 217d598798bb2d6ea318a8a114727d0b977c395a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 18:27:55 -0700 Subject: [PATCH 04/25] feat(kubernetes): supervise gateway-topology generations as shared sessions With experimental_supervisor_topology = "gateway", each new runtime generation becomes a session on the gateway's shared supervisor instead of a supervisor Pod and boundary Service of its own. The generation records its placement and session route on its Sandbox resource when it starts, and every later decision reads that record, so changing the setting never reinterprets a running generation and unknown records fail closed. The runtime identity ends with the session route, which no Pod UID can equal, so Pod authentication cannot vouch for a shared generation. Create and start add a per-sandbox egress allowance to the Pods the shared supervisor's Service selects, bind it into the boundary evidence, and register the session over session control before releasing the workload; registration does not wait for the runtime, so create still returns before the gateway persists the generation. Readiness follows the session's own status. Stop, restart, and delete remove only that session, and reconciliation suspends a workload whose session is gone and removes sessions that no current generation may run. Configurations with an upstream proxy or provider SPIFFE identity are refused. Closes #4700 Signed-off-by: Drew Newberry --- Cargo.lock | 2 + crates/openshell-driver-kubernetes/Cargo.toml | 10 +- crates/openshell-driver-kubernetes/README.md | 40 + .../openshell-driver-kubernetes/src/config.rs | 166 ++- .../openshell-driver-kubernetes/src/driver.rs | 1233 ++++++++++++++--- .../src/isolation.rs | 129 +- crates/openshell-driver-kubernetes/src/lib.rs | 5 +- .../src/lifecycle_tests.rs | 1 + .../openshell-driver-kubernetes/src/main.rs | 1 + .../src/sandbox_runtime.rs | 27 + .../src/shared_supervisor.rs | 296 ++++ 11 files changed, 1669 insertions(+), 241 deletions(-) create mode 100644 crates/openshell-driver-kubernetes/src/shared_supervisor.rs diff --git a/Cargo.lock b/Cargo.lock index 42b642600d..3bb245b31b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4513,11 +4513,13 @@ dependencies = [ "prost", "prost-types", "rcgen", + "rustls", "serde", "serde_json", "temp-env", "thiserror 2.0.20", "tokio", + "tokio-rustls", "tokio-stream", "toml", "tonic", diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 9da035039c..29eb9f98ae 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -42,15 +42,17 @@ thiserror = { workspace = true } miette = { workspace = true } notify = "8" rcgen = { workspace = true } - -[dev-dependencies] -openshell-otel-test-support = { path = "../openshell-otel-test-support" } -opentelemetry_sdk = { workspace = true, features = ["testing"] } bytes = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } +rustls = { workspace = true } +tokio-rustls = { workspace = true } + +[dev-dependencies] +openshell-otel-test-support = { path = "../openshell-otel-test-support" } +opentelemetry_sdk = { workspace = true, features = ["testing"] } temp-env = "0.3" tokio = { workspace = true, features = ["test-util"] } toml = { workspace = true } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 9a9d6dca18..aba9517d8d 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -179,6 +179,46 @@ must not create permissive policies, create Pods, read bootstrap Secrets, or spoof the OpenShell role labels. Exact supervisor-to-sandbox authorization is still enforced by TLS, JWT claims, session generation, and recorded Pod UIDs. +### Shared supervisor topology + +With `supervisor_topology = "gateway"`, each new runtime +generation becomes a session on one supervisor process that Helm deploys in the +gateway namespace, instead of a supervisor Pod and boundary Service of its own. +The generation's placement is recorded on the Sandbox resource +(`openshell.ai/sandbox-runtime-supervisor-topology: gateway` and the session +route in `openshell.ai/sandbox-runtime-supervisor-session`) when the generation +starts. Every later lifecycle decision reads the record, so changing the setting +never reinterprets a running generation, and an unknown or incomplete record +fails closed. + +- **Create and start.** The driver creates a per-sandbox egress allowance + `os-shared-control-` from the workload Pod to the Pods the shared + supervisor's Service selects, on the runtime port, binds its UID and + resourceVersion into the boundary evidence, and registers the session over + session control before releasing the workload Pod. Registration does not wait + for the runtime, so create still returns before the gateway persists the + generation. The runtime identity's last component is the session route, which + no Pod UID can equal, so `authenticate_sandbox` cannot vouch for a shared + generation. +- **Readiness.** A shared generation is available while its session is + registered for the sandbox and accepted by the gateway and its fence and + egress allowance exist. A missing or ended session is unavailable, and + reconciliation suspends the workload; an unreachable supervisor is unknown. +- **Stop and delete.** The driver removes the session before suspending or + deleting the workload. Removal is best effort because ending the workload ends + supervision anyway. Reconciliation removes sessions that no Sandbox's current + generation may run; it lists sessions before Sandboxes, because a generation + records its route before registering it. +- **Isolation.** The admission credential is separate from every session's + credentials. Sandboxes cannot reach session control, and the routing preface + only selects which generation-pinned TLS identity the supervisor demands. + +The topology does not support upstream proxies or provider SPIFFE identity, and +the shared supervisor is a single point of failure for the sandboxes it hosts. +When the setting is removed, reconciliation suspends shared generations, and a +restart creates the dedicated generation's boundary Service the shared one never +had. + ## Sandbox Resource The driver works with the `agents.x-k8s.io` `Sandbox` custom resource. It diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index ccf130fab6..aaf2dd8195 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -202,12 +202,72 @@ pub enum SupervisorTopology { Workspace, } +/// Connection to the supervisor Helm deploys for the gateway topology. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SharedSupervisorConfig { + /// Service selecting the shared supervisor Pod. + pub service_name: String, + pub service_namespace: String, + /// Service port for runtime-initiated Sandbox Protocol connections. + pub runtime_port: u16, + /// Service port for the mutually authenticated session-control API. + pub control_port: u16, + /// Directory with the session-control client `tls.crt` and `tls.key` and + /// the CA `ca.crt` that issued the supervisor's certificate. + pub control_tls_dir: String, +} + +impl SharedSupervisorConfig { + fn validate(&self) -> Result<(), String> { + if !is_dns_label(&self.service_name) || !is_dns_label(&self.service_namespace) { + return Err( + "shared_supervisor service_name and service_namespace must be DNS labels".into(), + ); + } + if self.runtime_port == 0 + || self.control_port == 0 + || self.runtime_port == self.control_port + { + return Err( + "shared_supervisor runtime_port and control_port must be distinct and nonzero" + .into(), + ); + } + if !Path::new(&self.control_tls_dir).is_absolute() { + return Err("shared_supervisor control_tls_dir must be absolute".into()); + } + Ok(()) + } + + /// DNS name in the supervisor's session-control certificate. + #[must_use] + pub fn service_host(&self) -> String { + format!("{}.{}.svc", self.service_name, self.service_namespace) + } +} + +fn is_dns_label(value: &str) -> bool { + !value.is_empty() + && value.len() <= 63 + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && !value.starts_with('-') + && !value.ends_with('-') +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesComputeConfig { - /// Deploy an idle shared supervisor through Helm. Does not route sessions. + /// Supervise new sandbox generations with one supervisor shared by the + /// gateway instead of a supervisor Pod per sandbox. Existing generations + /// keep the topology recorded when they started. #[serde(skip_serializing_if = "Option::is_none")] pub supervisor_topology: Option, + /// Shared supervisor connection rendered by Helm with the gateway topology. + #[serde(skip_serializing_if = "Option::is_none")] + pub shared_supervisor: Option, /// Permit caller-supplied driver JSON. Does not waive resource admission. pub allow_driver_config: bool, /// Operator-owned external attachment approval policy. @@ -401,6 +461,7 @@ impl Default for KubernetesComputeConfig { fn default() -> Self { Self { supervisor_topology: None, + shared_supervisor: None, workspace_mode: WorkspaceMode::default(), allow_driver_config: false, resource_admission: @@ -454,9 +515,7 @@ impl Default for KubernetesComputeConfig { impl KubernetesComputeConfig { /// Validate startup configuration without connecting to Kubernetes. pub fn validate_configuration(&self) -> Result<(), String> { - if self.supervisor_topology == Some(SupervisorTopology::Workspace) { - return Err("supervisor_topology=workspace is not implemented".into()); - } + self.validate_supervisor_topology()?; self.validate_workspace_mode()?; self.validate_provider_spiffe_workload_api_socket_path()?; self.validate_sandbox_identity_config()?; @@ -466,6 +525,41 @@ impl KubernetesComputeConfig { self.validate_allowed_pod_labels() } + /// The shared supervisor serves only sessions it can isolate: it holds no + /// per-sandbox workload identity, upstream proxy credential, or proxy CA. + pub fn validate_supervisor_topology(&self) -> Result<(), String> { + match self.supervisor_topology { + None => { + if self.shared_supervisor.is_some() { + return Err("shared_supervisor requires supervisor_topology=gateway".into()); + } + Ok(()) + } + Some(SupervisorTopology::Workspace) => { + Err("supervisor_topology=workspace is not implemented".into()) + } + Some(SupervisorTopology::Gateway) => { + let shared = self.shared_supervisor.as_ref().ok_or_else(|| { + "supervisor_topology=gateway requires shared_supervisor".to_string() + })?; + shared.validate()?; + if self.provider_spiffe_enabled() { + return Err("supervisor_topology=gateway does not support provider_spiffe_workload_api_socket_path; a shared supervisor has no per-sandbox workload identity".into()); + } + if self.https_proxy.is_some() + || self.proxy_auth_secret_name.is_some() + || self.proxy_ca_bundle.is_some() + { + return Err( + "supervisor_topology=gateway does not support upstream proxy settings" + .into(), + ); + } + Ok(()) + } + } + } + pub fn validate_allowed_pod_annotations(&self) -> Result<(), String> { validate_pod_metadata_allowlist("allowed_pod_annotations", &self.allowed_pod_annotations) } @@ -1010,6 +1104,70 @@ mod tests { use super::*; use std::collections::BTreeMap as HashMap; + #[test] + fn gateway_supervisor_topology_requires_its_connection_and_refuses_unsupported_inputs() { + let shared = || SharedSupervisorConfig { + service_name: "openshell-supervisor".to_string(), + service_namespace: "openshell".to_string(), + runtime_port: 8443, + control_port: 9443, + control_tls_dir: "/etc/openshell-tls/session-control".to_string(), + }; + let gateway = |shared: Option| KubernetesComputeConfig { + supervisor_topology: Some(SupervisorTopology::Gateway), + shared_supervisor: shared, + ..KubernetesComputeConfig::default() + }; + assert!( + gateway(Some(shared())) + .validate_supervisor_topology() + .is_ok() + ); + assert!(gateway(None).validate_supervisor_topology().is_err()); + // The connection alone does not opt a gateway in. + let unset = KubernetesComputeConfig { + shared_supervisor: Some(shared()), + ..KubernetesComputeConfig::default() + }; + assert!(unset.validate_supervisor_topology().is_err()); + for invalid in [ + SharedSupervisorConfig { + control_port: 8443, + ..shared() + }, + SharedSupervisorConfig { + service_namespace: "Not_A_Label".to_string(), + ..shared() + }, + SharedSupervisorConfig { + control_tls_dir: "relative".to_string(), + ..shared() + }, + ] { + assert!( + gateway(Some(invalid)) + .validate_supervisor_topology() + .is_err() + ); + } + // A shared supervisor holds no per-sandbox workload identity or proxy credential. + let with_proxy = KubernetesComputeConfig { + https_proxy: Some("http://proxy.example:3128".to_string()), + ..gateway(Some(shared())) + }; + assert!(with_proxy.validate_supervisor_topology().is_err()); + let with_proxy_ca = KubernetesComputeConfig { + proxy_ca_bundle: Some("/etc/proxy/ca.pem".to_string()), + ..gateway(Some(shared())) + }; + assert!(with_proxy_ca.validate_supervisor_topology().is_err()); + let workspace = KubernetesComputeConfig { + supervisor_topology: Some(SupervisorTopology::Workspace), + ..KubernetesComputeConfig::default() + }; + assert!(workspace.validate_supervisor_topology().is_err()); + } + #[test] fn image_pull_policy_accepts_config_and_kubernetes_spellings() { for (value, expected) in [ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 0a1acc7be2..28938bf1ce 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -11,7 +11,7 @@ use crate::config::{ }; use crate::isolation::{ BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesControlPath, - KubernetesSandboxRuntimeBoundarySpec, + KubernetesSandboxRuntimeBoundarySpec, KubernetesSupervisorClaim, }; use crate::lifecycle::LifecycleGates; use crate::sandbox_runtime::{ @@ -19,9 +19,13 @@ use crate::sandbox_runtime::{ SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, SupervisorClientTls, boundary_service, boundary_service_matches_mode, control_policy, generate_proxy_ca_material, image_pull_secret_copy, sandbox_bootstrap_secret, - sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, + sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, shared_control_policy, supervisor_bootstrap_secret, supervisor_pod, workload_fence, }; +use crate::shared_supervisor::{ + ANNOTATION_SUPERVISOR_SESSION, ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY, + SharedSupervisor, SupervisorPlacement, supervisor_placement, +}; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::authentication::v1::{ TokenReview, TokenReviewSpec, TokenReviewStatus, UserInfo, @@ -189,6 +193,37 @@ struct StagedBootstrapMaterial { } /// Stored coordinates of a validated runtime-initiated control policy. +/// The supervisor a generation is installed with. +enum GenerationSupervisor { + /// A supervisor Pod paired with this generation's workload Pod. + Dedicated { + pod_uid: String, + connection: KubernetesControlConnection, + }, + /// A session on the gateway's shared supervisor. + Shared { route: String }, +} + +impl GenerationSupervisor { + /// Third component of the runtime identity. A session route can never + /// equal a Pod UID, so no Pod can authenticate as a shared generation. + fn runtime_identity_component(&self) -> &str { + match self { + Self::Dedicated { pod_uid, .. } => pod_uid, + Self::Shared { route } => route, + } + } +} + +/// Where workload runtimes reach the gateway's shared supervisor. +struct SharedSupervisorTarget { + namespace: String, + authority: String, + address: std::net::SocketAddr, + /// Labels of the shared supervisor Pods, from their Service. + selector: BTreeMap, +} + struct VerifiedControlPolicy { uid: String, resource_version: String, @@ -353,6 +388,8 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError { } } const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); +/// Unsupervised shared sessions removed at once by one reconcile pass. +const SHARED_SESSION_GC_CONCURRENCY: usize = 4; /// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5); @@ -828,6 +865,8 @@ pub struct KubernetesComputeDriver { lifecycle_gates: Arc, config: KubernetesComputeConfig, operator_allowlist: Option, + /// Session-control client while new generations use the gateway topology. + shared_supervisor: Option, } impl std::fmt::Debug for KubernetesComputeDriver { @@ -876,6 +915,7 @@ impl KubernetesComputeDriver { lifecycle_gates: Arc::default(), config, operator_allowlist: None, + shared_supervisor: None, } } @@ -908,6 +948,9 @@ impl KubernetesComputeDriver { config .validate_allowed_pod_labels() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_supervisor_topology() + .map_err(KubernetesDriverError::Precondition)?; // Read the bundle once at startup so an unreadable path or a // certificate-free file fails the gateway here, rather than as a // repeated sandbox-create failure. The authoritative read still @@ -956,6 +999,7 @@ impl KubernetesComputeDriver { None }; + let shared_supervisor = config.shared_supervisor.clone().map(SharedSupervisor::new); let driver = Self { client, watch_client, @@ -963,6 +1007,7 @@ impl KubernetesComputeDriver { lifecycle_gates: Arc::default(), config, operator_allowlist, + shared_supervisor, }; if driver.workspace_mode() == WorkspaceMode::Shared { @@ -1920,12 +1965,15 @@ impl KubernetesComputeDriver { .namespace .clone() .unwrap_or_else(|| self.config.namespace.clone()); - Ok( - sandbox_from_object_with_sandbox_runtime_readiness(&self.client, &ns, obj) - .await - .ok() - .map(|(_, sandbox)| sandbox), + Ok(sandbox_from_object_with_sandbox_runtime_readiness( + &self.client, + self.shared_supervisor.as_ref(), + &ns, + obj, ) + .await + .ok() + .map(|(_, sandbox)| sandbox)) } Ok(Err(err)) => { warn!( @@ -1976,8 +2024,13 @@ impl KubernetesComputeDriver { .namespace .clone() .unwrap_or_else(|| self.config.namespace.clone()); - match sandbox_from_object_with_sandbox_runtime_readiness(&self.client, &ns, obj) - .await + match sandbox_from_object_with_sandbox_runtime_readiness( + &self.client, + self.shared_supervisor.as_ref(), + &ns, + obj, + ) + .await { Ok((_, sandbox)) => sandboxes.push(sandbox), Err(err) => { @@ -2105,6 +2158,7 @@ impl KubernetesComputeDriver { .await; let generation = random_sandbox_runtime_token()?; + let placement = self.new_generation_placement(); let proxy_names = SandboxRuntimeNames::for_generation(&sandbox.id, &generation); let image_pull_secrets = self.generation_image_pull_secret_names(&proxy_names); let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( @@ -2199,6 +2253,13 @@ impl KubernetesComputeDriver { ANNOTATION_SANDBOX_RUNTIME_CONTROL_GENERATION.to_string(), generation.clone(), ); + if let SupervisorPlacement::Shared { route } = &placement { + annotations.insert( + ANNOTATION_SUPERVISOR_TOPOLOGY.to_string(), + GATEWAY_TOPOLOGY.to_string(), + ); + annotations.insert(ANNOTATION_SUPERVISOR_SESSION.to_string(), route.clone()); + } for key in [ crate::config::ANNOTATION_SCC_UID_RANGE, crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS, @@ -2261,6 +2322,7 @@ impl KubernetesComputeDriver { &created, &proxy_names, control_connection, + &placement, resolved_user_id, resolved_group_id, &main_process_spec, @@ -2271,6 +2333,7 @@ impl KubernetesComputeDriver { Ok(runtime_identity) => runtime_identity, Err(error) => { warn!(sandbox_id = %sandbox.id, %error, "sandbox-runtime provisioning failed; rolling back Sandbox CR"); + self.remove_shared_session(&placement).await; let _ = agent_sandbox_api .api .delete(&kube_name, &DeleteParams::default()) @@ -2364,6 +2427,11 @@ impl KubernetesComputeDriver { ) -> Result<(), KubernetesDriverError> { let mode = sandbox_runtime_control_connection(object)?; let names = SandboxRuntimeNames::new(sandbox_id); + if let SupervisorPlacement::Shared { .. } = supervisor_placement(object)? { + return self + .verify_sandbox_runtime_shared_control_policy(object, namespace, &names) + .await; + } let service_name = names.control_service(mode); let services = Api::::namespaced(self.client.clone(), namespace); let service = tokio::time::timeout(KUBE_API_TIMEOUT, services.get_opt(service_name)) @@ -2431,6 +2499,53 @@ impl KubernetesComputeDriver { Ok(()) } + /// A shared generation's egress allowance must remain the object bound + /// into its boundary evidence; any change can widen workload egress. + async fn verify_sandbox_runtime_shared_control_policy( + &self, + object: &DynamicObject, + namespace: &str, + names: &SandboxRuntimeNames, + ) -> Result<(), KubernetesDriverError> { + let policies = Api::::namespaced(self.client.clone(), namespace); + let stored = tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.get_opt(&names.shared_control_policy), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading the shared control NetworkPolicy".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + // A bootstrap still installing has not bound its policy yet. + if stored.is_none() + && sandbox_runtime_bootstrap_phase(object) + == Some(SandboxRuntimeBootstrapPhase::Preparing) + { + return Ok(()); + } + let stored = stored.ok_or_else(|| { + KubernetesDriverError::Precondition(format!( + "shared control NetworkPolicy {} is missing", + names.shared_control_policy + )) + })?; + if !sandbox_runtime_policy_generation_matches( + &stored, + object, + ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID, + ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION, + ) { + return Err(KubernetesDriverError::Precondition(format!( + "shared control NetworkPolicy {} changed after it was bound", + names.shared_control_policy + ))); + } + Ok(()) + } + async fn create_sandbox_runtime_fence( &self, namespace: &str, @@ -2711,6 +2826,7 @@ impl KubernetesComputeDriver { sandbox_cr: &DynamicObject, names: &SandboxRuntimeNames, control_connection: KubernetesControlConnection, + placement: &SupervisorPlacement, agent_uid: u32, agent_gid: u32, main_process_spec: &str, @@ -2736,35 +2852,48 @@ impl KubernetesComputeDriver { &sandbox_api.resource.api_version, false, ); - // Preserve the legacy workload endpoint even for newly created - // sandboxes: an old gateway may handle their next start mid-rollout. - for mode in [ - KubernetesControlConnection::SupervisorInitiated, - KubernetesControlConnection::RuntimeInitiated, - ] { - self.ensure_sandbox_runtime_service( - namespace, - names, - &sandbox.id, - mode, - dependent_owner.clone(), - ) - .await?; - } - let supervisor_uid = self - .create_sandbox_runtime_supervisor( - namespace, - names, - &sandbox.id, - &sandbox.name, - dependent_owner, - agent_uid, - agent_gid, - main_process_spec, - log_level, - staged.upstream_proxy_ca_bundle.is_some(), - ) - .await?; + let supervisor = match placement { + SupervisorPlacement::Dedicated => { + // Preserve the legacy workload endpoint even for newly created + // sandboxes: an old gateway may handle their next start mid-rollout. + for mode in [ + KubernetesControlConnection::SupervisorInitiated, + KubernetesControlConnection::RuntimeInitiated, + ] { + self.ensure_sandbox_runtime_service( + namespace, + names, + &sandbox.id, + mode, + dependent_owner.clone(), + ) + .await?; + } + let pod_uid = self + .create_sandbox_runtime_supervisor( + namespace, + names, + &sandbox.id, + &sandbox.name, + dependent_owner, + agent_uid, + agent_gid, + main_process_spec, + log_level, + staged.upstream_proxy_ca_bundle.is_some(), + ) + .await?; + GenerationSupervisor::Dedicated { + pod_uid, + connection: control_connection, + } + } + // The shared supervisor already runs; this generation becomes a + // session on it during installation. + SupervisorPlacement::Shared { route } => GenerationSupervisor::Shared { + route: route.clone(), + }, + }; // Release the Sandbox CR only far enough for the controller to create // the workload Pod. The Pod remains unschedulable because its template @@ -2793,8 +2922,7 @@ impl KubernetesComputeDriver { &sandbox.id, cr_uid, names, - control_connection, - &supervisor_uid, + &supervisor, agent_uid, agent_gid, child_env, @@ -2807,11 +2935,11 @@ impl KubernetesComputeDriver { // control process without deadlocking behind this driver call. Only // boundary PID 1 is running at this point; the agent process cannot // start until control attaches and confirms enforcement. Reconcile - // removes the marker after the supervisor Pod becomes Ready. + // removes the marker after the supervisor becomes ready. Ok(kubernetes_runtime_identity( namespace, cr_uid, - &supervisor_uid, + supervisor.runtime_identity_component(), )) } @@ -2851,6 +2979,35 @@ impl KubernetesComputeDriver { Ok(()) } + /// Placement for a generation this driver starts now. + fn new_generation_placement(&self) -> SupervisorPlacement { + if self.shared_supervisor.is_some() { + SupervisorPlacement::Shared { + route: openshell_sandbox_backend::runtime_routed::new_route_handle(), + } + } else { + SupervisorPlacement::Dedicated + } + } + + /// Remove a shared generation's session, if any. Session loss also ends + /// supervision when the workload is suspended, so failures only log. + async fn remove_shared_session(&self, placement: &SupervisorPlacement) { + let SupervisorPlacement::Shared { route } = placement else { + return; + }; + let Some(shared) = &self.shared_supervisor else { + warn!( + route, + "shared supervisor is not configured; its session cannot be removed" + ); + return; + }; + if let Err(error) = shared.remove(route).await { + warn!(route, %error, "could not remove shared supervisor session"); + } + } + /// Read operator-provided bootstrap material once per generation, so the /// whole generation stages, mounts, and references the same bytes, and so /// unreadable material fails before any Service, Pod, or Secret is created. @@ -2927,6 +3084,91 @@ impl KubernetesComputeDriver { .ok_or_else(|| KubernetesDriverError::Message("supervisor Pod has no UID".to_string())) } + /// Read the gateway's shared supervisor Service: its address and the + /// labels of the Pods it selects. + async fn shared_supervisor_target( + &self, + ) -> Result { + let shared = self.shared_supervisor.as_ref().ok_or_else(|| { + KubernetesDriverError::Precondition( + "the gateway supervisor topology is not configured".to_string(), + ) + })?; + let config = shared.config(); + let services = Api::::namespaced(self.client.clone(), &config.service_namespace); + let service = tokio::time::timeout(KUBE_API_TIMEOUT, services.get(&config.service_name)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading the shared supervisor Service".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + let spec = service.spec.unwrap_or_default(); + let selector = spec + .selector + .filter(|selector| !selector.is_empty()) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "the shared supervisor Service selects no Pods".to_string(), + ) + })?; + let address: std::net::IpAddr = spec + .cluster_ip + .filter(|ip| ip != "None") + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "the shared supervisor Service has no ClusterIP".to_string(), + ) + })? + .parse() + .map_err(|error| { + KubernetesDriverError::Message(format!( + "invalid shared supervisor Service ClusterIP: {error}" + )) + })?; + Ok(SharedSupervisorTarget { + namespace: config.service_namespace.clone(), + authority: format!("{}:{}", config.service_host(), config.runtime_port), + address: std::net::SocketAddr::new(address, config.runtime_port), + selector, + }) + } + + /// Create or validate this sandbox's egress allowance to the shared + /// supervisor and return the stored coordinates bound into its evidence. + async fn ensure_sandbox_runtime_shared_control_policy( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + target: &SharedSupervisorTarget, + owner: OwnerReference, + ) -> Result { + let policies: Api = Api::namespaced(self.client.clone(), namespace); + let expected = shared_control_policy( + namespace, + names, + sandbox_id, + &target.namespace, + target.selector.clone(), + target.address.port(), + owner, + ); + create_or_validate_sandbox_runtime_fence(&policies, &expected).await?; + let stored = + tokio::time::timeout(KUBE_API_TIMEOUT, policies.get(&names.shared_control_policy)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading the shared control NetworkPolicy".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + validate_sandbox_runtime_fence(&stored, &expected)?; + VerifiedControlPolicy::from_policy(&stored) + } + #[allow(clippy::too_many_arguments, clippy::similar_names)] async fn install_sandbox_runtime_generation( &self, @@ -2936,8 +3178,7 @@ impl KubernetesComputeDriver { sandbox_id: &str, cr_uid: &str, names: &SandboxRuntimeNames, - control_connection: KubernetesControlConnection, - supervisor_uid: &str, + supervisor: &GenerationSupervisor, agent_uid: u32, agent_gid: u32, child_env: std::collections::HashMap, @@ -2953,33 +3194,43 @@ impl KubernetesComputeDriver { .ok_or_else(|| { KubernetesDriverError::Message("sandbox namespace has no UID".to_string()) })?; - let services = Api::::namespaced(self.client.clone(), namespace); - let service_name = names.control_service(control_connection); - let service = services - .get(service_name) - .await - .map_err(KubernetesDriverError::from_kube)?; - // Running generations must retain the Service role prepared for them. - if !boundary_service_matches_mode(&service, sandbox_id, control_connection) { - return Err(KubernetesDriverError::Precondition(format!( - "boundary Service {} does not select the {} listener recorded for this sandbox", - service_name, - control_connection.as_str() - ))); - } - let service_ip: std::net::IpAddr = service - .spec - .and_then(|spec| spec.cluster_ip) - .filter(|ip| ip != "None") - .ok_or_else(|| { - KubernetesDriverError::Message("boundary Service has no ClusterIP".to_string()) - })? - .parse() - .map_err(|error| { - KubernetesDriverError::Message(format!( - "invalid boundary Service ClusterIP: {error}" - )) - })?; + let (service_ip, shared_target) = match supervisor { + GenerationSupervisor::Dedicated { connection, .. } => { + let services = Api::::namespaced(self.client.clone(), namespace); + let service_name = names.control_service(*connection); + let service = services + .get(service_name) + .await + .map_err(KubernetesDriverError::from_kube)?; + // Running generations must retain the Service role prepared for them. + if !boundary_service_matches_mode(&service, sandbox_id, *connection) { + return Err(KubernetesDriverError::Precondition(format!( + "boundary Service {} does not select the {} listener recorded for this sandbox", + service_name, + connection.as_str() + ))); + } + let service_ip: std::net::IpAddr = service + .spec + .and_then(|spec| spec.cluster_ip) + .filter(|ip| ip != "None") + .ok_or_else(|| { + KubernetesDriverError::Message( + "boundary Service has no ClusterIP".to_string(), + ) + })? + .parse() + .map_err(|error| { + KubernetesDriverError::Message(format!( + "invalid boundary Service ClusterIP: {error}" + )) + })?; + (Some(service_ip), None) + } + GenerationSupervisor::Shared { .. } => { + (None, Some(self.shared_supervisor_target().await?)) + } + }; let policies = Api::::namespaced(self.client.clone(), namespace); let fence = policies .get(&names.workload_policy) @@ -2996,22 +3247,52 @@ impl KubernetesComputeDriver { "workload NetworkPolicy has no resourceVersion".to_string(), ) })?; - let control_policy = match control_connection { - KubernetesControlConnection::SupervisorInitiated => None, - KubernetesControlConnection::RuntimeInitiated => Some( + let policy_owner = || { + sandbox_runtime_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, + ) + }; + let control_policy = match (supervisor, &shared_target) { + ( + GenerationSupervisor::Dedicated { + connection: KubernetesControlConnection::SupervisorInitiated, + .. + }, + _, + ) => None, + ( + GenerationSupervisor::Dedicated { + connection: KubernetesControlConnection::RuntimeInitiated, + .. + }, + _, + ) => Some( self.ensure_sandbox_runtime_control_policy( namespace, names, sandbox_id, - sandbox_runtime_sandbox_owner_reference( - cr_name, - cr_uid, - &sandbox_api.resource.api_version, - false, - ), + policy_owner(), + ) + .await?, + ), + (GenerationSupervisor::Shared { .. }, Some(target)) => Some( + self.ensure_sandbox_runtime_shared_control_policy( + namespace, + names, + sandbox_id, + target, + policy_owner(), ) .await?, ), + (GenerationSupervisor::Shared { .. }, None) => { + return Err(KubernetesDriverError::Message( + "shared supervisor target was not resolved".to_string(), + )); + } }; let pods = Api::::namespaced(self.client.clone(), namespace); @@ -3043,10 +3324,12 @@ impl KubernetesComputeDriver { })?; let mut annotations = serde_json::json!({ ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid.clone(), - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid.clone(), ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: fence_generation.to_string(), }); + if let GenerationSupervisor::Dedicated { pod_uid, .. } = supervisor { + annotations[ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID] = pod_uid.clone().into(); + } if let Some(policy) = &control_policy { annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID] = policy.uid.clone().into(); annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION] = @@ -3073,6 +3356,35 @@ impl KubernetesComputeDriver { format!("sandbox:{cr_uid}"), ) .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let (supervisor_claim, control_path) = match (supervisor, shared_target, service_ip) { + (GenerationSupervisor::Dedicated { pod_uid, .. }, _, Some(service_ip)) => ( + KubernetesSupervisorClaim::Pod(pod_uid.clone()), + self.sandbox_runtime_control_path(namespace, names, service_ip, control_policy), + ), + (GenerationSupervisor::Shared { route }, Some(target), _) => { + let policy = control_policy.ok_or_else(|| { + KubernetesDriverError::Message( + "shared control policy was not verified".to_string(), + ) + })?; + ( + KubernetesSupervisorClaim::Session(route.clone()), + KubernetesControlPath::RuntimeInitiatedShared { + control_authority: target.authority, + supervisor_address: target.address, + supervisor_namespace: target.namespace, + route: route.clone(), + control_policy_uid: policy.uid, + control_policy_resource_version: policy.resource_version, + }, + ) + } + _ => { + return Err(KubernetesDriverError::Message( + "sandbox-runtime control target was not resolved".to_string(), + )); + } + }; let provisioned = KubernetesSandboxRuntimeBoundarySpec { boundary_id: sandbox_id.to_string(), generation: launch_authentication @@ -3088,15 +3400,10 @@ impl KubernetesComputeDriver { sandbox_resource_uid: cr_uid.to_string(), workload_pod_uid: workload_pod_uid.clone(), workload_pod_uid_path: PathBuf::from(SANDBOX_POD_UID_PATH), - supervisor_pod_uid: supervisor_uid.to_string(), + supervisor: supervisor_claim, egress_policy_uid: fence_uid.clone(), egress_policy_resource_version: fence_resource_version.clone(), - control_path: self.sandbox_runtime_control_path( - namespace, - names, - service_ip, - control_policy, - ), + control_path, sandbox_tls: SandboxTlsServerConfig { certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_PATH), @@ -3111,12 +3418,7 @@ impl KubernetesComputeDriver { } .provision() .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - let descriptor = provisioned - .runtime_descriptor - .backend_descriptor() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; let workload_owner = runtime_pod_owner(&workload_pod_name, &workload_pod_uid); - let supervisor_owner = runtime_pod_owner(&names.supervisor_pod, supervisor_uid); let sandbox_secret = sandbox_bootstrap_secret( namespace, names, @@ -3129,44 +3431,81 @@ impl KubernetesComputeDriver { tls.private_key_pem.into_bytes(), workload_owner.clone(), ); - let supervisor_secret = supervisor_bootstrap_secret( - namespace, - names, - sandbox_id, - descriptor.payload, - serde_json::to_vec(&launch_authentication.supervisor).map_err(|error| { - KubernetesDriverError::Message(format!("encode supervisor auth bundle: {error}")) - })?, - proxy_ca.certificate_pem.into_bytes(), - proxy_ca.private_key_pem.into_bytes(), - staged.upstream_proxy_ca_bundle, - staged.client_tls, - supervisor_owner.clone(), - ); let secrets = Api::::namespaced(self.client.clone(), namespace); secrets .create(&PostParams::default(), &sandbox_secret) .await .map_err(KubernetesDriverError::from_kube)?; - secrets - .create(&PostParams::default(), &supervisor_secret) - .await - .map_err(KubernetesDriverError::from_kube)?; - self.create_generation_image_pull_secrets( - namespace, - names, - sandbox_id, - vec![workload_owner, supervisor_owner], - ) - .await?; - - pods.patch( - &names.supervisor_pod, - &PatchParams::default(), - &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), - ) - .await - .map_err(KubernetesDriverError::from_kube)?; + match supervisor { + GenerationSupervisor::Dedicated { pod_uid, .. } => { + let descriptor = provisioned + .runtime_descriptor + .backend_descriptor() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let supervisor_owner = runtime_pod_owner(&names.supervisor_pod, pod_uid); + let supervisor_secret = supervisor_bootstrap_secret( + namespace, + names, + sandbox_id, + descriptor.payload, + serde_json::to_vec(&launch_authentication.supervisor).map_err(|error| { + KubernetesDriverError::Message(format!( + "encode supervisor auth bundle: {error}" + )) + })?, + proxy_ca.certificate_pem.into_bytes(), + proxy_ca.private_key_pem.into_bytes(), + staged.upstream_proxy_ca_bundle, + staged.client_tls, + supervisor_owner.clone(), + ); + secrets + .create(&PostParams::default(), &supervisor_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + self.create_generation_image_pull_secrets( + namespace, + names, + sandbox_id, + vec![workload_owner, supervisor_owner], + ) + .await?; + pods.patch( + &names.supervisor_pod, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + } + GenerationSupervisor::Shared { route } => { + self.create_generation_image_pull_secrets( + namespace, + names, + sandbox_id, + vec![workload_owner], + ) + .await?; + let shared = self.shared_supervisor.as_ref().ok_or_else(|| { + KubernetesDriverError::Precondition( + "the gateway supervisor topology is not configured".to_string(), + ) + })?; + // Registration only admits the session; it does not wait for + // the runtime, so the gateway can persist this generation + // before the supervisor reads its configuration. + shared + .register(&shared_session_registration( + &admission_object, + route, + sandbox_id, + provisioned.runtime_descriptor, + launch_authentication, + proxy_ca, + )?) + .await?; + } + } pods.patch( &workload_pod_name, &PatchParams::default(), @@ -3178,13 +3517,30 @@ impl KubernetesComputeDriver { sandbox_runtime_bootstrap_phase_patch(version, SandboxRuntimeBootstrapPhase::Released) }) .await?; - spawn_sandbox_runtime_bootstrap_completion( - pods, - sandbox_api.api.clone(), - names.supervisor_pod.clone(), - cr_name.to_string(), - Some(cr_uid.to_string()), - ); + match supervisor { + GenerationSupervisor::Dedicated { .. } => spawn_sandbox_runtime_bootstrap_completion( + BootstrapControl::SupervisorPod { + pods, + name: names.supervisor_pod.clone(), + }, + sandbox_api.api.clone(), + cr_name.to_string(), + Some(cr_uid.to_string()), + ), + GenerationSupervisor::Shared { route } => { + if let Some(shared) = self.shared_supervisor.clone() { + spawn_sandbox_runtime_bootstrap_completion( + BootstrapControl::SharedSession { + shared, + route: route.clone(), + }, + sandbox_api.api.clone(), + cr_name.to_string(), + Some(cr_uid.to_string()), + ); + } + } + } Ok(()) } @@ -3205,10 +3561,10 @@ impl KubernetesComputeDriver { } async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout, phase) = + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout, phase, placement) = self.prepare_sandbox_stop(sandbox_id).await?; if phase != Some(SandboxRuntimeBootstrapPhase::Suspending) { - self.delete_sandbox_runtime_supervisor_pod(sandbox_id, &namespace) + self.release_generation_supervisor(sandbox_id, &namespace, &placement) .await?; patch_dynamic_object_with_resource_version_retry( &agent_sandbox_api.api, @@ -3360,7 +3716,14 @@ impl KubernetesComputeDriver { .namespace .as_deref() .unwrap_or(&self.config.namespace); - if sandbox_runtime_control_availability(&self.client, namespace, sandbox_id).await + if generation_control_availability( + &self.client, + self.shared_supervisor.as_ref(), + namespace, + sandbox_id, + &object, + ) + .await == SandboxRuntimeControlAvailability::Available && sandbox_runtime_bootstrap_is_ready_to_complete(&object) { @@ -3369,15 +3732,14 @@ impl KubernetesComputeDriver { let cr_uid = object.metadata.uid.as_deref().ok_or_else(|| { KubernetesDriverError::Message("sandbox resource has no UID".to_string()) })?; - let supervisor_uid = required_sandbox_annotation( - &object, - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID, - )?; - return Ok(kubernetes_runtime_identity( - namespace, - cr_uid, - &supervisor_uid, - )); + let supervisor = match supervisor_placement(&object)? { + SupervisorPlacement::Dedicated => required_sandbox_annotation( + &object, + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID, + )?, + SupervisorPlacement::Shared { route } => route, + }; + return Ok(kubernetes_runtime_identity(namespace, cr_uid, &supervisor)); } } @@ -3440,6 +3802,7 @@ impl KubernetesComputeDriver { } let control_connection = KubernetesControlConnection::RuntimeInitiated; + let placement = self.new_generation_placement(); let names = SandboxRuntimeNames::for_generation(sandbox_id, generation.as_str()); self.create_sandbox_runtime_fence(&namespace, &names) .await?; @@ -3448,35 +3811,55 @@ impl KubernetesComputeDriver { self.delete_sandbox_runtime_supervisor(sandbox_id, &namespace, &stale_generations) .await?; // The legacy Service keeps selecting the workload for old gateways. - // Add the supervisor endpoint when upgrading a legacy sandbox. - let services = Api::::namespaced(self.client.clone(), &namespace); - let service = services - .get(&names.boundary_service) - .await - .map_err(KubernetesDriverError::from_kube)?; - if !boundary_service_matches_mode( - &service, - sandbox_id, - KubernetesControlConnection::SupervisorInitiated, - ) { - return Err(KubernetesDriverError::Precondition(format!( - "boundary Service {} no longer selects the workload", - names.boundary_service - ))); - } - self.ensure_sandbox_runtime_service( - &namespace, - &names, - sandbox_id, - control_connection, - sandbox_runtime_sandbox_owner_reference( + // Add the supervisor endpoint when upgrading a legacy sandbox. A + // shared generation has no endpoint of its own, so a dedicated + // generation that follows one creates both. + if placement == SupervisorPlacement::Dedicated { + let owner = sandbox_runtime_sandbox_owner_reference( cr_name, cr_uid, &sandbox_api.resource.api_version, false, - ), - ) - .await?; + ); + let services = Api::::namespaced(self.client.clone(), &namespace); + let legacy = services + .get_opt(&names.boundary_service) + .await + .map_err(KubernetesDriverError::from_kube)?; + match legacy { + Some(service) + if !boundary_service_matches_mode( + &service, + sandbox_id, + KubernetesControlConnection::SupervisorInitiated, + ) => + { + return Err(KubernetesDriverError::Precondition(format!( + "boundary Service {} no longer selects the workload", + names.boundary_service + ))); + } + Some(_) => {} + None => { + self.ensure_sandbox_runtime_service( + &namespace, + &names, + sandbox_id, + KubernetesControlConnection::SupervisorInitiated, + owner.clone(), + ) + .await?; + } + } + self.ensure_sandbox_runtime_service( + &namespace, + &names, + sandbox_id, + control_connection, + owner, + ) + .await?; + } let main_process_spec = required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC)?; let log_level = required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL)?; @@ -3502,25 +3885,33 @@ impl KubernetesComputeDriver { agent_gid, ), )?; - let supervisor_uid = self - .create_sandbox_runtime_supervisor( - &namespace, - &names, - sandbox_id, - &sandbox_name, - sandbox_runtime_sandbox_owner_reference( - cr_name, - cr_uid, - &sandbox_api.resource.api_version, - false, - ), - agent_uid, - agent_gid, - &main_process_spec, - &log_level, - staged.upstream_proxy_ca_bundle.is_some(), - ) - .await?; + let supervisor = match &placement { + SupervisorPlacement::Dedicated => GenerationSupervisor::Dedicated { + pod_uid: self + .create_sandbox_runtime_supervisor( + &namespace, + &names, + sandbox_id, + &sandbox_name, + sandbox_runtime_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, + ), + agent_uid, + agent_gid, + &main_process_spec, + &log_level, + staged.upstream_proxy_ca_bundle.is_some(), + ) + .await?, + connection: control_connection, + }, + SupervisorPlacement::Shared { route } => GenerationSupervisor::Shared { + route: route.clone(), + }, + }; let mut volumes = object .data @@ -3553,6 +3944,16 @@ impl KubernetesComputeDriver { |version| { let mut running_patch = sandbox_operating_state_patch(&sandbox_api.resource.version, version, true); + let (supervisor_uid, topology, session) = match &supervisor { + GenerationSupervisor::Dedicated { pod_uid, .. } => { + (pod_uid.as_str().into(), serde_json::Value::Null, serde_json::Value::Null) + } + GenerationSupervisor::Shared { route } => ( + serde_json::Value::Null, + GATEWAY_TOPOLOGY.into(), + route.as_str().into(), + ), + }; running_patch["metadata"]["annotations"] = serde_json::json!({ ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), @@ -3560,9 +3961,11 @@ impl KubernetesComputeDriver { ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::Preparing.as_str(), ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: launch_authentication.supervisor.session_id.to_string(), ANNOTATION_SANDBOX_RUNTIME_GENERATION: generation.as_str(), - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION: control_connection.as_str(), ANNOTATION_SANDBOX_RUNTIME_CONTROL_GENERATION: generation.as_str(), + ANNOTATION_SUPERVISOR_TOPOLOGY: topology, + ANNOTATION_SUPERVISOR_SESSION: session, }); running_patch["spec"]["podTemplate"]["spec"] = restart_pod_template_spec( &volumes, @@ -3582,8 +3985,7 @@ impl KubernetesComputeDriver { sandbox_id, cr_uid, &names, - control_connection, - &supervisor_uid, + &supervisor, agent_uid, agent_gid, child_env, @@ -3595,6 +3997,8 @@ impl KubernetesComputeDriver { .await; if let Err(error) = restart { + // The restart patch may not have recorded the session yet. + self.remove_shared_session(&placement).await; if let Err(rollback_error) = self.stop_sandbox_inner(sandbox_id).await { return Err(KubernetesDriverError::Message(format!( "restart failed: {error}; rollback remains pending: {rollback_error}" @@ -3605,7 +4009,7 @@ impl KubernetesComputeDriver { Ok(kubernetes_runtime_identity( &namespace, cr_uid, - &supervisor_uid, + supervisor.runtime_identity_component(), )) } @@ -3620,6 +4024,7 @@ impl KubernetesComputeDriver { String, Duration, Option, + SupervisorPlacement, ), KubernetesDriverError, > { @@ -3648,6 +4053,7 @@ impl KubernetesComputeDriver { .next() .ok_or(KubernetesDriverError::NotFound)?; let phase = sandbox_runtime_bootstrap_phase(&object); + let placement = placement_for_release(&object, sandbox_id); let namespace = object .metadata .namespace @@ -3695,9 +4101,33 @@ impl KubernetesComputeDriver { namespace, stop_timeout, phase, + placement, )) } + /// End supervision of the current generation before its workload stops. + /// + /// A shared session is removed best effort: suspending the workload ends + /// that session's supervision anyway, and reconciliation removes any + /// session left behind once the shared supervisor is reachable. + async fn release_generation_supervisor( + &self, + sandbox_id: &str, + namespace: &str, + placement: &SupervisorPlacement, + ) -> Result<(), KubernetesDriverError> { + match placement { + SupervisorPlacement::Dedicated => { + self.delete_sandbox_runtime_supervisor_pod(sandbox_id, namespace) + .await + } + SupervisorPlacement::Shared { .. } => { + self.remove_shared_session(placement).await; + Ok(()) + } + } + } + async fn delete_sandbox_runtime_supervisor_pod( &self, sandbox_id: &str, @@ -3815,7 +4245,7 @@ impl KubernetesComputeDriver { .await?; let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, obj_namespace, _workspace, preconditions, pod_name) = + let (kube_name, obj_namespace, _workspace, preconditions, pod_name, placement) = match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { Ok(Ok(list)) => { if let Some(obj) = list.items.into_iter().next() { @@ -3832,6 +4262,8 @@ impl KubernetesComputeDriver { .as_ref() .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) .unwrap_or_default(); + // An unreadable placement leaves any session to reconciliation. + let placement = supervisor_placement(&obj).ok(); let pc = Preconditions { uid: obj.metadata.uid, resource_version: obj.metadata.resource_version, @@ -3845,7 +4277,7 @@ impl KubernetesComputeDriver { }) .cloned() .unwrap_or_else(|| name.clone()); - (name, ns, ws, pc, pod_name) + (name, ns, ws, pc, pod_name, placement) } None => return Ok(false), } @@ -3882,6 +4314,9 @@ impl KubernetesComputeDriver { match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { Ok(Ok(_response)) => { info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); + if let Some(placement) = &placement { + self.remove_shared_session(placement).await; + } { let pod_api = Api::::namespaced(self.client.clone(), &obj_namespace); let deadline = tokio::time::Instant::now() @@ -3976,6 +4411,19 @@ impl KubernetesComputeDriver { return; } }; + // List sessions before Sandboxes: a generation records its route on + // its Sandbox before registering, so every session listed here whose + // Sandbox still supervises it appears in the Sandbox list below. + let sessions = match &self.shared_supervisor { + Some(shared) => match shared.list().await { + Ok(list) => Some(list.sessions), + Err(error) => { + warn!(%error, "could not list shared supervisor sessions; reconciliation will retry"); + None + } + }, + None => None, + }; let list = match tokio::time::timeout( KUBE_API_TIMEOUT, lookup_api @@ -3995,6 +4443,11 @@ impl KubernetesComputeDriver { } }; + if let Some(sessions) = sessions { + self.remove_unsupervised_sessions(sessions, &list.items) + .await; + } + for object in list.items { let Ok(sandbox_id) = sandbox_id_from_object(&object) else { continue; @@ -4135,7 +4588,14 @@ impl KubernetesComputeDriver { } } if sandbox_runtime_bootstrap_in_progress(&object) { - if sandbox_runtime_control_availability(&self.client, namespace, &sandbox_id).await + if generation_control_availability( + &self.client, + self.shared_supervisor.as_ref(), + namespace, + &sandbox_id, + &object, + ) + .await == SandboxRuntimeControlAvailability::Available && sandbox_runtime_bootstrap_is_ready_to_complete(&object) { @@ -4203,7 +4663,7 @@ impl KubernetesComputeDriver { &sandbox_id, namespace, desired_running, - sandbox_runtime_generation(&object).as_slice(), + &object, ) .await { @@ -4217,8 +4677,14 @@ impl KubernetesComputeDriver { warn!(sandbox_id, %error, "failed to reconcile sandbox-runtime supervisor Pod"); } } - let availability = - sandbox_runtime_control_availability(&self.client, namespace, &sandbox_id).await; + let availability = generation_control_availability( + &self.client, + self.shared_supervisor.as_ref(), + namespace, + &sandbox_id, + &object, + ) + .await; if desired_running && availability == SandboxRuntimeControlAvailability::Unavailable { warn!( sandbox_id, @@ -4233,6 +4699,35 @@ impl KubernetesComputeDriver { } } + /// Remove shared sessions that no Sandbox's current generation may still + /// run, such as those of deleted Sandboxes or of stops that could not + /// reach the shared supervisor. + async fn remove_unsupervised_sessions( + &self, + sessions: Vec, + sandboxes: &[DynamicObject], + ) { + let Some(shared) = &self.shared_supervisor else { + return; + }; + let supervised = supervised_shared_routes(sandboxes); + let orphans = sessions + .into_iter() + .filter(|session| !supervised.contains(&session.route)); + // Removals hold session-control connections while workloads stop. + futures::stream::iter(orphans) + .for_each_concurrent(SHARED_SESSION_GC_CONCURRENCY, |session| async move { + info!( + sandbox_id = session.sandbox_id, + "removing shared supervisor session that no sandbox generation supervises" + ); + if let Err(error) = shared.remove(&session.route).await { + warn!(sandbox_id = session.sandbox_id, %error, "could not remove unsupervised shared session"); + } + }) + .await; + } + async fn reconcile_sandbox_runtime_suspending( &self, lookup_api: &AgentSandboxApi, @@ -4257,6 +4752,10 @@ impl KubernetesComputeDriver { return; } } + // Completion forgets the placement, so end a shared session now. + if let Ok(placement) = supervisor_placement(object) { + self.remove_shared_session(&placement).await; + } if let Err(error) = self .delete_sandbox_runtime_supervisor( sandbox_id, @@ -4302,8 +4801,9 @@ impl KubernetesComputeDriver { namespace: &str, cr_name: &str, ) { + let placement = placement_for_release(object, sandbox_id); if let Err(error) = self - .delete_sandbox_runtime_supervisor_pod(sandbox_id, namespace) + .release_generation_supervisor(sandbox_id, namespace, &placement) .await { warn!(sandbox_id, %error, "could not release sandbox-runtime control; reconciliation will retry"); @@ -4530,13 +5030,18 @@ impl KubernetesComputeDriver { sandbox_id: &str, namespace: &str, desired_running: bool, - generations: &[&str], + object: &DynamicObject, ) -> Result<(), KubernetesDriverError> { + let generations = sandbox_runtime_generation(object); if !desired_running { return self - .delete_sandbox_runtime_supervisor(sandbox_id, namespace, generations) + .delete_sandbox_runtime_supervisor(sandbox_id, namespace, generations.as_slice()) .await; } + if let SupervisorPlacement::Shared { .. } = supervisor_placement(object)? { + // Session availability is part of control availability. + return Ok(()); + } // The boundary Service was checked by verify_sandbox_runtime_control_path. let names = SandboxRuntimeNames::new(sandbox_id); let pods = Api::::namespaced(self.client.clone(), namespace); @@ -4622,6 +5127,7 @@ impl KubernetesComputeDriver { let (tx, rx) = mpsc::channel(256); self.spawn_sandbox_runtime_periodic_reconcile(tx.clone()); let readiness_client = self.watch_client.clone(); + let readiness_shared = self.shared_supervisor.clone(); tokio::spawn(async move { let mut sandbox_name_to_id = std::collections::HashMap::::new(); @@ -4631,7 +5137,7 @@ impl KubernetesComputeDriver { tokio::select! { event = sandbox_stream.next() => match event { Some(Event::Apply(obj) | Event::InitApply(obj)) => { - if let Ok((kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, &namespace, obj).await { + if let Ok((kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, readiness_shared.as_ref(), &namespace, obj).await { update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( @@ -4725,6 +5231,29 @@ impl KubernetesComputeDriver { } } +/// Placement to release before a stop. An unreadable record must not block +/// suspending the workload, which ends any supervision on its own. +fn placement_for_release(object: &DynamicObject, sandbox_id: &str) -> SupervisorPlacement { + supervisor_placement(object).unwrap_or_else(|error| { + warn!(sandbox_id, %error, "invalid sandbox-runtime supervisor placement; stopping without releasing a session"); + SupervisorPlacement::Dedicated + }) +} + +/// Session routes of the shared generations that may currently run. +fn supervised_shared_routes(sandboxes: &[DynamicObject]) -> HashSet { + sandboxes + .iter() + .filter(|object| { + sandbox_runtime_should_run(object) || sandbox_runtime_bootstrap_in_progress(object) + }) + .filter_map(|object| match supervisor_placement(object) { + Ok(SupervisorPlacement::Shared { route }) => Some(route), + _ => None, + }) + .collect() +} + fn sandbox_runtime_bootstrap_completion_patch(resource_version: &str) -> serde_json::Value { serde_json::json!({ "metadata": { @@ -4741,33 +5270,126 @@ fn sandbox_runtime_bootstrap_completion_patch(resource_version: &str) -> serde_j }) } +/// Trusted launch material for one shared-supervisor session, taken from the +/// generation's own Sandbox resource and gateway-issued credentials. +fn shared_session_registration( + object: &DynamicObject, + route: &str, + sandbox_id: &str, + runtime_descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor, + launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, + proxy_ca: crate::sandbox_runtime::ProxyCaMaterial, +) -> Result +{ + use openshell_sandbox_backend::session_control::{ + ProxyCaMaterial, RegistrationTraceContext, SecretPem, SessionRegistration, + }; + let trace = openshell_otel::current_trace_context_environment(); + let trace_value = |name: &str| { + trace + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.clone()) + }; + let registration = SessionRegistration { + route: route.to_string(), + sandbox_id: sandbox_id.to_string(), + sandbox_name: annotation_or_label(object, LABEL_SANDBOX_NAME).ok_or_else(|| { + KubernetesDriverError::Precondition( + "Sandbox resource is missing its OpenShell sandbox name".to_string(), + ) + })?, + runtime_descriptor, + auth: launch_authentication.supervisor.clone(), + main_process_spec: required_sandbox_annotation( + object, + ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC, + )?, + workdir: Some(WORKSPACE_MOUNT_PATH.to_string()), + container_image: object + .data + .pointer("/spec/podTemplate/spec/containers") + .and_then(serde_json::Value::as_array) + .and_then(|containers| { + containers.iter().find(|container| { + container.get("name").and_then(serde_json::Value::as_str) == Some("agent") + }) + }) + .and_then(|container| container.get("image")) + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string(), + proxy_ca: ProxyCaMaterial { + certificate_pem: proxy_ca.certificate_pem, + private_key_pem: SecretPem::new(proxy_ca.private_key_pem), + }, + trace_context: trace_value(openshell_otel::TRACEPARENT_ENV).map(|traceparent| { + RegistrationTraceContext { + traceparent, + tracestate: trace_value(openshell_otel::TRACESTATE_ENV), + } + }), + }; + registration + .validate() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + Ok(registration) +} + +/// What bootstrap completion waits for before the runtime is declared ready. +enum BootstrapControl { + /// The generation's dedicated supervisor Pod becomes Ready. + SupervisorPod { pods: Api, name: String }, + /// The gateway accepts the generation's shared-supervisor session. + SharedSession { + shared: SharedSupervisor, + route: String, + }, +} + +impl BootstrapControl { + async fn wait_available(self) -> bool { + match self { + Self::SupervisorPod { pods, name } => { + let available = |pod: Option<&Pod>| { + pod.is_some_and(|pod| { + sandbox_runtime_control_availability_from_pod(pod) + == SandboxRuntimeControlAvailability::Available + }) + }; + match await_condition(pods, &name, available).await { + Ok(Some(_)) => true, + Ok(None) => false, + Err(error) => { + debug!(%error, pod = name, "sandbox-runtime bootstrap availability watch failed; reconciliation will retry"); + false + } + } + } + Self::SharedSession { shared, route } => loop { + match shared.status(&route).await { + Ok(Some(status)) if status.ready => return true, + Ok(Some(status)) if status.phase.is_terminal() => return false, + Ok(None) => return false, + Ok(Some(_)) | Err(_) => {} + } + tokio::time::sleep(Duration::from_secs(1)).await; + }, + } + } +} + fn spawn_sandbox_runtime_bootstrap_completion( - pods: Api, + control: BootstrapControl, sandboxes: Api, - supervisor_pod_name: String, sandbox_name: String, expected_sandbox_uid: Option, ) { tokio::spawn(async move { let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE; - let available = |pod: Option<&Pod>| { - pod.is_some_and(|pod| { - sandbox_runtime_control_availability_from_pod(pod) - == SandboxRuntimeControlAvailability::Available - }) - }; - match tokio::time::timeout_at( - deadline, - await_condition(pods, &supervisor_pod_name, available), - ) - .await - { - Ok(Ok(Some(_))) => {} - Ok(Ok(None)) | Err(_) => return, - Ok(Err(error)) => { - debug!(%error, sandbox = sandbox_name, "sandbox-runtime bootstrap availability watch failed; reconciliation will retry"); - return; - } + match tokio::time::timeout_at(deadline, control.wait_available()).await { + Ok(true) => {} + Ok(false) | Err(_) => return, } // A replacement control can become Available before the Agent @@ -4944,6 +5566,7 @@ where { let (tx, rx) = mpsc::channel(256); driver.spawn_sandbox_runtime_periodic_reconcile(tx.clone()); + let readiness_shared = driver.shared_supervisor; tokio::spawn(async move { loop { @@ -4952,7 +5575,7 @@ where Some(Event::Apply(obj) | Event::InitApply(obj)) => { let ns = obj.metadata.namespace.clone() .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, &ns, obj).await { + if let Ok((_kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, readiness_shared.as_ref(), &ns, obj).await { let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } @@ -5697,6 +6320,92 @@ async fn sandbox_runtime_control_availability( combine_sandbox_runtime_control_availability([control, service, fence, supervisor_fence]) } +/// Availability of a generation's supervision, by its recorded placement. +async fn generation_control_availability( + client: &Client, + shared: Option<&SharedSupervisor>, + namespace: &str, + sandbox_id: &str, + object: &DynamicObject, +) -> SandboxRuntimeControlAvailability { + match supervisor_placement(object) { + Ok(SupervisorPlacement::Dedicated) => { + sandbox_runtime_control_availability(client, namespace, sandbox_id).await + } + Ok(SupervisorPlacement::Shared { route }) => { + shared_session_control_availability(client, shared, namespace, sandbox_id, &route).await + } + Err(error) => { + warn!(sandbox_id, %error, "sandbox-runtime supervisor placement is invalid"); + SandboxRuntimeControlAvailability::Unavailable + } + } +} + +/// A shared generation is supervised while its session is registered for +/// this sandbox and accepted by the gateway, and its fence and egress +/// allowance exist. +async fn shared_session_control_availability( + client: &Client, + shared: Option<&SharedSupervisor>, + namespace: &str, + sandbox_id: &str, + route: &str, +) -> SandboxRuntimeControlAvailability { + let Some(shared) = shared else { + warn!( + sandbox_id, + "the gateway supervisor topology is not configured for a shared generation" + ); + return SandboxRuntimeControlAvailability::Unavailable; + }; + let names = SandboxRuntimeNames::new(sandbox_id); + let policies = Api::::namespaced(client.clone(), namespace); + let exists = |result: Result, KubeError>, _>, what: &str| { + match result { + Ok(Ok(Some(_))) => SandboxRuntimeControlAvailability::Available, + Ok(Ok(None)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine sandbox-runtime {what} availability"); + SandboxRuntimeControlAvailability::Unknown + } + Err(tokio::time::error::Elapsed { .. }) => { + warn!( + sandbox_id, + "timed out checking sandbox-runtime {what} availability" + ); + SandboxRuntimeControlAvailability::Unknown + } + } + }; + let (session, fence, control) = tokio::join!( + shared.status(route), + tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(&names.workload_policy)), + tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.get_opt(&names.shared_control_policy) + ), + ); + let session = match session { + Ok(Some(status)) if status.sandbox_id != sandbox_id || status.phase.is_terminal() => { + SandboxRuntimeControlAvailability::Unavailable + } + Ok(Some(status)) if status.ready => SandboxRuntimeControlAvailability::Available, + Ok(Some(_)) => SandboxRuntimeControlAvailability::Degraded, + Ok(None) => SandboxRuntimeControlAvailability::Unavailable, + Err(error) => { + warn!(sandbox_id, %error, "could not determine shared supervisor session availability"); + SandboxRuntimeControlAvailability::Unknown + } + }; + combine_sandbox_runtime_control_availability([ + session, + exists(fence, "workload fence"), + exists(control, "shared control policy"), + SandboxRuntimeControlAvailability::Available, + ]) +} + fn combine_sandbox_runtime_control_availability( dependencies: [SandboxRuntimeControlAvailability; 4], ) -> SandboxRuntimeControlAvailability { @@ -5762,6 +6471,14 @@ async fn sandbox_runtime_supervisor_generation_matches( sandbox_id: &str, sandbox: &DynamicObject, ) -> SandboxRuntimeControlAvailability { + match supervisor_placement(sandbox) { + Ok(SupervisorPlacement::Dedicated) => {} + // The session's sandbox is checked with its availability. + Ok(SupervisorPlacement::Shared { .. }) => { + return SandboxRuntimeControlAvailability::Available; + } + Err(_) => return SandboxRuntimeControlAvailability::Unavailable, + } let Some(expected_uid) = sandbox .metadata .annotations @@ -5793,6 +6510,7 @@ async fn sandbox_runtime_supervisor_generation_matches( async fn sandbox_from_object_with_sandbox_runtime_readiness( client: &Client, + shared: Option<&SharedSupervisor>, namespace: &str, obj: DynamicObject, ) -> Result<(String, Sandbox), String> { @@ -5808,10 +6526,12 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness( mark_sandbox_runtime_bootstrapping(&mut sandbox); } if !sandbox_id.is_empty() { - let dependencies = Box::pin(sandbox_runtime_control_availability( + let dependencies = Box::pin(generation_control_availability( client, + shared, &object_namespace, &sandbox_id, + &obj, )); let workload_generation = Box::pin(sandbox_runtime_workload_generation_matches( client, @@ -7508,6 +8228,8 @@ fn sandbox_runtime_suspension_completion_patch(resource_version: &str) -> serde_ ANNOTATION_SANDBOX_RUNTIME_GENERATION: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: serde_json::Value::Null, + ANNOTATION_SUPERVISOR_TOPOLOGY: serde_json::Value::Null, + ANNOTATION_SUPERVISOR_SESSION: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", @@ -8083,6 +8805,7 @@ mod tests { lifecycle_gates: Arc::default(), config: KubernetesComputeConfig::default(), operator_allowlist: None, + shared_supervisor: None, }; driver .sandbox_api_version @@ -8663,6 +9386,62 @@ mod tests { sandbox } + #[test] + fn only_runnable_shared_generations_keep_their_sessions() { + let shared = |route: &str, operating_mode: &str, annotations: &[(&str, &str)]| { + let mut object = sandbox_object_for_test(route, route); + object.data = serde_json::json!({"spec": {"operatingMode": operating_mode}}); + let mut recorded = BTreeMap::from([ + ( + ANNOTATION_SUPERVISOR_TOPOLOGY.to_string(), + GATEWAY_TOPOLOGY.to_string(), + ), + (ANNOTATION_SUPERVISOR_SESSION.to_string(), route.to_string()), + ]); + recorded.extend( + annotations + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())), + ); + object.metadata.annotations = Some(recorded); + object + }; + let running = "rt-running-route-0001"; + let stopping = "rt-stopping-route-0001"; + let stopped = "rt-stopped-route-00001"; + let mut dedicated = sandbox_object_for_test("dedicated", "dedicated"); + dedicated.data = serde_json::json!({"spec": {"operatingMode": "Running"}}); + let routes = supervised_shared_routes(&[ + shared(running, "Running", &[]), + // A stop in progress still ends its session itself. + shared( + stopping, + "Suspended", + &[(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING, "true")], + ), + shared(stopped, "Suspended", &[]), + dedicated, + ]); + assert_eq!( + routes, + HashSet::from([running.to_string(), stopping.to_string()]) + ); + } + + #[test] + fn suspension_completion_forgets_the_generation_placement() { + let patch = sandbox_runtime_suspension_completion_patch("7"); + let annotations = &patch["metadata"]["annotations"]; + assert!(annotations[ANNOTATION_SUPERVISOR_TOPOLOGY].is_null()); + assert!(annotations[ANNOTATION_SUPERVISOR_SESSION].is_null()); + assert!( + annotations + .as_object() + .unwrap() + .contains_key(ANNOTATION_SUPERVISOR_SESSION) + ); + } + #[test] fn restart_runtime_selection_preserves_namespace_and_resource_uid() { let expected = parse_kubernetes_runtime_identity( @@ -9104,6 +9883,7 @@ mod tests { lifecycle_gates: Arc::default(), config: KubernetesComputeConfig::default(), operator_allowlist: None, + shared_supervisor: None, }; driver .sandbox_api_version @@ -9195,6 +9975,7 @@ mod tests { lifecycle_gates: Arc::default(), config: KubernetesComputeConfig::default(), operator_allowlist: None, + shared_supervisor: None, }; driver @@ -11934,6 +12715,7 @@ mod tests { lifecycle_gates: Arc::default(), config, operator_allowlist: None, + shared_supervisor: None, }; (driver, steps, bodies) } @@ -12238,6 +13020,7 @@ mod tests { lifecycle_gates: Arc::default(), config: managed_config(&[]), operator_allowlist: None, + shared_supervisor: None, }; let service = crate::ComputeDriverService::new(driver); diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs index 45ea8a5baa..76fa455496 100644 --- a/crates/openshell-driver-kubernetes/src/isolation.rs +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -53,6 +53,10 @@ struct KubernetesControlPolicyEvidence<'a> { network_policy_uid: &'a str, network_policy_resource_version: &'a str, port: u16, + /// Namespace of a supervisor shared by sandboxes. Omitted for a + /// dedicated supervisor so its evidence encoding is unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + supervisor_namespace: Option<&'a str>, } impl KubernetesControlPolicyEvidence<'_> { @@ -281,6 +285,68 @@ pub fn runtime_control_policy(namespace: &str, name: &str, pair: &str, port: u16 } } +/// Per-sandbox allowance for runtime-initiated control to a shared supervisor. +/// +/// It selects only this pair's workload Pod and admits one egress flow, to the +/// shared supervisor Pods in their namespace on the runtime port. The +/// namespace fence keeps denying every other workload connection. Which +/// session a connection reaches is decided by its routing preface and pinned +/// TLS identity, not by this policy. +#[must_use] +pub fn shared_runtime_control_policy( + namespace: &str, + name: &str, + pair: &str, + supervisor_namespace: &str, + supervisor_selector: BTreeMap, + port: u16, +) -> NetworkPolicy { + NetworkPolicy { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(pair_role_labels(pair, WORKLOAD_ROLE)), + ..Default::default() + }, + policy_types: Some(vec!["Egress".to_string()]), + egress: Some(vec![NetworkPolicyEgressRule { + to: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + supervisor_namespace.to_string(), + )])), + ..Default::default() + }), + pod_selector: Some(LabelSelector { + match_labels: Some(supervisor_selector), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(i32::from(port))), + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + }]), + ..Default::default() + }), + } +} + +/// The supervisor bound into a generation's resource claims. +pub enum KubernetesSupervisorClaim { + /// UID of the generation's dedicated supervisor Pod. + Pod(String), + /// Route of the generation's session on the shared supervisor. + Session(String), +} + /// Which side opens the Sandbox Protocol connection, with the verified /// Kubernetes coordinates that mode needs. pub enum KubernetesControlPath { @@ -301,6 +367,18 @@ pub enum KubernetesControlPath { control_policy_uid: String, control_policy_resource_version: String, }, + /// The workload runtime dials the gateway's shared supervisor and names + /// this generation's route before TLS. + RuntimeInitiatedShared { + control_authority: String, + /// Shared Service address the runtime dials. + supervisor_address: SocketAddr, + supervisor_namespace: String, + route: String, + /// Verified per-sandbox shared control policy coordinates. + control_policy_uid: String, + control_policy_resource_version: String, + }, } /// Driver-owned inputs that bind one workload/supervisor pair to one boundary. @@ -321,7 +399,7 @@ pub struct KubernetesSandboxRuntimeBoundarySpec { pub sandbox_resource_uid: String, pub workload_pod_uid: String, pub workload_pod_uid_path: PathBuf, - pub supervisor_pod_uid: String, + pub supervisor: KubernetesSupervisorClaim, pub egress_policy_uid: String, pub egress_policy_resource_version: String, pub control_path: KubernetesControlPath, @@ -352,10 +430,14 @@ impl KubernetesSandboxRuntimeBoundarySpec { "kubernetes.workload_pod_uid".to_string(), self.workload_pod_uid, ), - ( - "kubernetes.supervisor_pod_uid".to_string(), - self.supervisor_pod_uid, - ), + match self.supervisor { + KubernetesSupervisorClaim::Pod(uid) => { + ("kubernetes.supervisor_pod_uid".to_string(), uid) + } + KubernetesSupervisorClaim::Session(route) => { + ("kubernetes.supervisor_session".to_string(), route) + } + }, ( "kubernetes.egress_policy_uid".to_string(), self.egress_policy_uid, @@ -365,6 +447,8 @@ impl KubernetesSandboxRuntimeBoundarySpec { self.egress_policy_resource_version, ), ]); + let mut supervisor_namespace = None; + let mut shared_port = None; let (listener, transport) = match self.control_path { KubernetesControlPath::SupervisorInitiated { boundary_listener, @@ -408,11 +492,43 @@ impl KubernetesSandboxRuntimeBoundarySpec { }, ) } + KubernetesControlPath::RuntimeInitiatedShared { + control_authority, + supervisor_address, + supervisor_namespace: shared_namespace, + route, + control_policy_uid, + control_policy_resource_version, + } => { + resource_claims.insert( + "kubernetes.control_policy_uid".to_string(), + control_policy_uid, + ); + resource_claims.insert( + "kubernetes.control_policy_resource_version".to_string(), + control_policy_resource_version, + ); + supervisor_namespace = Some(shared_namespace); + shared_port = Some(supervisor_address.port()); + ( + BoundaryListener::RuntimeInitiatedTlsTcp { + authority: control_authority.clone(), + addresses: vec![supervisor_address], + tls: self.sandbox_tls, + route: Some(route.clone()), + }, + SandboxTransport::RuntimeInitiatedRouted { + authority: control_authority, + route, + }, + ) + } }; let control_port = match &transport { SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } => { Some(listen_address.port()) } + SandboxTransport::RuntimeInitiatedRouted { .. } => shared_port, _ => None, }; let provisioning_evidence = KubernetesOuterFenceEvidence { @@ -427,6 +543,7 @@ impl KubernetesSandboxRuntimeBoundarySpec { network_policy_resource_version: &resource_claims ["kubernetes.control_policy_resource_version"], port, + supervisor_namespace: supervisor_namespace.as_deref(), }), } .project(&self.generation)?; @@ -532,7 +649,7 @@ mod tests { sandbox_resource_uid: "sandbox-resource-uid".to_string(), workload_pod_uid: "pod-uid".to_string(), workload_pod_uid_path: PathBuf::from("/.openshell/pod-identity/uid"), - supervisor_pod_uid: "supervisor-pod-uid".to_string(), + supervisor: KubernetesSupervisorClaim::Pod("supervisor-pod-uid".to_string()), egress_policy_uid: "network-policy-uid".to_string(), egress_policy_resource_version: "1945".to_string(), control_path: KubernetesControlPath::SupervisorInitiated { diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 963935abef..4d21fbff51 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -9,12 +9,13 @@ mod lifecycle; pub mod otel_tracing; mod resource_admission; mod sandbox_runtime; +mod shared_supervisor; pub use config::{ DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesControlConnection, KubernetesImagePullPolicy, - KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, SupervisorTopology, WorkspaceMode, - managed_namespace_prefix, + KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, SharedSupervisorConfig, + SupervisorTopology, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs b/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs index 16c8f82981..d8ec7791c0 100644 --- a/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs +++ b/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs @@ -36,6 +36,7 @@ mod lifecycle_reconciliation { lifecycle_gates: Arc::default(), config: KubernetesComputeConfig::default(), operator_allowlist: None, + shared_supervisor: None, }; (driver, steps, Arc::default()) } diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 30c7f899e7..209d286370 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -261,6 +261,7 @@ async fn main() -> Result<()> { let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { supervisor_topology: None, + shared_supervisor: None, allow_driver_config: args.admission_config_json.allow_driver_config, resource_admission: args.admission_config_json.resource_admission.clone(), workspace_mode: args.workspace_mode, diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index d191ca5201..94a5dd1c65 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -118,6 +118,8 @@ pub struct SandboxRuntimeNames { pub supervisor_service: String, /// Per-sandbox egress allowance used only by runtime-initiated control. pub control_policy: String, + /// Per-sandbox egress allowance to the gateway's shared supervisor. + pub shared_control_policy: String, pub supervisor_pod: String, pub workload_policy: String, pub supervisor_policy: String, @@ -134,6 +136,7 @@ impl SandboxRuntimeNames { boundary_service: format!("os-boundary-{suffix}"), supervisor_service: format!("os-supervisor-{suffix}"), control_policy: format!("os-control-{suffix}"), + shared_control_policy: format!("os-shared-control-{suffix}"), supervisor_pod: format!("os-supervisor-{suffix}"), workload_policy: NAMESPACE_WORKLOAD_POLICY_NAME.to_string(), supervisor_policy: NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME.to_string(), @@ -283,6 +286,30 @@ pub fn control_policy( policy } +/// Egress allowance from this sandbox's workload Pod to the shared supervisor. +#[must_use] +pub fn shared_control_policy( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + supervisor_namespace: &str, + supervisor_selector: BTreeMap, + runtime_port: u16, + owner: OwnerReference, +) -> NetworkPolicy { + let mut policy = crate::isolation::shared_runtime_control_policy( + namespace, + &names.shared_control_policy, + &pair_label_value(sandbox_id), + supervisor_namespace, + supervisor_selector, + runtime_port, + ); + policy.metadata.owner_references = Some(vec![owner]); + policy.metadata.labels = Some(common_labels(sandbox_id, "sandbox-shared-control-egress")); + policy +} + #[allow(clippy::too_many_arguments, clippy::similar_names)] pub fn supervisor_pod( namespace: &str, diff --git a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs new file mode 100644 index 0000000000..21df3cf20d --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs @@ -0,0 +1,296 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Generations supervised by the gateway's shared supervisor. +//! +//! Each runtime generation records where its supervisor runs when it starts. +//! Every later lifecycle decision reads that record, so changing the gateway +//! topology setting never reinterprets an existing generation. + +use std::time::Duration; + +use bytes::Bytes; +use http_body_util::{BodyExt as _, Full}; +use kube::core::DynamicObject; +use openshell_sandbox_backend::session_control::{ + SESSIONS_PATH, SessionList, SessionRegistration, SessionStatusReport, +}; + +use crate::config::SharedSupervisorConfig; +use crate::driver::KubernetesDriverError; + +/// Supervisor placement of the current generation; absent means a dedicated +/// supervisor Pod. +pub const ANNOTATION_SUPERVISOR_TOPOLOGY: &str = "openshell.ai/sandbox-runtime-supervisor-topology"; +/// Route of a generation's session on the shared supervisor. +pub const ANNOTATION_SUPERVISOR_SESSION: &str = "openshell.ai/sandbox-runtime-supervisor-session"; +pub const GATEWAY_TOPOLOGY: &str = "gateway"; + +/// Bound on one session-control request, including connection and TLS. +const SESSION_CONTROL_TIMEOUT: Duration = Duration::from_secs(15); + +/// Where a generation's supervisor runs, as recorded when it started. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SupervisorPlacement { + /// A supervisor Pod paired with this generation's workload Pod. + Dedicated, + /// A session on the gateway's shared supervisor. + Shared { route: String }, +} + +/// Read the recorded placement. An unknown or incomplete record fails closed. +pub fn supervisor_placement( + object: &DynamicObject, +) -> Result { + let annotations = object.metadata.annotations.as_ref(); + let annotation = |key: &str| annotations.and_then(|annotations| annotations.get(key)); + match annotation(ANNOTATION_SUPERVISOR_TOPOLOGY).map(String::as_str) { + None => Ok(SupervisorPlacement::Dedicated), + Some(GATEWAY_TOPOLOGY) => { + let route = annotation(ANNOTATION_SUPERVISOR_SESSION) + .filter(|route| { + openshell_sandbox_backend::runtime_routed::validate_route_handle(route).is_ok() + }) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "shared supervisor generation has no valid session route".to_string(), + ) + })?; + Ok(SupervisorPlacement::Shared { + route: route.clone(), + }) + } + Some(other) => Err(KubernetesDriverError::Precondition(format!( + "unknown sandbox-runtime supervisor topology {other:?}" + ))), + } +} + +/// Client for the shared supervisor's session-control API. It authenticates +/// with the gateway's admission certificate, which only Helm issues. +#[derive(Clone, Debug)] +pub struct SharedSupervisor { + config: SharedSupervisorConfig, +} + +impl SharedSupervisor { + pub fn new(config: SharedSupervisorConfig) -> Self { + Self { config } + } + + pub fn config(&self) -> &SharedSupervisorConfig { + &self.config + } + + /// Register a generation; an identical registration is accepted again. + pub async fn register( + &self, + registration: &SessionRegistration, + ) -> Result { + let body = serde_json::to_vec(registration).map_err(|error| { + KubernetesDriverError::Message(format!("encode session registration: {error}")) + })?; + let (status, body) = self + .request( + http::Method::PUT, + &format!("{SESSIONS_PATH}/{}", registration.route), + Some(body), + ) + .await?; + match status { + http::StatusCode::CREATED | http::StatusCode::OK => decode(&body), + http::StatusCode::CONFLICT => Err(KubernetesDriverError::Precondition( + "shared supervisor already hosts this route with different launch material" + .to_string(), + )), + status => Err(rejected("register session", status, &body)), + } + } + + /// Status of one session, or `None` when it is not registered. + pub async fn status( + &self, + route: &str, + ) -> Result, KubernetesDriverError> { + let (status, body) = self + .request(http::Method::GET, &format!("{SESSIONS_PATH}/{route}"), None) + .await?; + match status { + http::StatusCode::OK => decode(&body).map(Some), + http::StatusCode::NOT_FOUND => Ok(None), + status => Err(rejected("read session status", status, &body)), + } + } + + /// Stop and forget one session. A missing session is already removed. + pub async fn remove(&self, route: &str) -> Result<(), KubernetesDriverError> { + let (status, body) = self + .request( + http::Method::DELETE, + &format!("{SESSIONS_PATH}/{route}"), + None, + ) + .await?; + match status { + http::StatusCode::OK | http::StatusCode::NOT_FOUND => Ok(()), + status => Err(rejected("remove session", status, &body)), + } + } + + /// Every session the shared supervisor hosts. + pub async fn list(&self) -> Result { + let (status, body) = self.request(http::Method::GET, SESSIONS_PATH, None).await?; + match status { + http::StatusCode::OK => decode(&body), + status => Err(rejected("list sessions", status, &body)), + } + } + + async fn request( + &self, + method: http::Method, + path: &str, + body: Option>, + ) -> Result<(http::StatusCode, Bytes), KubernetesDriverError> { + tokio::time::timeout( + SESSION_CONTROL_TIMEOUT, + self.request_once(method, path, body), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out calling the shared supervisor session-control API".to_string(), + ) + })? + } + + async fn request_once( + &self, + method: http::Method, + path: &str, + body: Option>, + ) -> Result<(http::StatusCode, Bytes), KubernetesDriverError> { + let unavailable = |what: &str, error: &dyn std::fmt::Display| { + KubernetesDriverError::Message(format!("shared supervisor {what}: {error}")) + }; + // Read the admission material per request so rotated Secrets apply. + let directory = std::path::Path::new(&self.config.control_tls_dir); + let read = |name: &str| { + std::fs::read(directory.join(name)) + .map_err(|error| unavailable(&format!("session-control TLS file {name}"), &error)) + }; + let tls = openshell_sandbox_backend::session_control::client_tls_config( + &read("tls.crt")?, + &read("tls.key")?, + &read("ca.crt")?, + ) + .map_err(|error| unavailable("session-control TLS", &error))?; + let host = self.config.service_host(); + let server_name = rustls::pki_types::ServerName::try_from(host.clone()) + .map_err(|error| unavailable("session-control server name", &error))?; + let addresses = tokio::net::lookup_host((host.as_str(), self.config.control_port)) + .await + .map_err(|error| unavailable("resolve", &error))? + .collect::>(); + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&addresses) + .await + .map_err(|error| unavailable("connect", &error))?; + let stream = tokio_rustls::TlsConnector::from(tls) + .connect(server_name, stream) + .await + .map_err(|error| unavailable("TLS", &error))?; + let (mut sender, connection) = + hyper::client::conn::http1::handshake(hyper_util::rt::TokioIo::new(stream)) + .await + .map_err(|error| unavailable("HTTP handshake", &error))?; + tokio::spawn(connection); + let request = http::Request::builder() + .method(method) + .uri(path) + .header(http::header::HOST, host) + .header(http::header::CONTENT_TYPE, "application/json") + .body(Full::new(Bytes::from(body.unwrap_or_default()))) + .map_err(|error| unavailable("request", &error))?; + let response = sender + .send_request(request) + .await + .map_err(|error| unavailable("request", &error))?; + let status = response.status(); + let body = response + .into_body() + .collect() + .await + .map_err(|error| unavailable("response", &error))? + .to_bytes(); + Ok((status, body)) + } +} + +fn decode(body: &[u8]) -> Result { + serde_json::from_slice(body).map_err(|error| { + KubernetesDriverError::Message(format!("decode shared supervisor response: {error}")) + }) +} + +fn rejected(what: &str, status: http::StatusCode, body: &[u8]) -> KubernetesDriverError { + let detail = serde_json::from_slice::(body) + .ok() + .and_then(|value| { + value + .get("error") + .and_then(|error| error.as_str().map(str::to_string)) + }) + .unwrap_or_default(); + KubernetesDriverError::Message(format!( + "shared supervisor rejected {what} ({status}): {detail}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use kube::core::{ApiResource, GroupVersionKind}; + + fn object(annotations: &[(&str, &str)]) -> DynamicObject { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + "agents.x-k8s.io", + "v1beta1", + "Sandbox", + )); + let mut object = DynamicObject::new("sandbox-a", &resource); + object.metadata.annotations = Some( + annotations + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(), + ); + object + } + + #[test] + fn recorded_placement_is_read_and_unknown_records_fail_closed() { + assert_eq!( + supervisor_placement(&object(&[])).unwrap(), + SupervisorPlacement::Dedicated + ); + let route = openshell_sandbox_backend::runtime_routed::new_route_handle(); + assert_eq!( + supervisor_placement(&object(&[ + (ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY), + (ANNOTATION_SUPERVISOR_SESSION, &route), + ])) + .unwrap(), + SupervisorPlacement::Shared { route } + ); + for annotations in [ + &[(ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY)][..], + &[ + (ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY), + (ANNOTATION_SUPERVISOR_SESSION, "Not/A/Route"), + ], + &[(ANNOTATION_SUPERVISOR_TOPOLOGY, "workspace")], + ] { + assert!(supervisor_placement(&object(annotations)).is_err()); + } + } +} From f4400e0a82059dbd242f00b6498fcafcd4ec45db Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 18:28:41 -0700 Subject: [PATCH 05/25] feat(helm): deploy and wire the shared supervisor With the gateway topology enabled, the chart runs the session host as a single-replica Deployment with its Service and a NetworkPolicy that admits session control only from the gateway and runtime connections only from sandbox workload Pods. The certgen hook issues a private session-control PKI whose client credential only the gateway mounts, the gateway may read only that Service, and the chart renders the matching experimental_shared_supervisor table. The supervisor Pod has no service account token. Settings the shared supervisor cannot isolate, and a hand-written connection table, fail at render time. Signed-off-by: Drew Newberry --- .../openshell/templates/_gateway-workload.tpl | 11 + deploy/helm/openshell/templates/_helpers.tpl | 60 ++++++ deploy/helm/openshell/templates/certgen.yaml | 27 ++- .../helm/openshell/templates/peer-role.yaml | 12 ++ .../helm/openshell/templates/supervisor.yaml | 169 +++++++++++++-- .../helm/openshell/tests/supervisor_test.yaml | 200 ++++++++++++++++-- .../tests/supervisor_wiring_test.yaml | 83 ++++++++ 7 files changed, 517 insertions(+), 45 deletions(-) create mode 100644 deploy/helm/openshell/tests/supervisor_wiring_test.yaml diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 12120a89ce..c1c2d511d8 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -196,6 +196,11 @@ spec: mountPath: {{ dir $spiffeSocketPath | quote }} readOnly: true {{- end }} + {{- if include "openshell.supervisorEnabled" . }} + - name: supervisor-control + mountPath: /etc/openshell-tls/supervisor-control + readOnly: true + {{- end }} {{- with .Values.server.extraVolumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} @@ -290,6 +295,12 @@ spec: - key: ca.crt path: ca.crt {{- end }} + {{- if include "openshell.supervisorEnabled" . }} + # Admission credential for the shared supervisor's session-control API. + - name: supervisor-control + secret: + secretName: {{ include "openshell.supervisorName" . }}-control-client + {{- end }} {{- if .Values.upstreamProxy.caBundle.configMapName }} - name: upstream-proxy-ca configMap: diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 2421463d3d..8292b2a71e 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -353,6 +353,53 @@ the in-cluster Service DNS, release namespace, service port, and disableTls flag — so the default value works for any release name or namespace without override. */}} +{{/* +Whether the gateway supervisor topology is enabled. Read from the +raw gateway configuration so openshell.effectiveKubernetesConfig can use it. +*/}} +{{- define "openshell.supervisorEnabled" -}} +{{- $kubernetes := get (.Values.gatewayConfig | default dict) "openshell.drivers.kubernetes" | default dict -}} +{{- if eq (get $kubernetes "supervisor_topology" | default "") "gateway" -}}true{{- end -}} +{{- end }} + +{{/* +Name of the shared supervisor Deployment, Service, and session-control Secrets. +*/}} +{{- define "openshell.supervisorName" -}} +{{- printf "%s-supervisor" (include "openshell.fullname" .) | trunc 52 | trimSuffix "-" -}} +{{- end }} + +{{/* +Pod labels of the shared supervisor; its Service selects exactly these. +*/}} +{{- define "openshell.supervisorSelectorLabels" -}} +app.kubernetes.io/name: openshell-supervisor +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: supervisor +{{- end }} + +{{/* +Namespace selector for the sandbox namespaces of this gateway's workspace +mode: the shared namespace, the managed namespaces it labels, or the +operator-labeled namespaces. +*/}} +{{- define "openshell.supervisorSandboxNamespaces" -}} +{{- $kubernetes := .kubernetes -}} +{{- $mode := get $kubernetes "workspace_mode" | default "shared" -}} +{{- if eq $mode "managed" -}} +matchLabels: + openshell.ai/managed-by: openshell + openshell.ai/gateway-id: {{ get $kubernetes "gateway_id" | quote }} +{{- else if eq $mode "operator" -}} +{{- $label := regexSplit "==?" (get $kubernetes "operator_namespace_label") 2 -}} +matchLabels: + {{ index $label 0 }}: {{ index $label 1 | quote }} +{{- else -}} +matchLabels: + kubernetes.io/metadata.name: {{ get $kubernetes "namespace" | quote }} +{{- end -}} +{{- end }} + {{- define "openshell.grpcEndpoint" -}} {{- if .Values.server.grpcEndpoint -}} {{- .Values.server.grpcEndpoint -}} @@ -561,6 +608,10 @@ like user-supplied schema-v2 fields during compatibility resolution. */}} {{- if not (hasKey $kubernetes "supervisor_image") -}}{{- $_ := set $kubernetes "supervisor_image" (include "openshell.supervisorImage" .) -}}{{- end -}} {{- if not (hasKey $runtimeConfig "boundary_port") -}}{{- $_ := set $runtimeConfig "boundary_port" 5500 -}}{{- end -}} {{- $_ := set $kubernetes "sandbox_runtime" $runtimeConfig -}} +{{/* The chart owns the shared supervisor and its session-control credential. */}} +{{- if include "openshell.supervisorEnabled" . -}} +{{- $_ := set $kubernetes "shared_supervisor" (dict "service_name" (include "openshell.supervisorName" .) "service_namespace" .Release.Namespace "runtime_port" 5500 "control_port" 5501 "control_tls_dir" "/etc/openshell-tls/supervisor-control") -}} +{{- end -}} {{- toYaml $kubernetes -}} {{- end }} @@ -581,6 +632,15 @@ Validate chart values that Helm would otherwise accept silently. {{- if ne (get $gateway "compute_driver" | default "kubernetes") "kubernetes" -}} {{- fail "supervisor_topology requires the kubernetes compute driver" -}} {{- end -}} +{{- if or (get $kubernetes "https_proxy") (get $kubernetes "proxy_ca_bundle") (get $kubernetes "proxy_auth_secret_name") (get $kubernetes "provider_spiffe_workload_api_socket_path") -}} +{{- fail "supervisor_topology=gateway does not support upstream proxy settings or provider SPIFFE identity" -}} +{{- end -}} +{{- if and (eq (get $kubernetes "workspace_mode" | default "shared") "operator") (not (regexMatch "^[A-Za-z0-9][-A-Za-z0-9_./]*==?[-A-Za-z0-9_.]*$" (get $kubernetes "operator_namespace_label" | default ""))) -}} +{{- fail "supervisor_topology=gateway in operator workspace mode requires operator_namespace_label of the form key=value" -}} +{{- end -}} +{{- end -}} +{{- if hasKey (get (.Values.gatewayConfig | default dict) "openshell.drivers.kubernetes" | default dict) "shared_supervisor" -}} +{{- fail "shared_supervisor is rendered by the chart; set only supervisor_topology" -}} {{- end -}} {{- $workloadKind := include "openshell.workloadKind" . -}} {{- $workload := .Values.workload | default dict -}} diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index aea3b8d2cc..d4b5a7aa81 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} +{{- $supervisor := include "openshell.supervisorEnabled" . }} +{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled $supervisor }} {{- $hookName := printf "%s-certgen" (include "openshell.fullname" .) }} {{- $ns := .Release.Namespace }} apiVersion: v1 @@ -84,6 +85,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} containers: + {{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} - name: certgen image: {{ include "openshell.image" . | quote }} imagePullPolicy: {{ .Values.gateway.image.pullPolicy | default .Values.global.image.pullPolicy }} @@ -119,6 +121,29 @@ spec: {{- if and .Values.grpcRoute.backendTLSPolicy.enabled (not .Values.certManager.enabled) }} - --backend-ca-configmap-name={{ include "openshell.backendCaConfigMapName" . }} {{- end }} + {{- end }} + {{- if $supervisor }} + {{- $supervisorName := include "openshell.supervisorName" . }} + # Private PKI for the shared supervisor's session-control API. Only + # the gateway receives its client credential. + - name: supervisor-certgen + image: {{ include "openshell.image" . | quote }} + imagePullPolicy: {{ .Values.gateway.image.pullPolicy | default .Values.global.image.pullPolicy }} + securityContext: + {{- toYaml .Values.pkiInitJob.securityContext | nindent 12 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: ["/usr/local/bin/openshell-gateway"] + args: + - generate-certs + - --tls-only + - --server-secret-name={{ $supervisorName }}-control + - --client-secret-name={{ $supervisorName }}-control-client + - --server-san={{ $supervisorName }}.{{ $ns }}.svc + {{- end }} {{- if and .Values.certManager.enabled .Values.grpcRoute.backendTLSPolicy.enabled }} --- # Post-install Job that polls for the cert-manager-issued server Secret and diff --git a/deploy/helm/openshell/templates/peer-role.yaml b/deploy/helm/openshell/templates/peer-role.yaml index 3aa2502f46..38a640cd2a 100644 --- a/deploy/helm/openshell/templates/peer-role.yaml +++ b/deploy/helm/openshell/templates/peer-role.yaml @@ -18,6 +18,18 @@ rules: - pods verbs: - get + {{- if include "openshell.supervisorEnabled" . }} + # The Kubernetes driver reads the shared supervisor Service to scope each + # sandbox's control egress to the supervisor Pods it selects. + - apiGroups: + - "" + resources: + - services + resourceNames: + - {{ include "openshell.supervisorName" . }} + verbs: + - get + {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index c31fbb610a..d556c6756f 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -4,34 +4,42 @@ {{- range .Values.imagePullSecrets -}} {{- $imagePullSecrets = append $imagePullSecrets .name -}} {{- end -}} -{{- if eq (get $kubernetes "supervisor_topology" | default "") "gateway" }} +{{- if include "openshell.supervisorEnabled" . }} # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $shared := get $kubernetes "shared_supervisor" }} +{{- $name := include "openshell.supervisorName" . }} +{{- $endpoint := get $kubernetes "grpc_endpoint" }} +{{- $endpointUrl := urlParse $endpoint }} +{{- $gatewayTls := and (not .Values.server.disableTls) .Values.server.tls.clientTlsSecretName }} +# One supervisor process hosting a session per sandbox generation. Sessions +# are registered only by the gateway, over a mutually authenticated API whose +# client credential only the gateway mounts. The Pod holds no Kubernetes or +# gateway identity of its own: each session acts only with the credentials +# the gateway issued for its generation. apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "openshell.fullname" . }}-supervisor + name: {{ $name }} namespace: {{ .Release.Namespace }} labels: {{- include "openshell.labels" . | nindent 4 }} app.kubernetes.io/component: supervisor spec: + # Sessions live in this process; a second replica could not serve them. replicas: 1 strategy: type: Recreate selector: matchLabels: - app.kubernetes.io/name: openshell-supervisor - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: supervisor + {{- include "openshell.supervisorSelectorLabels" . | nindent 6 }} template: metadata: labels: - app.kubernetes.io/name: openshell-supervisor - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: supervisor + {{- include "openshell.supervisorSelectorLabels" . | nindent 8 }} spec: automountServiceAccountToken: false + enableServiceLinks: false terminationGracePeriodSeconds: 30 {{- with ($imagePullSecrets | uniq) }} imagePullSecrets: @@ -48,14 +56,36 @@ spec: image: {{ get $kubernetes "supervisor_image" | quote }} imagePullPolicy: {{ get (dict "always" "Always" "if_not_present" "IfNotPresent" "never" "Never") (get $kubernetes "supervisor_image_pull_policy" | default "if_not_present") | quote }} args: - - --mode=shared - - --gateway-endpoint - - {{ include "openshell.grpcEndpoint" . | quote }} - - --address - - 0.0.0.0:5501 + - experimental-host + - --health-socket + - /run/openshell/health.sock + - --session-control-listen + - 0.0.0.0:{{ get $shared "control_port" }} + - --session-control-tls-dir + - /etc/openshell/session-control + - --runtime-listen + - 0.0.0.0:{{ get $shared "runtime_port" }} + - --session-root + - /run/openshell/sessions + env: + - name: OPENSHELL_ENDPOINT + value: {{ $endpoint | quote }} + - name: OPENSHELL_LOG_LEVEL + value: info + {{- if $gatewayTls }} + - name: OPENSHELL_TLS_CA + value: /var/run/secrets/openshell-tls/ca.crt + {{- end }} + {{- if eq (get $endpointUrl "scheme") "https" }} + - name: OPENSHELL_GATEWAY_TLS_SERVER_NAME + value: {{ get $endpointUrl "hostname" | quote }} + {{- end }} ports: - - name: supervisor - containerPort: 5501 + - name: runtime + containerPort: {{ get $shared "runtime_port" }} + protocol: TCP + - name: session-control + containerPort: {{ get $shared "control_port" }} protocol: TCP securityContext: allowPrivilegeEscalation: false @@ -64,14 +94,109 @@ spec: drop: [ALL] resources: requests: - cpu: 10m - memory: 32Mi + cpu: 50m + memory: 64Mi limits: - memory: 128Mi + memory: 1Gi + # Process health only; each session reports its own readiness to + # the gateway through the session-control API. readinessProbe: - tcpSocket: - port: supervisor + exec: + command: [/openshell-supervisor, health, --socket, /run/openshell/health.sock] livenessProbe: - tcpSocket: - port: supervisor + exec: + command: [/openshell-supervisor, health, --socket, /run/openshell/health.sock] + volumeMounts: + - name: run + mountPath: /run/openshell + - name: logs + mountPath: /var/log + - name: session-control + mountPath: /etc/openshell/session-control + readOnly: true + {{- if $gatewayTls }} + - name: gateway-ca + mountPath: /var/run/secrets/openshell-tls + readOnly: true + {{- end }} + volumes: + - name: run + emptyDir: {} + - name: logs + emptyDir: {} + - name: session-control + secret: + secretName: {{ $name }}-control + {{- if $gatewayTls }} + - name: gateway-ca + secret: + secretName: {{ .Values.server.tls.clientTlsSecretName }} + items: + - key: ca.crt + path: ca.crt + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: supervisor +spec: + selector: + {{- include "openshell.supervisorSelectorLabels" . | nindent 4 }} + ports: + - name: runtime + port: {{ get $shared "runtime_port" }} + targetPort: runtime + protocol: TCP + - name: session-control + port: {{ get $shared "control_port" }} + targetPort: session-control + protocol: TCP +{{- if .Values.networkPolicy.enabled }} +--- +# Only the gateway reaches session control, and only sandbox workloads in this +# gateway's sandbox namespaces reach the routed runtime listener. Each +# sandbox's own egress allowance limits its workload to that port, and the +# routing preface plus the generation's pinned TLS identity decide which +# session it reaches. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: supervisor +spec: + podSelector: + matchLabels: + {{- include "openshell.supervisorSelectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - podSelector: + matchLabels: + {{- include "openshell.selectorLabels" . | nindent 14 }} + ports: + - protocol: TCP + port: {{ get $shared "control_port" }} + - from: + - namespaceSelector: + {{- include "openshell.supervisorSandboxNamespaces" (dict "kubernetes" $kubernetes) | nindent 12 }} + podSelector: + matchLabels: + openshell.ai/boundary-role: workload + ports: + - protocol: TCP + port: {{ get $shared "runtime_port" }} + # Sessions carry policy-approved sandbox egress, as dedicated supervisors do. + egress: + - {} +{{- end }} {{- end }} diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index 2276b3469b..ca2c0caf01 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -13,7 +13,7 @@ tests: asserts: - hasDocuments: count: 0 - - it: deploys one isolated idle process in the gateway namespace + - it: deploys one session host without Kubernetes identity in the gateway namespace template: templates/supervisor.yaml set: gatewayConfig: @@ -21,50 +21,181 @@ tests: supervisor_topology: gateway supervisor_image: example.com/supervisor:shared asserts: + - hasDocuments: + count: 3 + - equal: + path: kind + value: Deployment + documentIndex: 0 - equal: path: metadata.namespace value: gateway + documentIndex: 0 - equal: path: spec.replicas value: 1 + documentIndex: 0 - equal: path: spec.template.metadata.labels["app.kubernetes.io/name"] value: openshell-supervisor + documentIndex: 0 - equal: path: spec.template.spec.containers[0].image value: example.com/supervisor:shared + documentIndex: 0 - equal: path: spec.template.spec.containers[0].args value: - - --mode=shared - - --gateway-endpoint - - https://openshell.gateway.svc.cluster.local:8080 - - --address + - experimental-host + - --health-socket + - /run/openshell/health.sock + - --session-control-listen - 0.0.0.0:5501 + - --session-control-tls-dir + - /etc/openshell/session-control + - --runtime-listen + - 0.0.0.0:5500 + - --session-root + - /run/openshell/sessions + documentIndex: 0 + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_ENDPOINT + value: https://openshell.gateway.svc.cluster.local:8080 + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_GATEWAY_TLS_SERVER_NAME + value: openshell.gateway.svc.cluster.local + documentIndex: 0 + - contains: + path: spec.template.spec.volumes + content: + name: session-control + secret: + secretName: openshell-supervisor-control + documentIndex: 0 + - contains: + path: spec.template.spec.volumes + content: + name: gateway-ca + secret: + secretName: openshell-client-tls + items: + - key: ca.crt + path: ca.crt + documentIndex: 0 - equal: - path: spec.template.spec.containers[0].ports + path: kind + value: Service + documentIndex: 1 + - equal: + path: spec.selector value: - - name: supervisor - containerPort: 5501 - protocol: TCP + app.kubernetes.io/name: openshell-supervisor + app.kubernetes.io/instance: openshell + app.kubernetes.io/component: supervisor + documentIndex: 1 + - equal: + path: kind + value: NetworkPolicy + documentIndex: 2 - equal: - path: spec.template.spec.containers[0].readinessProbe + path: spec.ingress[0] value: - tcpSocket: - port: supervisor + from: + - podSelector: + matchLabels: + app.kubernetes.io/name: openshell + app.kubernetes.io/instance: openshell + ports: + - protocol: TCP + port: 5501 + documentIndex: 2 - equal: - path: spec.template.spec.containers[0].livenessProbe + path: spec.ingress[1] value: - tcpSocket: - port: supervisor - - notExists: - path: spec.template.spec.containers[0].volumeMounts - - notExists: - path: spec.template.spec.volumes + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gateway + podSelector: + matchLabels: + openshell.ai/boundary-role: workload + ports: + - protocol: TCP + port: 5500 + documentIndex: 2 + - it: admits runtimes only from this gateway's managed namespaces + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + workspace_mode: managed + gateway_id: gw-a + asserts: - equal: - path: spec.template.spec.automountServiceAccountToken - value: false - - it: renders the same setting into gateway TOML + path: spec.ingress[1].from[0].namespaceSelector + value: + matchLabels: + openshell.ai/managed-by: openshell + openshell.ai/gateway-id: gw-a + documentIndex: 2 + - it: admits runtimes only from operator-labeled namespaces + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + workspace_mode: operator + operator_namespace_label: example.com/openshell-workspace=true + asserts: + - equal: + path: spec.ingress[1].from[0].namespaceSelector + value: + matchLabels: + example.com/openshell-workspace: "true" + documentIndex: 2 + - it: rejects operator namespace selectors it cannot scope + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + workspace_mode: operator + operator_namespace_file: /etc/openshell/namespaces + asserts: + - failedTemplate: + errorMessage: supervisor_topology=gateway in operator workspace mode requires operator_namespace_label of the form key=value + - it: omits the gateway CA when gateway TLS is disabled + template: templates/supervisor.yaml + set: + server: + disableTls: true + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_TLS_CA + value: /var/run/secrets/openshell-tls/ca.crt + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_ENDPOINT + value: http://openshell.gateway.svc.cluster.local:8080 + documentIndex: 0 + - it: renders the chart-owned shared supervisor connection into gateway TOML template: templates/gateway-config.yaml set: gatewayConfig: @@ -74,6 +205,31 @@ tests: - matchRegex: path: data["gateway.toml"] pattern: 'supervisor_topology = "gateway"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'shared_supervisor = \{ control_port = 5501, control_tls_dir = "/etc/openshell-tls/supervisor-control", runtime_port = 5500, service_name = "openshell-supervisor", service_namespace = "gateway" \}' + - it: rejects a hand-written shared supervisor connection + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + shared_supervisor: + service_name: elsewhere + asserts: + - failedTemplate: + errorMessage: shared_supervisor is rendered by the chart; set only supervisor_topology + - it: rejects settings the shared supervisor cannot isolate + template: templates/supervisor.yaml + set: + upstreamProxy: + url: http://proxy.example:3128 + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - failedTemplate: + errorMessage: supervisor_topology=gateway does not support upstream proxy settings or provider SPIFFE identity - it: rejects workspace until implemented template: templates/supervisor.yaml set: diff --git a/deploy/helm/openshell/tests/supervisor_wiring_test.yaml b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml new file mode 100644 index 0000000000..aca63a5963 --- /dev/null +++ b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: supervisor gateway wiring +templates: + - templates/certgen.yaml + - templates/peer-role.yaml + - templates/statefulset.yaml + - templates/gateway-config.yaml +release: + name: openshell + namespace: gateway +tests: + - it: issues the session-control PKI with the hook + template: templates/certgen.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - equal: + path: spec.template.spec.containers[1].args + value: + - generate-certs + - --tls-only + - --server-secret-name=openshell-supervisor-control + - --client-secret-name=openshell-supervisor-control-client + - --server-san=openshell-supervisor.gateway.svc + documentIndex: 3 + - it: issues only the session-control PKI when the gateway PKI is external + template: templates/certgen.yaml + set: + pkiInitJob: + enabled: false + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + documentIndex: 3 + - equal: + path: spec.template.spec.containers[0].name + value: supervisor-certgen + documentIndex: 3 + - it: lets the gateway read only the shared supervisor Service + template: templates/peer-role.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - services + resourceNames: + - openshell-supervisor + verbs: + - get + documentIndex: 0 + - it: mounts the session-control client credential only on the gateway + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: supervisor-control + secret: + secretName: openshell-supervisor-control-client + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: supervisor-control + mountPath: /etc/openshell-tls/supervisor-control + readOnly: true From 76d5b4a0569da2278b3672c68e31b5cf9dab9b62 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 17:00:24 -0700 Subject: [PATCH 06/25] refactor(ocsf): attribute events to session-bound contexts A supervisor process that hosts several sandboxes cannot use one process-wide OCSF context: every event would carry the wrong sandbox identity, and middleware requests read their sandbox identity from the same context. Builders now accept a ContextRef, either a borrowed context or a context shared by a hosted session, so per-session contexts need not be leaked. ctx() returns the context bound to the calling thread when present and falls back to the process-wide context otherwise. OCSF JSON output settings can likewise be bound per thread, and OpenTelemetry can parent a span from an explicit W3C trace context instead of process environment. A dedicated supervisor keeps using the process-wide context. Signed-off-by: Drew Newberry --- .../src/builders/api_activity.rs | 8 ++-- .../src/builders/authentication.rs | 8 ++-- crates/openshell-ocsf/src/builders/base.rs | 8 ++-- crates/openshell-ocsf/src/builders/config.rs | 8 ++-- crates/openshell-ocsf/src/builders/finding.rs | 8 ++-- crates/openshell-ocsf/src/builders/http.rs | 8 ++-- .../openshell-ocsf/src/builders/lifecycle.rs | 8 ++-- crates/openshell-ocsf/src/builders/mod.rs | 40 ++++++++++++++++++ crates/openshell-ocsf/src/builders/network.rs | 8 ++-- crates/openshell-ocsf/src/builders/process.rs | 8 ++-- crates/openshell-ocsf/src/builders/ssh.rs | 8 ++-- crates/openshell-ocsf/src/ctx.rs | 42 +++++++++++++------ crates/openshell-ocsf/src/lib.rs | 7 ++-- .../src/tracing_layers/jsonl_layer.rs | 32 +++++++++++++- .../openshell-ocsf/src/tracing_layers/mod.rs | 2 +- crates/openshell-otel/src/lib.rs | 2 +- crates/openshell-otel/src/propagation.rs | 32 +++++++++----- crates/openshell-sandbox/src/delegated.rs | 2 +- .../src/l7/middleware.rs | 4 +- .../src/l7/relay.rs | 4 +- .../src/delegated.rs | 2 +- .../src/supervisor_session.rs | 15 +++---- 22 files changed, 181 insertions(+), 83 deletions(-) diff --git a/crates/openshell-ocsf/src/builders/api_activity.rs b/crates/openshell-ocsf/src/builders/api_activity.rs index fe88434804..4740b05053 100644 --- a/crates/openshell-ocsf/src/builders/api_activity.rs +++ b/crates/openshell-ocsf/src/builders/api_activity.rs @@ -3,7 +3,7 @@ //! Builder for API Activity [6003] events. -use crate::builders::{EventContext, EventOrigin}; +use crate::builders::{ContextRef, EventOrigin}; use crate::enums::{ActionId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{ApiActivityEvent, OcsfEvent}; @@ -28,7 +28,7 @@ fn sanitize_model_name(name: &str) -> String { /// Builder for API Activity [6003] events. pub struct ApiActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, message: Option, @@ -49,9 +49,9 @@ pub struct ApiActivityBuilder<'a> { impl<'a> ApiActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext, operation: impl Into) -> Self { + pub fn new(ctx: impl Into>, operation: impl Into) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, message: None, diff --git a/crates/openshell-ocsf/src/builders/authentication.rs b/crates/openshell-ocsf/src/builders/authentication.rs index a12cf2a5d0..492bdf65b7 100644 --- a/crates/openshell-ocsf/src/builders/authentication.rs +++ b/crates/openshell-ocsf/src/builders/authentication.rs @@ -3,7 +3,7 @@ //! Builder for Authentication [3002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, AuthProtocolId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{AuthenticationEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Endpoint, HttpRequest, Service, User}; /// Builder for Authentication [3002] Logon events. pub struct AuthenticationBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, user: User, service: Service, auth_protocol: Option, @@ -29,9 +29,9 @@ pub struct AuthenticationBuilder<'a> { impl<'a> AuthenticationBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext, user: User, service: Service) -> Self { + pub fn new(ctx: impl Into>, user: User, service: Service) -> Self { Self { - ctx, + ctx: ctx.into(), user, service, auth_protocol: None, diff --git a/crates/openshell-ocsf/src/builders/base.rs b/crates/openshell-ocsf/src/builders/base.rs index c2edb5b169..c08f1aa0aa 100644 --- a/crates/openshell-ocsf/src/builders/base.rs +++ b/crates/openshell-ocsf/src/builders/base.rs @@ -3,14 +3,14 @@ //! Builder for Base Event [0]. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{BaseEvent, OcsfEvent}; /// Builder for Base Event [0] — events without a specific OCSF class. pub struct BaseEventBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, message: Option, @@ -20,9 +20,9 @@ pub struct BaseEventBuilder<'a> { impl<'a> BaseEventBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, message: None, diff --git a/crates/openshell-ocsf/src/builders/config.rs b/crates/openshell-ocsf/src/builders/config.rs index 1bc415b612..906470408d 100644 --- a/crates/openshell-ocsf/src/builders/config.rs +++ b/crates/openshell-ocsf/src/builders/config.rs @@ -3,14 +3,14 @@ //! Builder for Device Config State Change [5019] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{SecurityLevelId, SeverityId, StateId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{DeviceConfigStateChangeEvent, OcsfEvent}; /// Builder for Device Config State Change [5019] events. pub struct ConfigStateChangeBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, severity: SeverityId, status: Option, state_id: Option, @@ -23,9 +23,9 @@ pub struct ConfigStateChangeBuilder<'a> { impl<'a> ConfigStateChangeBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), severity: SeverityId::Informational, status: None, state_id: None, diff --git a/crates/openshell-ocsf/src/builders/finding.rs b/crates/openshell-ocsf/src/builders/finding.rs index aea063a527..ccb45d29a8 100644 --- a/crates/openshell-ocsf/src/builders/finding.rs +++ b/crates/openshell-ocsf/src/builders/finding.rs @@ -3,7 +3,7 @@ //! Builder for Detection Finding [2004] events. -use crate::builders::{EventContext, EventOrigin}; +use crate::builders::{ContextRef, EventOrigin}; use crate::enums::{ActionId, ActivityId, ConfidenceId, DispositionId, RiskLevelId, SeverityId}; use crate::events::base_event::BaseEventData; use crate::events::{DetectionFindingEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Attack, Evidence, FindingInfo, Remediation}; /// Builder for Detection Finding [2004] events. pub struct DetectionFindingBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, action: Option, @@ -31,9 +31,9 @@ pub struct DetectionFindingBuilder<'a> { impl<'a> DetectionFindingBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Open, severity: SeverityId::Medium, action: None, diff --git a/crates/openshell-ocsf/src/builders/http.rs b/crates/openshell-ocsf/src/builders/http.rs index b027421bd0..86a22f63ac 100644 --- a/crates/openshell-ocsf/src/builders/http.rs +++ b/crates/openshell-ocsf/src/builders/http.rs @@ -3,7 +3,7 @@ //! Builder for HTTP Activity [4002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{HttpActivityEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Endpoint, FirewallRule, HttpRequest, HttpResponse}; /// Builder for HTTP Activity [4002] events. pub struct HttpActivityBuilder<'a, Context = MissingHttpContext> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, action: Option, disposition: Option, @@ -56,9 +56,9 @@ impl<'a> HttpActivityBuilder<'a, MissingHttpContext> { /// HttpActivityBuilder::new(&ctx).build(); /// ``` #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, action: None, disposition: None, diff --git a/crates/openshell-ocsf/src/builders/lifecycle.rs b/crates/openshell-ocsf/src/builders/lifecycle.rs index 8fc648ee37..b425b3e79b 100644 --- a/crates/openshell-ocsf/src/builders/lifecycle.rs +++ b/crates/openshell-ocsf/src/builders/lifecycle.rs @@ -3,7 +3,7 @@ //! Builder for Application Lifecycle [6002] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActivityId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{ApplicationLifecycleEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::Product; /// Builder for Application Lifecycle [6002] events. pub struct AppLifecycleBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, status: Option, @@ -20,9 +20,9 @@ pub struct AppLifecycleBuilder<'a> { impl<'a> AppLifecycleBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, severity: SeverityId::Informational, status: None, diff --git a/crates/openshell-ocsf/src/builders/mod.rs b/crates/openshell-ocsf/src/builders/mod.rs index 96c966b732..5c2219cc6d 100644 --- a/crates/openshell-ocsf/src/builders/mod.rs +++ b/crates/openshell-ocsf/src/builders/mod.rs @@ -157,6 +157,7 @@ pub use process::ProcessActivityBuilder; pub use ssh::SshActivityBuilder; use std::net::IpAddr; +use std::sync::Arc; use crate::OCSF_VERSION; use crate::enums::StatusId; @@ -199,6 +200,45 @@ pub struct EventContext { pub origin: EventOrigin, } +/// Event context borrowed from the caller or shared by a hosted session. +/// +/// Builders accept either form, so events attributed to a hosted supervisor +/// session need not leak a `'static` context for each session. +#[derive(Debug, Clone)] +pub enum ContextRef<'a> { + Borrowed(&'a EventContext), + Shared(Arc), +} + +impl std::ops::Deref for ContextRef<'_> { + type Target = EventContext; + + fn deref(&self) -> &EventContext { + match self { + Self::Borrowed(context) => context, + Self::Shared(context) => context, + } + } +} + +impl<'a> From<&'a EventContext> for ContextRef<'a> { + fn from(context: &'a EventContext) -> Self { + Self::Borrowed(context) + } +} + +impl From> for ContextRef<'_> { + fn from(context: Arc) -> Self { + Self::Shared(context) + } +} + +impl<'a> From<&'a ContextRef<'_>> for ContextRef<'a> { + fn from(context: &'a ContextRef<'_>) -> Self { + Self::Borrowed(context) + } +} + impl EventContext { /// Build the OCSF `Metadata` object for any event. #[must_use] diff --git a/crates/openshell-ocsf/src/builders/network.rs b/crates/openshell-ocsf/src/builders/network.rs index 09f4c4ff0d..7907abe4a6 100644 --- a/crates/openshell-ocsf/src/builders/network.rs +++ b/crates/openshell-ocsf/src/builders/network.rs @@ -3,7 +3,7 @@ //! Builder for Network Activity [4001] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{NetworkActivityEvent, OcsfEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, ConnectionInfo, Endpoint, FirewallRule}; /// Builder for Network Activity [4001] events. pub struct NetworkActivityBuilder<'a, EndpointState = MissingNetworkEndpoint> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, activity_name: Option, action: Option, @@ -58,9 +58,9 @@ impl<'a> NetworkActivityBuilder<'a, MissingNetworkEndpoint> { /// NetworkActivityBuilder::new(&ctx).build(); /// ``` #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, activity_name: None, action: None, diff --git a/crates/openshell-ocsf/src/builders/process.rs b/crates/openshell-ocsf/src/builders/process.rs index 6e06e6c6e7..5acdb75f01 100644 --- a/crates/openshell-ocsf/src/builders/process.rs +++ b/crates/openshell-ocsf/src/builders/process.rs @@ -3,7 +3,7 @@ //! Builder for Process Activity [1007] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, DispositionId, LaunchTypeId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{OcsfEvent, ProcessActivityEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Process}; /// Builder for Process Activity [1007] events. pub struct ProcessActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, severity: SeverityId, status: Option, @@ -26,9 +26,9 @@ pub struct ProcessActivityBuilder<'a> { impl<'a> ProcessActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, severity: SeverityId::Informational, status: None, diff --git a/crates/openshell-ocsf/src/builders/ssh.rs b/crates/openshell-ocsf/src/builders/ssh.rs index 36383ae9f4..2777bbd05b 100644 --- a/crates/openshell-ocsf/src/builders/ssh.rs +++ b/crates/openshell-ocsf/src/builders/ssh.rs @@ -3,7 +3,7 @@ //! Builder for SSH Activity [4007] events. -use crate::builders::EventContext; +use crate::builders::ContextRef; use crate::enums::{ActionId, ActivityId, AuthTypeId, DispositionId, SeverityId, StatusId}; use crate::events::base_event::BaseEventData; use crate::events::{OcsfEvent, SshActivityEvent}; @@ -11,7 +11,7 @@ use crate::objects::{Actor, Endpoint}; /// Builder for SSH Activity [4007] events. pub struct SshActivityBuilder<'a> { - ctx: &'a EventContext, + ctx: ContextRef<'a>, activity: ActivityId, action: Option, disposition: Option, @@ -28,9 +28,9 @@ pub struct SshActivityBuilder<'a> { impl<'a> SshActivityBuilder<'a> { #[must_use] - pub fn new(ctx: &'a EventContext) -> Self { + pub fn new(ctx: impl Into>) -> Self { Self { - ctx, + ctx: ctx.into(), activity: ActivityId::Unknown, action: None, disposition: None, diff --git a/crates/openshell-ocsf/src/ctx.rs b/crates/openshell-ocsf/src/ctx.rs index 7e1c4c7796..9dd542a9aa 100644 --- a/crates/openshell-ocsf/src/ctx.rs +++ b/crates/openshell-ocsf/src/ctx.rs @@ -1,15 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Process-wide [`EventContext`] singleton. +//! Process-wide and session-bound [`EventContext`]. //! -//! Initialised once via [`set_ctx`] during sandbox start; read by every event -//! builder via [`ctx`]. Falls back to a default context when the singleton has -//! not been set (e.g. unit tests that exercise builders without booting the -//! sandbox). +//! A dedicated supervisor initialises the process-wide context once via +//! [`set_ctx`]. A process hosting several supervisor sessions instead binds +//! each session's context to the threads of that session's runtime with +//! [`bind_thread_ctx`]; a bound context takes precedence. [`ctx`] falls back +//! to a default context when neither is set (e.g. unit tests that exercise +//! builders without booting the sandbox). -use crate::{EventContext, EventOrigin}; -use std::sync::{LazyLock, OnceLock}; +use crate::{ContextRef, EventContext, EventOrigin}; +use std::cell::RefCell; +use std::sync::{Arc, LazyLock, OnceLock}; static OCSF_CTX: OnceLock = OnceLock::new(); @@ -24,6 +27,11 @@ static OCSF_CTX_FALLBACK: LazyLock = LazyLock::new(|| EventContext origin: EventOrigin::Supervisor, }); +std::thread_local! { + /// Context of the hosted session whose runtime owns this thread. + static THREAD_CTX: RefCell>> = const { RefCell::new(None) }; +} + /// Initialise the process-wide OCSF sandbox context. /// /// Returns `false` if the context was already set; the caller may log and @@ -32,11 +40,21 @@ pub fn set_ctx(ctx: EventContext) -> bool { OCSF_CTX.set(ctx).is_ok() } -/// Return a reference to the process-wide [`EventContext`]. +/// Attribute events emitted on the calling thread to one hosted session. +/// +/// Session runtimes bind every thread they start; `None` unbinds it. +pub fn bind_thread_ctx(ctx: Option>) { + THREAD_CTX.with(|current| *current.borrow_mut() = ctx); +} + +/// Return the context for events emitted on the calling thread. /// -/// Falls back to a default context if [`set_ctx`] has not been called (e.g. -/// during unit tests that exercise individual builders). +/// A session-bound context wins over the process-wide one. Falls back to a +/// default context if neither is set. #[must_use] -pub fn ctx() -> &'static EventContext { - OCSF_CTX.get().unwrap_or(&OCSF_CTX_FALLBACK) +pub fn ctx() -> ContextRef<'static> { + if let Some(bound) = THREAD_CTX.with(|current| current.borrow().clone()) { + return ContextRef::Shared(bound); + } + ContextRef::Borrowed(OCSF_CTX.get().unwrap_or(&OCSF_CTX_FALLBACK)) } diff --git a/crates/openshell-ocsf/src/lib.rs b/crates/openshell-ocsf/src/lib.rs index b9878eb68a..310cd9afe4 100644 --- a/crates/openshell-ocsf/src/lib.rs +++ b/crates/openshell-ocsf/src/lib.rs @@ -68,12 +68,13 @@ pub use format::downgrade::OcsfSchemaVersion; // --- Builders --- pub use builders::{ ApiActivityBuilder, AppLifecycleBuilder, AuthenticationBuilder, BaseEventBuilder, - ConfigStateChangeBuilder, DetectionFindingBuilder, EventContext, EventOrigin, + ConfigStateChangeBuilder, ContextRef, DetectionFindingBuilder, EventContext, EventOrigin, HttpActivityBuilder, NetworkActivityBuilder, ProcessActivityBuilder, SshActivityBuilder, }; // --- Tracing layers --- pub use tracing_layers::{ - OCSF_TARGET, OcsfJsonlLayer, OcsfShorthandLayer, clear_current_event, clone_current_event, - emit_ocsf_event, emit_ocsf_event_routed, set_current_event, + OCSF_TARGET, OcsfJsonSettings, OcsfJsonlLayer, OcsfShorthandLayer, bind_thread_json_settings, + clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, + set_current_event, }; diff --git a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs index a0746b8621..3f0936e841 100644 --- a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs +++ b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs @@ -16,6 +16,27 @@ use tracing_subscriber::layer::Context; use crate::format::downgrade::downgrade_event; use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event}; +/// OCSF JSON output settings owned by one hosted supervisor session. +/// +/// Sessions apply their own `ocsf_json_enabled` and schema-version settings +/// to these values. Binding them to the session's runtime threads makes every +/// [`OcsfJsonlLayer`] use them for events those threads emit. +#[derive(Clone, Debug, Default)] +pub struct OcsfJsonSettings { + pub enabled: Arc, + pub target_version: Arc>, +} + +std::thread_local! { + static THREAD_JSON_SETTINGS: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +/// Apply one hosted session's JSON settings to events emitted on this thread. +pub fn bind_thread_json_settings(settings: Option) { + THREAD_JSON_SETTINGS.with(|current| *current.borrow_mut() = settings); +} + /// A tracing `Layer` that intercepts OCSF events and writes JSONL output. /// /// Only events with `target: "ocsf"` are processed; non-OCSF events are ignored. @@ -90,8 +111,15 @@ where return; } + // A hosted session's settings replace the process-wide ones. + let session = THREAD_JSON_SETTINGS.with(|current| current.borrow().clone()); + let (enabled, target_version) = session.as_ref().map_or( + (self.enabled.as_ref(), self.target_version.as_ref()), + |settings| (Some(&settings.enabled), Some(&settings.target_version)), + ); + // If an enabled flag is set and it reads `false`, skip writing. - if let Some(ref flag) = self.enabled + if let Some(flag) = enabled && !flag.load(Ordering::Relaxed) { return; @@ -100,7 +128,7 @@ where if let Some(ocsf_event) = clone_current_event() && let Ok(mut w) = self.writer.lock() { - let line = if let Some(ref target) = self.target_version + let line = if let Some(target) = target_version && let Ok(version) = target.lock() && !version.is_empty() { diff --git a/crates/openshell-ocsf/src/tracing_layers/mod.rs b/crates/openshell-ocsf/src/tracing_layers/mod.rs index b57ba1364a..808051b061 100644 --- a/crates/openshell-ocsf/src/tracing_layers/mod.rs +++ b/crates/openshell-ocsf/src/tracing_layers/mod.rs @@ -15,5 +15,5 @@ pub use event_bridge::{ OCSF_TARGET, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed, set_current_event, }; -pub use jsonl_layer::OcsfJsonlLayer; +pub use jsonl_layer::{OcsfJsonSettings, OcsfJsonlLayer, bind_thread_json_settings}; pub use shorthand_layer::OcsfShorthandLayer; diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index b10c300833..914bbea861 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -23,7 +23,7 @@ pub use ocsf::{OcsfOtlp, OcsfOtlpLayer}; pub use propagation::{ HeaderMapExtractor, MetadataMapInjector, TRACEPARENT_ENV, TRACESTATE_ENV, TraceContextInterceptor, current_trace_context_carrier, current_trace_context_environment, - set_parent_from_environment, trace_context_environment, + set_parent_from_environment, set_parent_from_trace_context, trace_context_environment, }; use opentelemetry::KeyValue; diff --git a/crates/openshell-otel/src/propagation.rs b/crates/openshell-otel/src/propagation.rs index 764e343146..82e5fa784d 100644 --- a/crates/openshell-otel/src/propagation.rs +++ b/crates/openshell-otel/src/propagation.rs @@ -114,17 +114,27 @@ pub fn current_trace_context_environment() -> Vec<(&'static str, String)> { /// /// Leaves `span` unchanged when the environment carries no valid context. pub fn set_parent_from_environment(span: &tracing::Span) { - let carrier = [ - ("traceparent", TRACEPARENT_ENV), - ("tracestate", TRACESTATE_ENV), - ] - .into_iter() - .filter_map(|(field, name)| { - std::env::var(name) - .ok() - .map(|value| (field.to_string(), value)) - }) - .collect::>(); + set_parent_from_trace_context( + span, + std::env::var(TRACEPARENT_ENV).ok().as_deref(), + std::env::var(TRACESTATE_ENV).ok().as_deref(), + ); +} + +/// Parent `span` under an explicit W3C `traceparent` and `tracestate`. +/// +/// A process supervising several sandboxes receives each launch's context +/// with its session instead of through process environment. Leaves `span` +/// unchanged when the values carry no valid context. +pub fn set_parent_from_trace_context( + span: &tracing::Span, + traceparent: Option<&str>, + tracestate: Option<&str>, +) { + let carrier = [("traceparent", traceparent), ("tracestate", tracestate)] + .into_iter() + .filter_map(|(field, value)| value.map(|value| (field.to_string(), value.to_string()))) + .collect::>(); set_parent_from_carrier(span, &carrier); } diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index d7ceb67024..45639c4c00 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -21,7 +21,7 @@ use openshell_ocsf::{ use crate::process::{ProcessHandle, ProcessStatus, ResolvedWorkspace}; -fn ocsf_ctx() -> &'static openshell_ocsf::EventContext { +fn ocsf_ctx() -> openshell_ocsf::ContextRef<'static> { openshell_ocsf::ctx::ctx() } diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 10fa59b36e..d2344958b7 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -568,7 +568,7 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id< // unresolved binding is handled before the body is read) and forward // the original request unchanged if the chain allows. let input = middleware_request_input_with_id( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), scheme, &req, ctx, @@ -615,7 +615,7 @@ pub async fn apply_middleware_chain_for_scheme_with_request_id< let headers = safe_middleware_headers(&buffered.headers)?; let query = raw_query_from_request_headers(&buffered.headers)?; let input = middleware_request_input_with_id( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), scheme, &req, ctx, diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 512c9096d1..f3e9e34741 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -1748,7 +1748,7 @@ pub(crate) async fn websocket_middleware_preflight( let requested_subprotocols = crate::l7::rest::websocket_requested_subprotocols(&req.raw_header[..header_end])?; let input = websocket_preflight_input( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), ctx, req, scheme, @@ -7604,7 +7604,7 @@ network_policies: }; let input = middleware_request_input( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), "http", &req, &ctx, diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 3529683403..91e1fa5337 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -16,7 +16,7 @@ use miette::Result; use openshell_ocsf::{ActivityId, AppLifecycleBuilder, SeverityId, StatusId, ocsf_emit}; #[cfg(unix)] -fn ocsf_ctx() -> &'static openshell_ocsf::EventContext { +fn ocsf_ctx() -> openshell_ocsf::ContextRef<'static> { openshell_ocsf::ctx::ctx() } diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 1f1c7767ff..77620a94d1 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -467,7 +467,7 @@ async fn run_session_loop(config: SessionConfig) { Ok(SessionOutcome::Closed) => { config.ready_tx.send_replace(false); let event = - session_closed_event(openshell_ocsf::ctx::ctx(), &target, &config.sandbox_id); + session_closed_event(&openshell_ocsf::ctx::ctx(), &target, &config.sandbox_id); ocsf_emit!(event); break; } @@ -489,7 +489,7 @@ async fn run_session_loop(config: SessionConfig) { Err(e) => { let accepted = config.ready_tx.send_replace(false); let event = session_failed_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &target, attempt, &e.to_string(), @@ -667,7 +667,7 @@ async fn run_single_session( updates.send_replace(Some(accepted.session_id.clone())); } let event = session_established_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), target, &accepted.session_id, u32::try_from(heartbeat_secs).unwrap_or(u32::MAX), @@ -904,7 +904,8 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< let expected_ssh_peer_pid = context.expected_ssh_peer_pid; let terminating = Arc::clone(context.terminating); - let event = relay_open_event(openshell_ocsf::ctx::ctx(), &relay_open, &ssh_socket_path); + let event = + relay_open_event(&openshell_ocsf::ctx::ctx(), &relay_open, &ssh_socket_path); ocsf_emit!(event); tokio::spawn(async move { @@ -922,7 +923,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< { Ok(()) => { let event = relay_closed_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &event_open, &ssh_socket_path, ); @@ -930,7 +931,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< } Err(e) => { let event = relay_failed_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &event_open, &ssh_socket_path, &e.to_string(), @@ -948,7 +949,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< } Some(gateway_message::Payload::RelayClose(close)) => { let event = relay_close_from_gateway_event( - openshell_ocsf::ctx::ctx(), + &openshell_ocsf::ctx::ctx(), &close.channel_id, &close.reason, ); From 3ae600f9315c02f74f77f53fd80566a5600c05eb Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 18:29:26 -0700 Subject: [PATCH 07/25] test(e2e): run two sandboxes on one shared Kubernetes supervisor The e2e:kubernetes:shared-supervisor lane enables the gateway topology and creates two sandboxes with different policies and provider credentials. It checks that one shared supervisor process hosts both, that each session enforces its own policy and injects its own credential, that stopping one leaves the other working, that the routed listener refuses stale and unknown routes and drops a peer that names a live route without its pinned identity, and that a restart registers a fresh session on the same process. Signed-off-by: Drew Newberry --- .../helm/openshell/ci/values-supervisor.yaml | 9 + e2e/rust/Cargo.toml | 2 + .../tests/kubernetes_shared_supervisor.rs | 642 ++++++++++++++++++ tasks/test.toml | 5 + 4 files changed, 658 insertions(+) create mode 100644 deploy/helm/openshell/ci/values-supervisor.yaml create mode 100644 e2e/rust/tests/kubernetes_shared_supervisor.rs diff --git a/deploy/helm/openshell/ci/values-supervisor.yaml b/deploy/helm/openshell/ci/values-supervisor.yaml new file mode 100644 index 0000000000..0361016d1b --- /dev/null +++ b/deploy/helm/openshell/ci/values-supervisor.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E overlay: supervise new sandbox generations with one supervisor process +# shared by the gateway. Layer it on a workspace-mode overlay +# to cover managed or operator namespaces. +gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index e75aa25ec2..1dc6f0b1cf 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -36,6 +36,8 @@ e2e-kubernetes-ha = ["e2e-kubernetes"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] +# One supervisor process shared by the gateway's sandboxes. +e2e-kubernetes-shared-supervisor = ["e2e-kubernetes", "e2e-host-gateway"] e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] e2e-oidc-pkce = [] diff --git a/e2e/rust/tests/kubernetes_shared_supervisor.rs b/e2e/rust/tests/kubernetes_shared_supervisor.rs new file mode 100644 index 0000000000..59514a7365 --- /dev/null +++ b/e2e/rust/tests/kubernetes_shared_supervisor.rs @@ -0,0 +1,642 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-shared-supervisor")] + +//! Two sandboxes supervised by one shared supervisor process +//! (`supervisor_topology = "gateway"`). +//! +//! Each sandbox keeps its own policy and provider credential, stopping one +//! leaves the other supervised, and the shared runtime listener refuses +//! connections that name a stale or unknown session route. + +use std::fmt::Write as _; +use std::io::Write as _; +use std::process::Stdio; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::sandbox::SandboxGuard; +use tempfile::NamedTempFile; +use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; + +const TEST_HOST: &str = "host.openshell.internal"; +const ALPHA: &str = "e2e-shared-alpha"; +const BETA: &str = "e2e-shared-beta"; +const TOKEN_ENV: &str = "E2E_SHARED_SUPERVISOR_TOKEN"; +const TOPOLOGY_ANNOTATION: &str = "openshell.ai/sandbox-runtime-supervisor-topology"; +const SESSION_ANNOTATION: &str = "openshell.ai/sandbox-runtime-supervisor-session"; +const SUPERVISOR_COMPONENT: &str = "app.kubernetes.io/component=supervisor"; +const RUNTIME_PORT: u16 = 5500; + +fn release_namespace() -> String { + std::env::var("OPENSHELL_E2E_KUBE_NAMESPACE").unwrap_or_else(|_| "openshell".to_string()) +} + +fn kubectl_command() -> tokio::process::Command { + let mut command = tokio::process::Command::new("kubectl"); + if let Ok(context) = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") + && !context.trim().is_empty() + { + command.arg("--context").arg(context); + } + command +} + +async fn kubectl(args: &[&str]) -> String { + let output = kubectl_command() + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .unwrap_or_else(|error| panic!("spawn kubectl {args:?}: {error}")); + assert!( + output.status.success(), + "kubectl {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +async fn cli(args: &[&str]) -> (bool, String) { + let output = openshell_cmd() + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("spawn openshell CLI"); + ( + output.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + ) +} + +/// The Sandbox resource of an `OpenShell` sandbox, in whichever namespace the +/// workspace mode placed it. +async fn sandbox_resource(name: &str) -> serde_json::Value { + let output = kubectl(&[ + "get", + "sandboxes.agents.x-k8s.io", + "--all-namespaces", + "-l", + &format!("openshell.ai/sandbox-name={name}"), + "-o", + "json", + ]) + .await; + let list: serde_json::Value = serde_json::from_str(&output).expect("parse Sandbox list"); + let [resource] = list["items"].as_array().expect("Sandbox items").as_slice() else { + panic!("expected one Sandbox resource for {name}: {list}"); + }; + resource.clone() +} + +/// The shared-supervisor session route recorded for a sandbox's current +/// generation. +async fn session_route(name: &str) -> String { + let resource = sandbox_resource(name).await; + let annotations = &resource["metadata"]["annotations"]; + assert_eq!( + annotations[TOPOLOGY_ANNOTATION], "gateway", + "{name} is not supervised by the shared supervisor: {annotations}" + ); + annotations[SESSION_ANNOTATION] + .as_str() + .unwrap_or_else(|| panic!("{name} has no session route: {annotations}")) + .to_string() +} + +/// UID and restart count of the one shared supervisor Pod. +async fn shared_supervisor_pod() -> (String, i64) { + let output = kubectl(&[ + "-n", + &release_namespace(), + "get", + "pods", + "-l", + SUPERVISOR_COMPONENT, + "-o", + "json", + ]) + .await; + let list: serde_json::Value = serde_json::from_str(&output).expect("parse Pod list"); + let pods = list["items"] + .as_array() + .expect("Pod items") + .iter() + .filter(|pod| pod["metadata"]["deletionTimestamp"].is_null()) + .collect::>(); + let [pod] = pods.as_slice() else { + panic!("expected exactly one shared supervisor Pod: {list}"); + }; + let restarts = pod["status"]["containerStatuses"] + .as_array() + .into_iter() + .flatten() + .map(|status| status["restartCount"].as_i64().unwrap_or_default()) + .sum(); + ( + pod["metadata"]["uid"] + .as_str() + .expect("supervisor Pod UID") + .to_string(), + restarts, + ) +} + +/// No dedicated supervisor Pod serves any sandbox. +async fn assert_no_dedicated_supervisors() { + let output = kubectl(&[ + "get", + "pods", + "--all-namespaces", + "-l", + "openshell.ai/boundary-role=supervisor", + "-o", + "name", + ]) + .await; + assert!( + output.trim().is_empty(), + "dedicated supervisor Pods exist under the gateway topology:\n{output}" + ); +} + +/// Credential a host-side endpoint received, as reported back to a sandbox. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Seen { + Alpha, + Beta, + Placeholder, + Other, +} + +impl Seen { + const fn marker(self) -> &'static str { + match self { + Self::Alpha => "SAW_ALPHA", + Self::Beta => "SAW_BETA", + Self::Placeholder => "SAW_PLACEHOLDER", + Self::Other => "SAW_OTHER", + } + } +} + +/// Host endpoint that reports which provider credential each request carried. +struct CredentialEndpoint { + port: u16, + seen: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl CredentialEndpoint { + async fn start(alpha: &str, beta: &str) -> Self { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .expect("bind credential endpoint"); + let port = listener.local_addr().expect("endpoint address").port(); + let seen = Arc::new(Mutex::new(Vec::new())); + let (alpha, beta, record) = (alpha.to_string(), beta.to_string(), seen.clone()); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let (alpha, beta, record) = (alpha.clone(), beta.clone(), record.clone()); + tokio::spawn(async move { + let _ = answer(stream, &alpha, &beta, &record).await; + }); + } + }); + Self { port, seen, task } + } + + fn seen(&self) -> Vec { + self.seen.lock().unwrap().clone() + } +} + +impl Drop for CredentialEndpoint { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn answer( + stream: TcpStream, + alpha: &str, + beta: &str, + record: &Mutex>, +) -> std::io::Result<()> { + let mut reader = tokio::io::BufReader::new(stream); + let mut authorization = String::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).await? == 0 || line == "\r\n" { + break; + } + if let Some((name, value)) = line.split_once(':') + && name.eq_ignore_ascii_case("authorization") + { + authorization = value.trim().to_string(); + } + } + let seen = if authorization.contains(alpha) { + Seen::Alpha + } else if authorization.contains(beta) { + Seen::Beta + } else if authorization.contains("openshell:resolve:env:") { + Seen::Placeholder + } else { + Seen::Other + }; + record.lock().unwrap().push(seen); + let body = seen.marker(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + reader.get_mut().write_all(response.as_bytes()).await +} + +fn temp_yaml(contents: &str) -> NamedTempFile { + let mut file = tempfile::Builder::new() + .suffix(".yaml") + .tempfile() + .expect("create temp file"); + file.write_all(contents.as_bytes()) + .expect("write temp file"); + file.flush().expect("flush temp file"); + file +} + +/// A profile whose credential binds only to `port` on the test host. An +/// attached provider also contributes that endpoint to the sandbox's policy. +fn profile(id: &str, port: u16) -> NamedTempFile { + temp_yaml(&format!( + r"id: {id} +display_name: E2E Shared Supervisor +category: other +credentials: + - name: token + env_vars: [{TOKEN_ENV}] + required: true + auth_style: bearer + header_name: authorization +endpoints: + - host: {TEST_HOST} + port: {port} + protocol: rest + access: full +binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +" + )) +} + +/// A sandbox policy allowing only `ports` on the test host. +fn policy(ports: &[u16]) -> NamedTempFile { + let endpoints = ports.iter().fold(String::new(), |mut endpoints, port| { + let _ = write!( + endpoints, + " - host: {TEST_HOST}\n port: {port}\n protocol: rest\n access: full\n" + ); + endpoints + }); + temp_yaml(&format!( + r"version: 1 +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +network_policies: + shared_supervisor: + name: shared_supervisor + endpoints: +{endpoints} binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +" + )) +} + +/// Call `port` on the test host, with the sandbox's provider placeholder when +/// `credential` is set, and print what the endpoint saw, or `DENIED`. +fn call_script(port: u16, credential: bool) -> String { + let authorization = if credential { + "Authorization: Bearer {token}\\r\\n" + } else { + "" + }; + format!( + r#" +import os +import socket + +try: + with socket.create_connection(({TEST_HOST:?}, {port}), timeout=15) as sock: + token = os.environ.get({TOKEN_ENV:?}, "") + sock.sendall( + (f"GET /credential HTTP/1.1\r\nHost: {TEST_HOST}:{port}\r\n" + f"{authorization}Connection: close\r\n\r\n").encode("ascii") + ) + sock.settimeout(15) + response = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk + markers = [m for m in ("SAW_ALPHA", "SAW_BETA", "SAW_PLACEHOLDER", "SAW_OTHER") if m.encode() in response] + print(markers[0] if markers else "DENIED") +except OSError: + print("DENIED") +"# + ) +} + +async fn call(sandbox: &SandboxGuard, port: u16) -> String { + call_with(sandbox, port, true).await +} + +async fn call_with(sandbox: &SandboxGuard, port: u16, credential: bool) -> String { + let output = sandbox + .exec(&["python3", "-c", &call_script(port, credential)]) + .await + .unwrap_or_else(|error| panic!("call from {}: {error}", sandbox.name)); + output + .lines() + .map(str::trim) + .find(|line| line.starts_with("SAW_") || *line == "DENIED") + .unwrap_or_else(|| panic!("no call result from {}:\n{output}", sandbox.name)) + .to_string() +} + +async fn delete_until_gone(args: &[&str]) { + for _ in 0..40 { + let (deleted, output) = cli(args).await; + if deleted || output.to_lowercase().contains("not found") { + return; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + eprintln!("'{}' did not settle", args.join(" ")); +} + +async fn remove_providers() { + for name in [ALPHA, BETA] { + delete_until_gone(&["provider", "delete", name]).await; + delete_until_gone(&["profile", "delete", name]).await; + } +} + +/// Install provider `name`, of a profile with the same name, holding `secret` +/// for `port`. +async fn install_provider(name: &str, port: u16, secret: &str) { + let profile = profile(name, port); + let (imported, output) = cli(&[ + "profile", + "import", + "--file", + profile.path().to_str().expect("UTF-8 profile path"), + ]) + .await; + assert!(imported, "profile import {name} failed:\n{output}"); + let credential = format!("{TOKEN_ENV}={secret}"); + let (created, output) = cli(&[ + "provider", + "create", + "--name", + name, + "--type", + name, + "--credential", + &credential, + ]) + .await; + assert!(created, "provider create {name} failed:\n{output}"); +} + +/// Port-forward to the shared supervisor's routed runtime listener. +struct RuntimeListener { + child: tokio::process::Child, + port: u16, +} + +impl RuntimeListener { + async fn forward() -> Self { + let service = format!( + "svc/{}-supervisor", + std::env::var("OPENSHELL_E2E_KUBE_RELEASE").unwrap_or_else(|_| "openshell".into()) + ); + let mut child = kubectl_command() + .args([ + "-n", + &release_namespace(), + "port-forward", + &service, + &format!(":{RUNTIME_PORT}"), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true) + .spawn() + .expect("spawn kubectl port-forward"); + let mut lines = tokio::io::BufReader::new(child.stdout.take().expect("stdout")).lines(); + let port = tokio::time::timeout(Duration::from_secs(30), async { + while let Some(line) = lines.next_line().await.expect("read port-forward") { + // "Forwarding from 127.0.0.1:43127 -> 5500" + if let Some(port) = line + .strip_prefix("Forwarding from 127.0.0.1:") + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|port| port.parse().ok()) + { + return port; + } + } + panic!("port-forward ended before it was ready"); + }) + .await + .expect("port-forward ready"); + // kubectl reports every forwarded connection on stdout and exits if + // that pipe closes. + tokio::spawn(async move { while let Ok(Some(_)) = lines.next_line().await {} }); + Self { child, port } + } + + /// Name `route` in a routing preface and return the first bytes the + /// supervisor answers with before it closes the connection or goes quiet. + async fn probe(&self, route: &str, reply: &[u8]) -> Vec { + let mut stream = TcpStream::connect(("127.0.0.1", self.port)) + .await + .expect("connect to forwarded runtime listener"); + let mut preface = b"OSRP\x01".to_vec(); + preface.push(u8::try_from(route.len()).expect("route length")); + preface.extend_from_slice(route.as_bytes()); + stream.write_all(&preface).await.expect("send preface"); + let mut answer = vec![0_u8; 5]; + let read = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut answer)) + .await + .map_or(0, |read| read.unwrap_or(0)); + answer.truncate(read); + if read > 0 { + // Answer the supervisor's TLS handshake without its pinned identity. + let _ = stream.write_all(reply).await; + let mut rest = Vec::new(); + let _ = + tokio::time::timeout(Duration::from_secs(10), stream.read_to_end(&mut rest)).await; + } + answer + } +} + +impl Drop for RuntimeListener { + fn drop(&mut self) { + let _ = self.child.start_kill(); + } +} + +const TLS_HANDSHAKE_RECORD: u8 = 0x16; + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn one_supervisor_isolates_and_outlives_concurrent_sandbox_sessions() { + let nonce = format!("{:08x}", rand::random::()); + let (alpha_secret, beta_secret) = (format!("alpha-{nonce}"), format!("beta-{nonce}")); + let alpha_endpoint = CredentialEndpoint::start(&alpha_secret, &beta_secret).await; + let beta_endpoint = CredentialEndpoint::start(&alpha_secret, &beta_secret).await; + let plain_endpoint = CredentialEndpoint::start(&alpha_secret, &beta_secret).await; + remove_providers().await; + install_provider(ALPHA, alpha_endpoint.port, &alpha_secret).await; + install_provider(BETA, beta_endpoint.port, &beta_secret).await; + let (supervisor_uid, supervisor_restarts) = shared_supervisor_pod().await; + + // Only alpha's own policy allows the plain endpoint. + let alpha_policy = policy(&[alpha_endpoint.port, plain_endpoint.port]); + let beta_policy = policy(&[beta_endpoint.port]); + let keep = ["sh", "-c", "echo shared-ready; exec sleep infinity"]; + let create = |policy: &NamedTempFile, provider: &str| { + let policy = policy + .path() + .to_str() + .expect("UTF-8 policy path") + .to_string(); + let provider = provider.to_string(); + async move { + SandboxGuard::create_keep_with_args( + &["--policy", &policy, "--provider", &provider], + &keep, + "shared-ready", + ) + .await + } + }; + let (alpha, beta) = tokio::join!(create(&alpha_policy, ALPHA), create(&beta_policy, BETA)); + let mut alpha = alpha.expect("create alpha sandbox"); + let mut beta = beta.expect("create beta sandbox"); + + // Both generations are sessions on the one shared supervisor process. + let alpha_route = session_route(&alpha.name).await; + let beta_route = session_route(&beta.name).await; + assert_ne!(alpha_route, beta_route); + assert_no_dedicated_supervisors().await; + assert_eq!(shared_supervisor_pod().await.0, supervisor_uid); + + // Each session injects only its own credential and enforces only its own + // policy, including the endpoint its own provider contributes. + assert_eq!(call(&alpha, alpha_endpoint.port).await, "SAW_ALPHA"); + assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); + assert_eq!(call(&alpha, beta_endpoint.port).await, "DENIED"); + assert_eq!(call(&beta, alpha_endpoint.port).await, "DENIED"); + assert_eq!( + call_with(&alpha, plain_endpoint.port, false).await, + "SAW_OTHER" + ); + assert_eq!(call_with(&beta, plain_endpoint.port, false).await, "DENIED"); + assert_eq!(alpha_endpoint.seen(), [Seen::Alpha]); + assert_eq!(beta_endpoint.seen(), [Seen::Beta]); + assert_eq!(plain_endpoint.seen(), [Seen::Other]); + + // Stopping alpha removes only its session. + let (stopped, output) = cli(&["sandbox", "stop", &alpha.name]).await; + assert!(stopped, "sandbox stop failed:\n{output}"); + assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); + + // A peer naming a live route must present that session's pinned TLS + // identity, and failing to does not displace the session. Stale and + // unknown routes are refused before any handshake. + let listener = RuntimeListener::forward().await; + for _ in 0..3 { + let answer = listener + .probe(&beta_route, b"\x15\x03\x03\x00\x02\x02\x28") + .await; + assert_eq!( + answer.first(), + Some(&TLS_HANDSHAKE_RECORD), + "a live route should demand the session's pinned TLS identity" + ); + } + assert!( + listener.probe(&alpha_route, b"").await.is_empty(), + "stale route reached a session" + ); + let unknown = format!("rt-{:032x}", rand::random::()); + assert!( + listener.probe(&unknown, b"").await.is_empty(), + "unknown route reached a session" + ); + drop(listener); + assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); + + // Restarting alpha registers a fresh session on the same process. + let (started, output) = cli(&["sandbox", "start", &alpha.name]).await; + assert!(started, "sandbox start failed:\n{output}"); + let restarted_route = session_route(&alpha.name).await; + assert_ne!(restarted_route, alpha_route); + let restarted_call = tokio::time::timeout(Duration::from_secs(120), async { + loop { + match alpha + .exec(&["python3", "-c", &call_script(alpha_endpoint.port, true)]) + .await + { + Ok(output) if output.contains("SAW_ALPHA") => return, + _ => tokio::time::sleep(Duration::from_secs(2)).await, + } + } + }) + .await; + assert!( + restarted_call.is_ok(), + "restarted alpha never regained supervision" + ); + assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); + let listener = RuntimeListener::forward().await; + assert!( + listener.probe(&alpha_route, b"").await.is_empty(), + "the replaced route reached a session" + ); + drop(listener); + + assert_eq!( + shared_supervisor_pod().await, + (supervisor_uid, supervisor_restarts), + "the shared supervisor must keep running throughout" + ); + assert_no_dedicated_supervisors().await; + alpha.cleanup().await; + beta.cleanup().await; + remove_providers().await; +} diff --git a/tasks/test.toml b/tasks/test.toml index 95e049d9ab..f9611b5430 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -266,6 +266,11 @@ description = "Run Kubernetes e2e with managed workspace mode (auto-created per- env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_CREDENTIAL_DRIVER = "kubernetes-secrets", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:shared-supervisor"] +description = "Run Kubernetes e2e with one supervisor shared by the gateway's sandboxes (gateway supervisor topology)" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-supervisor.yaml", OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-shared-supervisor" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:kubernetes:workspace-operator"] description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } From 0e7666de5bd270e9ede5b586e19b1424387ebbe6 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 17:02:44 -0700 Subject: [PATCH 08/25] refactor(supervisor): scope gateway credentials and network state to a session The supervisor kept its gateway bearer, Sandbox Protocol bearer, and renewal task in process-wide slots, and the network supervisor read provider token grants, proxy CA files, and the gateway identity from process environment. A second sandbox in the same process would have replaced or rotated the first sandbox's credentials, reused its provider access tokens, and exposed its denials through policy.local. GatewayCredentials now holds one session's gateway and Sandbox Protocol credentials and their renewal; every gateway client receives a GatewayEndpoint that pairs the URL with them, including log push, ConnectSupervisor and its redirects, and extension-credential rotation. run_networking takes a NetworkSession with the sandbox identity, the session's gateway access, its Workload API socket, its policy.local denial source, and its proxy CA files. Provider token grants use a cache and HTTP client owned by the session and fail closed without a Workload API socket. run_session supervises one generation from an explicit SessionConfig and owner-provided SessionControls. run_sandbox_with_backend remains the dedicated entry point: it keeps the process-wide OCSF context, reads the same environment, binds readiness endpoints to session readiness, and stops on SIGTERM or SIGINT. The binary shares one set of credentials between log push and supervision, as before. Signed-off-by: Drew Newberry --- crates/openshell-core/src/grpc_client.rs | 512 ++++++++++++------ crates/openshell-driver-mxc/src/driver.rs | 2 +- crates/openshell-supervisor-network/README.md | 12 + .../openshell-supervisor-network/src/host.rs | 5 +- .../src/l7/token_grant_injection.rs | 28 +- .../src/policy_local.rs | 169 +++++- .../openshell-supervisor-network/src/proxy.rs | 16 +- .../src/proxy/relay.rs | 9 +- .../openshell-supervisor-network/src/run.rs | 110 +++- .../src/token_grant.rs | 411 +++++++------- .../src/debug_rpc.rs | 7 +- .../src/delegated.rs | 27 +- .../src/log_push.rs | 168 ++++-- .../src/supervisor_session.rs | 39 +- crates/openshell-supervisor/README.md | 18 + .../src/backend_setup/tests.rs | 1 + .../src/config_runtime.rs | 21 +- .../src/legacy_startup_regression_tests.rs | 1 + crates/openshell-supervisor/src/lib.rs | 503 +++++++++++++---- crates/openshell-supervisor/src/main.rs | 36 +- .../src/provider_readiness.rs | 8 +- 21 files changed, 1431 insertions(+), 672 deletions(-) diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index a97deab7d6..80b20b0264 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -5,21 +5,23 @@ //! `OpenShell` server. //! //! Every request carries a sandbox bearer credential in the `Authorization` -//! header. The token is resolved at startup from one of three sources: +//! header. Each supervisor session owns its [`GatewayCredentials`], usually +//! built from the trusted launch bundle with +//! [`GatewayCredentials::from_auth_bundle`]. Diagnostic tools can instead +//! resolve a token with [`GatewayCredentials::from_environment`] from one of: //! //! 1. `OPENSHELL_SANDBOX_TOKEN` — raw JWT in the env (test harness path). -//! 2. `OPENSHELL_SANDBOX_TOKEN_FILE` — file containing the JWT (Docker / -//! Podman / VM drivers write this to a bundle file at sandbox-create -//! time). +//! 2. `OPENSHELL_SANDBOX_TOKEN_FILE` — file containing the JWT. //! 3. `OPENSHELL_K8S_SA_TOKEN_FILE` — projected `ServiceAccount` JWT; the -//! supervisor exchanges it for a gateway JWT via `IssueSandboxToken` -//! once at startup. +//! caller exchanges it for a gateway JWT via `IssueSandboxToken`. //! -//! The resolved bearer credential is held in process memory thereafter and -//! injected on every outbound call by [`AuthInterceptor`]. +//! Callers reach the gateway through a [`GatewayEndpoint`], which pairs the +//! URL with one session's credentials. No credential is installed +//! process-wide, so sessions hosted by one supervisor process cannot use or +//! rotate each other's identity. use std::collections::HashMap; -use std::sync::{Arc, OnceLock, RwLock}; +use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::endpoint_status::{EndpointResult, EndpointStatusSnapshot}; @@ -63,8 +65,8 @@ pub const MAX_SUPERVISOR_STREAM_MESSAGE_BYTES: usize = 9 * 1024 * 1024; /// Shared, refreshable Bearer header. All [`AuthInterceptor`] clones read /// the same slot, so the renewal task can replace the token in place without -/// rebuilding the channel. -type TokenSlot = Arc>; +/// rebuilding the channel. `None` means the owning session revoked it. +type TokenSlot = Arc>>; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum TokenSource { @@ -73,38 +75,222 @@ enum TokenSource { K8sServiceAccount, } -/// Process-wide token slot. Initialized by the first [`connect_channel`] -/// call and shared with every subsequent client and the renewal loop. -static TOKEN_SLOT: OnceLock = OnceLock::new(); +/// Gateway credentials owned by one supervisor session. +/// +/// Clients built from a [`GatewayEndpoint`] read only these slots, and the +/// renewal task started by the first connection updates only them. Dropping +/// the last clone or calling [`GatewayCredentials::revoke`] stops renewal. +#[derive(Clone)] +pub struct GatewayCredentials { + inner: Arc, +} -/// Refresh strategy used by the process-wide token slot. -static TOKEN_REFRESH_MODE: OnceLock = OnceLock::new(); +struct CredentialState { + bearer: TokenSlot, + source: TokenSource, + #[cfg(feature = "jwt")] + sandbox_bearer: Option, + renewal: std::sync::Mutex, +} -/// Serializes the first token acquisition. Several supervisor subsystems -/// connect during startup; without this guard they can all observe an empty -/// [`TOKEN_SLOT`] and perform duplicate K8s bootstrap exchanges. -static TOKEN_INIT_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); +enum Renewal { + Idle, + Started(tokio::task::AbortHandle), + Revoked, +} -/// One-shot guard so the renewal loop spawns at most once per process. -static REFRESH_SPAWNED: OnceLock<()> = OnceLock::new(); +impl Drop for CredentialState { + fn drop(&mut self) { + let renewal = self + .renewal + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Renewal::Started(task) = renewal { + task.abort(); + } + } +} -#[cfg(feature = "jwt")] -static SANDBOX_BEARER_SLOT: OnceLock = OnceLock::new(); +impl std::fmt::Debug for GatewayCredentials { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GatewayCredentials") + .field("source", &self.inner.source) + .finish_non_exhaustive() + } +} -#[derive(Clone, Debug)] -enum RefreshMode { - GatewayJwt(TokenSource), +impl GatewayCredentials { + fn new(bearer: AsciiMetadataValue, source: TokenSource) -> Self { + Self { + inner: Arc::new(CredentialState { + bearer: Arc::new(RwLock::new(Some(bearer))), + source, + #[cfg(feature = "jwt")] + sandbox_bearer: None, + renewal: std::sync::Mutex::new(Renewal::Idle), + }), + } + } + + /// Credentials issued for one exact sandbox launch. Renewal also rotates + /// the bundle's Sandbox Protocol credential, returned by + /// [`GatewayCredentials::sandbox_bearer`]. + #[cfg(feature = "jwt")] + pub fn from_auth_bundle(bundle: &crate::jwt::SupervisorAuthBundle) -> Result { + let bearer = validate_gateway_bearer(bundle.gateway_token.expose_secret())?; + let sandbox_bearer = bundle + .sandbox_bearer_slot() + .into_diagnostic() + .wrap_err("invalid Sandbox Protocol credential")?; + Ok(Self { + inner: Arc::new(CredentialState { + bearer: Arc::new(RwLock::new(Some(bearer))), + source: TokenSource::File, + sandbox_bearer: Some(sandbox_bearer), + renewal: std::sync::Mutex::new(Renewal::Idle), + }), + }) + } + + /// Use a caller-supplied gateway JWT. Renewal cannot rebootstrap it. + pub fn from_gateway_token(token: &str) -> Result { + Ok(Self::new(validate_gateway_bearer(token)?, TokenSource::Env)) + } + + /// Resolve a token for diagnostic tools from the environment, exchanging a + /// projected `ServiceAccount` token when no gateway JWT is present. + pub async fn from_environment(endpoint: &str) -> Result { + let plain_channel = build_plain_channel(endpoint).await?; + let acquired = acquire_sandbox_token(endpoint, &plain_channel).await?; + Ok(Self::new( + validate_gateway_bearer(&acquired.token)?, + acquired.source, + )) + } + + /// The Sandbox Protocol credential renewed with these gateway credentials. + #[cfg(feature = "jwt")] + #[must_use] + pub fn sandbox_bearer(&self) -> Option { + self.inner.sandbox_bearer.clone() + } + + /// Clear both credentials and stop renewal. Existing clients fail their + /// next request locally instead of presenting a revoked identity. + pub fn revoke(&self) { + let mut renewal = self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Renewal::Started(task) = &*renewal { + task.abort(); + } + *renewal = Renewal::Revoked; + *self + .inner + .bearer + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + #[cfg(feature = "jwt")] + if let Some(sandbox_bearer) = &self.inner.sandbox_bearer { + sandbox_bearer.clear(); + } + } + + /// Start the renewal task once for these credentials, on the caller's + /// runtime, so a hosted session's renewal stops with that session. + fn start_renewal(&self, channel: &AuthedChannel, plain_channel: &Channel, endpoint: &str) { + let mut renewal = self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !matches!(*renewal, Renewal::Idle) { + return; + } + let task = tokio::spawn(refresh_token_loop(RenewalTarget { + channel: channel.clone(), + bearer: self.inner.bearer.clone(), + #[cfg(feature = "jwt")] + sandbox_bearer: self.inner.sandbox_bearer.clone(), + source: self.inner.source, + endpoint: endpoint.to_string(), + plain_channel: plain_channel.clone(), + })); + *renewal = Renewal::Started(task.abort_handle()); + } + + fn is_revoked(&self) -> bool { + matches!( + *self + .inner + .renewal + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + Renewal::Revoked + ) + } + + fn install_gateway_bearer(&self, bearer: AsciiMetadataValue) { + let mut slot = self + .inner + .bearer + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // A revoked session must not be revived by an in-flight response. + if slot.is_some() { + *slot = Some(bearer); + } + } +} + +/// The gateway URL a session uses, paired with that session's credentials. +#[derive(Clone)] +pub struct GatewayEndpoint { + url: Arc, + credentials: GatewayCredentials, +} + +impl GatewayEndpoint { + pub fn new(url: impl Into>, credentials: GatewayCredentials) -> Self { + Self { + url: url.into(), + credentials, + } + } + + #[must_use] + pub fn url(&self) -> &str { + &self.url + } + + #[must_use] + pub fn credentials(&self) -> &GatewayCredentials { + &self.credentials + } + + /// Reach another gateway address with the same session credentials. + #[must_use] + pub fn with_url(&self, url: impl Into>) -> Self { + Self::new(url, self.credentials.clone()) + } +} + +impl std::fmt::Debug for GatewayEndpoint { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GatewayEndpoint") + .field("url", &self.url) + .finish_non_exhaustive() + } } #[derive(Debug)] struct AcquiredToken { token: String, - refresh_mode: RefreshMode, -} - -fn install_token_slot(token: &str) -> Result { - let bearer = validate_gateway_bearer(token)?; - Ok(install_validated_token_slot(bearer)) + source: TokenSource, } fn validate_gateway_bearer(token: &str) -> Result { @@ -113,18 +299,6 @@ fn validate_gateway_bearer(token: &str) -> Result { .wrap_err("sandbox JWT contained characters not valid for a header value") } -fn install_validated_token_slot(bearer: AsciiMetadataValue) -> TokenSlot { - if let Some(existing) = TOKEN_SLOT.get() { - *existing - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = bearer; - return existing.clone(); - } - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); - let _ = TOKEN_SLOT.set(slot.clone()); - TOKEN_SLOT.get().cloned().unwrap_or(slot) -} - #[cfg(feature = "jwt")] struct ValidatedSandboxRefresh { token: crate::jwt::SecretJwt, @@ -159,22 +333,6 @@ fn validate_sandbox_refresh( }) } -/// Install the gateway-session credential supplied in trusted supervisor -/// launch state before any gateway client is constructed. -#[cfg(feature = "jwt")] -pub fn install_supervisor_auth_bundle( - bundle: &crate::jwt::SupervisorAuthBundle, -) -> Result { - install_token_slot(bundle.gateway_token.expose_secret())?; - let _ = TOKEN_REFRESH_MODE.set(RefreshMode::GatewayJwt(TokenSource::File)); - let slot = bundle - .sandbox_bearer_slot() - .into_diagnostic() - .wrap_err("invalid Sandbox Protocol credential")?; - let _ = SANDBOX_BEARER_SLOT.set(slot.clone()); - Ok(SANDBOX_BEARER_SLOT.get().cloned().unwrap_or(slot)) -} - /// gRPC interceptor that injects `authorization: Bearer ` on every /// outbound request. The token lives in a shared [`TokenSlot`] so the renewal /// task can replace it without rebuilding clients. @@ -197,8 +355,9 @@ impl tonic::service::Interceptor for AuthInterceptor { let bearer = self .bearer .read() - .expect("auth interceptor token slot poisoned") - .clone(); + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + .ok_or_else(|| Status::unauthenticated("gateway session credentials were revoked"))?; req.metadata_mut().insert("authorization", bearer); #[cfg(feature = "trace-context")] let req = @@ -299,56 +458,23 @@ impl Drop for ClientSpanStatus { } } -/// Build a Bearer-authenticated channel to the gateway. -/// -/// First call per process resolves the sandbox JWT via the three-step -/// lookup (env → file → K8s SA bootstrap exchange) and installs it into -/// the process-wide [`TOKEN_SLOT`]. Subsequent calls reuse the cached -/// slot — the renewal loop keeps the value fresh, so re-running the -/// bootstrap is both unnecessary and (on the K8s SA path) expensive -/// (one apiserver round-trip per call). The renewal loop itself is -/// spawned once per process via [`REFRESH_SPAWNED`]. -async fn connect_channel(endpoint: &str) -> Result { - let channel = build_plain_channel(endpoint).await?; - let (slot, refresh_mode) = token_slot(endpoint, &channel).await?; - let plain_channel = channel.clone(); - let intercepted = InterceptedService::new(channel, AuthInterceptor::new(slot.clone())); - if REFRESH_SPAWNED.set(()).is_ok() { - let RefreshMode::GatewayJwt(source) = refresh_mode; - let refresh_channel = intercepted.clone(); - let endpoint = endpoint.to_string(); - tokio::spawn(async move { - refresh_token_loop(refresh_channel, slot, source, endpoint, plain_channel).await; - }); +/// Build a Bearer-authenticated channel to the gateway with one session's +/// credentials. The first connection starts that session's renewal task. +async fn connect_channel(gateway: &GatewayEndpoint) -> Result { + if gateway.credentials.is_revoked() { + return Err(miette::miette!("gateway session credentials were revoked")); } + let channel = build_plain_channel(gateway.url()).await?; + let intercepted = InterceptedService::new( + channel.clone(), + AuthInterceptor::new(gateway.credentials.inner.bearer.clone()), + ); + gateway + .credentials + .start_renewal(&intercepted, &channel, gateway.url()); Ok(intercepted) } -async fn token_slot(endpoint: &str, plain_channel: &Channel) -> Result<(TokenSlot, RefreshMode)> { - if let Some(existing) = TOKEN_SLOT.get() { - let refresh_mode = TOKEN_REFRESH_MODE - .get() - .cloned() - .unwrap_or(RefreshMode::GatewayJwt(TokenSource::Env)); - return Ok((existing.clone(), refresh_mode)); - } - - let _guard = TOKEN_INIT_LOCK.lock().await; - - if let Some(existing) = TOKEN_SLOT.get() { - let refresh_mode = TOKEN_REFRESH_MODE - .get() - .cloned() - .unwrap_or(RefreshMode::GatewayJwt(TokenSource::Env)); - return Ok((existing.clone(), refresh_mode)); - } - - let acquired = acquire_sandbox_token(endpoint, plain_channel).await?; - let slot = install_token_slot(&acquired.token)?; - let _ = TOKEN_REFRESH_MODE.set(acquired.refresh_mode.clone()); - Ok((slot, acquired.refresh_mode)) -} - /// Resolve the sandbox JWT used to authenticate every outbound RPC. /// /// `endpoint` is logged on errors but never used for transport here; the @@ -362,7 +488,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul debug!(source = "env", "loaded sandbox token"); return Ok(AcquiredToken { token: t, - refresh_mode: RefreshMode::GatewayJwt(TokenSource::Env), + source: TokenSource::Env, }); } @@ -375,7 +501,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul debug!(source = "file", path = %path, "loaded sandbox token"); return Ok(AcquiredToken { token: contents.trim().to_string(), - refresh_mode: RefreshMode::GatewayJwt(TokenSource::File), + source: TokenSource::File, }); } @@ -384,7 +510,7 @@ async fn acquire_sandbox_token(endpoint: &str, plain_channel: &Channel) -> Resul { return Ok(AcquiredToken { token: acquire_k8s_sandbox_token(endpoint, plain_channel, &sa_path).await?, - refresh_mode: RefreshMode::GatewayJwt(TokenSource::K8sServiceAccount), + source: TokenSource::K8sServiceAccount, }); } @@ -407,14 +533,13 @@ async fn acquire_k8s_sandbox_token( .trim() .to_string(); info!(endpoint = %endpoint, "exchanging K8s ServiceAccount token for sandbox JWT"); - // The bootstrap exchange uses a one-off interceptor pinned to the - // SA token; the resulting gateway JWT becomes the value in the - // shared `TOKEN_SLOT` once `connect_channel` returns. - let bootstrap_slot: TokenSlot = Arc::new(RwLock::new( + // The bootstrap exchange uses a one-off interceptor pinned to the SA + // token; the caller installs the resulting gateway JWT. + let bootstrap_slot: TokenSlot = Arc::new(RwLock::new(Some( AsciiMetadataValue::try_from(format!("Bearer {sa_token}")) .into_diagnostic() .wrap_err("SA token contained characters not valid for a header value")?, - )); + ))); let interceptor = AuthInterceptor::new(bootstrap_slot); let bootstrap = InterceptedService::new(plain_channel.clone(), interceptor); let mut client = OpenShellClient::new(bootstrap); @@ -428,8 +553,8 @@ async fn acquire_k8s_sandbox_token( /// Build an authenticated channel for direct external use (e.g. the /// long-lived `supervisor_session` control stream). -pub async fn connect_channel_pub(endpoint: &str) -> Result { - connect_channel(endpoint).await +pub async fn connect_channel_pub(gateway: &GatewayEndpoint) -> Result { + connect_channel(gateway).await } /// Report installed provider state for the current authenticated supervisor session. @@ -437,11 +562,11 @@ pub async fn connect_channel_pub(endpoint: &str) -> Result { /// The observation must carry the session ID returned by `ConnectSupervisor`. /// Reconnects start a new report sequence; retries preserve the complete report. pub async fn report_provider_readiness( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, observation: crate::proto::ProviderReadinessObservation, ) -> Result { - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; client .report_provider_readiness(crate::proto::ReportProviderReadinessRequest { sandbox_id: sandbox_id.to_string(), @@ -452,18 +577,33 @@ pub async fn report_provider_readiness( .into_diagnostic() } -/// Background task that renews the sandbox JWT at ~80% of its remaining -/// lifetime. The new token replaces the value in [`TOKEN_SLOT`], so all -/// in-flight and future clients pick it up on their next request. The -/// loop never panics: every failure is logged and re-attempted after a -/// bounded backoff. -async fn refresh_token_loop( +/// Slots and transport retained by one session's renewal task. It holds no +/// reference to [`GatewayCredentials`], so dropping them still stops it. +struct RenewalTarget { channel: AuthedChannel, - slot: TokenSlot, + bearer: TokenSlot, + #[cfg(feature = "jwt")] + sandbox_bearer: Option, source: TokenSource, endpoint: String, plain_channel: Channel, -) { +} + +/// Background task that renews the sandbox JWT at ~80% of its remaining +/// lifetime. The new token replaces the value in the session's slot, so all +/// in-flight and future clients of that session pick it up on their next +/// request. The loop never panics: every failure is logged and re-attempted +/// after a bounded backoff. +async fn refresh_token_loop(target: RenewalTarget) { + let RenewalTarget { + channel, + bearer: slot, + #[cfg(feature = "jwt")] + sandbox_bearer, + source, + endpoint, + plain_channel, + } = target; let mut client = OpenShellClient::new(channel); loop { let Some(sleep) = compute_refresh_delay(&slot) else { @@ -495,11 +635,17 @@ async fn refresh_token_loop( } }; - *slot - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) = gateway_bearer; + { + let mut current = slot + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.is_none() { + return; + } + *current = Some(gateway_bearer); + } #[cfg(feature = "jwt")] - if let Some(sandbox_slot) = SANDBOX_BEARER_SLOT.get() + if let Some(sandbox_slot) = sandbox_bearer.as_ref() && let Err(error) = sandbox_slot.update( sandbox_refresh.token, sandbox_refresh.expires_at, @@ -523,8 +669,10 @@ async fn refresh_token_loop( Ok(new_token) => { match AsciiMetadataValue::try_from(format!("Bearer {new_token}")) { Ok(value) => { - if let Ok(mut guard) = slot.write() { - *guard = value; + if let Ok(mut guard) = slot.write() + && guard.is_some() + { + *guard = Some(value); info!( "rebootstrapped gateway sandbox JWT after refresh authentication failure" ); @@ -585,6 +733,7 @@ fn authenticated_service_names( async fn refresh_extension_credentials_with_client( client: &mut OpenShellClient, + credentials: &GatewayCredentials, store: &ExtensionCredentialStore, services: &[crate::proto::SupervisorMiddlewareService], ) -> Result> { @@ -645,9 +794,9 @@ async fn refresh_extension_credentials_with_client( // Commit only after the gateway, Sandbox Protocol, and extension // credentials have all been parsed and validated. The remaining updates // repeat those validations but cannot fail for the validated inputs. - install_validated_token_slot(gateway_bearer); + credentials.install_gateway_bearer(gateway_bearer); #[cfg(feature = "jwt")] - if let Some(sandbox_slot) = SANDBOX_BEARER_SLOT.get() { + if let Some(sandbox_slot) = credentials.inner.sandbox_bearer.as_ref() { match sandbox_slot.update( sandbox_refresh.token, sandbox_refresh.expires_at, @@ -679,11 +828,14 @@ async fn refresh_extension_credentials_with_client( /// non-expiring session credential and needs no periodic refresh. If the token /// can't be parsed (for example, an opaque bootstrap bearer), default to 6 h. fn compute_refresh_delay(slot: &TokenSlot) -> Option { - let token = slot - .read() - .ok() - .and_then(|v| v.to_str().ok().map(str::to_string)) - .unwrap_or_default(); + let token = { + let current = slot + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + // A revoked session has nothing to renew. + current.as_ref()?.to_str().ok().map(str::to_string) + } + .unwrap_or_default(); let now_ms = i64::try_from( SystemTime::now() .duration_since(UNIX_EPOCH) @@ -819,7 +971,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(payload_json); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expiring token needs refresh"); // 800 s baseline + up to 10 % jitter → 800..=880 s, with some slack // for the 1-second resolution of the exp claim. @@ -840,7 +992,7 @@ mod auth_tests { base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(format!(r#"{{"exp":{exp}}}"#)); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expired token needs refresh"); assert!((1..60).contains(&delay.as_secs())); } @@ -851,7 +1003,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(r#"{"exp":0}"#); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); assert_eq!(compute_refresh_delay(&slot), None); } @@ -867,7 +1019,7 @@ mod auth_tests { let payload = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(payload_json); let token = format!("h.{payload}.s"); let bearer = AsciiMetadataValue::try_from(format!("Bearer {token}")).unwrap(); - let slot: TokenSlot = Arc::new(RwLock::new(bearer)); + let slot: TokenSlot = Arc::new(RwLock::new(Some(bearer))); let delay = compute_refresh_delay(&slot).expect("expiring token needs refresh"); assert!( delay.as_secs() < 30, @@ -906,9 +1058,9 @@ mod workspace_tests { } } -/// Connect to the `OpenShell` server. -async fn connect(endpoint: &str) -> Result> { - let channel = connect_channel(endpoint).await?; +/// Connect to the `OpenShell` server with one session's credentials. +async fn connect(gateway: &GatewayEndpoint) -> Result> { + let channel = connect_channel(gateway).await?; Ok(OpenShellClient::new(channel)) } @@ -918,12 +1070,12 @@ async fn connect(endpoint: &str) -> Result> { /// or `Ok(None)` when the sandbox was created without a policy (the sandbox /// should discover one from disk or use the restrictive default). pub async fn fetch_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_name: &str, ) -> Result> { - debug!(endpoint = %endpoint, sandbox_name = %sandbox_name, "Connecting to OpenShell server"); + debug!(endpoint = %gateway.url(), sandbox_name = %sandbox_name, "Connecting to OpenShell server"); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; debug!("Connected, fetching sandbox policy"); @@ -942,12 +1094,12 @@ pub async fn fetch_policy( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn fetch_settings_snapshot( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_name: &str, ) -> Result { let status = ClientSpanStatus::current(); - debug!(endpoint = %endpoint, sandbox_name = %sandbox_name, "Connecting to fetch OpenShell settings snapshot"); - let mut client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), sandbox_name = %sandbox_name, "Connecting to fetch OpenShell settings snapshot"); + let mut client = connect(gateway).await?; status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox_name, None).await) } @@ -1014,18 +1166,18 @@ async fn sync_policy_with_client( /// Performs the full discovery flow (fetch → sync → re-fetch) over one /// channel instead of establishing three separate connections. pub async fn discover_and_sync_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, discovered_policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result { debug!( - endpoint = %endpoint, + endpoint = %gateway.url(), sandbox = %sandbox, "Syncing discovered policy and re-fetching canonical version" ); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; // Sync the discovered policy to the gateway. sync_policy_with_client(&mut client, sandbox, discovered_policy, workspace).await?; @@ -1044,13 +1196,13 @@ pub async fn discover_and_sync_policy( /// Used by the supervisor to push baseline-path-enriched policies so the /// gateway stores the effective policy users see via `openshell sandbox get`. pub async fn sync_policy( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result<()> { - debug!(endpoint = %endpoint, sandbox = %sandbox, "Syncing enriched policy to gateway"); - let mut client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), sandbox = %sandbox, "Syncing enriched policy to gateway"); + let mut client = connect(gateway).await?; sync_policy_with_client(&mut client, sandbox, policy, workspace).await } @@ -1061,13 +1213,13 @@ pub async fn sync_policy( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn sync_policy_and_fetch_snapshot( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox: &str, policy: &ProtoSandboxPolicy, workspace: &str, ) -> Result { let status = ClientSpanStatus::current(); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; sync_policy_with_client(&mut client, sandbox, policy, workspace).await?; status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox, Some(workspace)).await) } @@ -1080,7 +1232,7 @@ pub async fn sync_policy_and_fetch_snapshot( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn report_sandbox_configuration( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, instance_id: &str, snapshot: Option<&SettingsPollResult>, @@ -1088,7 +1240,7 @@ pub async fn report_sandbox_configuration( error: &str, ) -> Result<()> { let status = ClientSpanStatus::current(); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; client .report_sandbox_configuration(crate::proto::ReportSandboxConfigurationRequest { sandbox_id: sandbox_id.to_string(), @@ -1118,10 +1270,10 @@ pub async fn report_sandbox_configuration( /// environment represents a sandbox without provider credentials. Transport /// failure returns an error so callers can revoke credentials and retry. pub async fn fetch_provider_environment( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, ) -> Result { - fetch_provider_environment_with_stable_placeholders(endpoint, sandbox_id, false).await + fetch_provider_environment_with_stable_placeholders(gateway, sandbox_id, false).await } /// Fetch a snapshot without downgrading previously activated external stable credentials. @@ -1135,14 +1287,14 @@ pub async fn fetch_provider_environment( fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn fetch_provider_environment_with_stable_placeholders( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { let status = ClientSpanStatus::current(); - debug!(endpoint = %endpoint, sandbox_id = %sandbox_id, "Fetching provider environment"); + debug!(endpoint = %gateway.url(), sandbox_id = %sandbox_id, "Fetching provider environment"); - let mut client = connect(endpoint).await?; + let mut client = connect(gateway).await?; let response = client .get_sandbox_provider_environment(GetSandboxProviderEnvironmentRequest { @@ -1366,7 +1518,7 @@ mod provider_environment_tests { fields(otel.kind = "client", otel.status_code = tracing::field::Empty) )] pub async fn exchange_provider_subject_token( - endpoint: &str, + gateway: &GatewayEndpoint, sandbox_id: &str, provider: &str, credential_key: &str, @@ -1374,7 +1526,7 @@ pub async fn exchange_provider_subject_token( request_id: Option<&str>, ) -> std::result::Result { let status = ClientSpanStatus::current(); - let endpoint = token_grant_gateway_endpoint(endpoint)?; + let endpoint = token_grant_gateway_endpoint(gateway.url())?; if sandbox_id.is_empty() || provider.is_empty() || credential_key.is_empty() { return Err(gateway_grant_failure(TokenGrantFailureKind::Configuration)); } @@ -1388,7 +1540,7 @@ pub async fn exchange_provider_subject_token( // Discard those sources at the grant boundary; never diagnose from prose. // The shared channel bounds the connection attempt; the gateway bounds its // token-service calls, and a gateway deadline arrives as DeadlineExceeded. - let mut client = connect(endpoint.as_str()) + let mut client = connect(&gateway.with_url(endpoint.as_str())) .await .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Unclassified))?; let request = provider_subject_token_exchange_request( @@ -1559,6 +1711,8 @@ mod token_grant_exchange_tests { #[derive(Clone)] pub struct CachedOpenShellClient { client: OpenShellClient, + /// Session credentials rotated alongside extension credentials. + credentials: GatewayCredentials, workspace: Arc>, /// Extension credentials for this supervisor. Cloning the client shares /// the store, so the middleware registry and the polling loop that rotates @@ -1887,8 +2041,8 @@ pub struct ProviderSubjectTokenExchangeResult { } impl CachedOpenShellClient { - pub async fn connect(endpoint: &str) -> Result { - Self::connect_with_credentials(endpoint, ExtensionCredentialStore::new()).await + pub async fn connect(gateway: &GatewayEndpoint) -> Result { + Self::connect_with_credentials(gateway, ExtensionCredentialStore::new()).await } /// Connect while sharing an existing credential store. @@ -1898,13 +2052,14 @@ impl CachedOpenShellClient { /// credentials handed to the middleware registry are not the ones the loop /// rotates. pub async fn connect_with_credentials( - endpoint: &str, + gateway: &GatewayEndpoint, extension_credentials: ExtensionCredentialStore, ) -> Result { - debug!(endpoint = %endpoint, "Connecting openshell gRPC client for policy polling"); - let client = connect(endpoint).await?; + debug!(endpoint = %gateway.url(), "Connecting openshell gRPC client for policy polling"); + let client = connect(gateway).await?; Ok(Self { client, + credentials: gateway.credentials.clone(), workspace: Arc::new(tokio::sync::OnceCell::new()), extension_credentials, }) @@ -1972,6 +2127,7 @@ impl CachedOpenShellClient { let mut client = self.client.clone(); refresh_extension_credentials_with_client( &mut client, + &self.credentials, &self.extension_credentials, services, ) diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index c7a966a32b..2772f1a69c 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -870,7 +870,7 @@ async fn run_lifecycle( binary_path: host_proxy_binary_path(&sandbox_config), sandbox_id: Some(sandbox_id.clone()), sandbox_name: Some(sandbox_name.clone()), - openshell_endpoint: None, + gateway: None, provider_credentials: None, agent_proposals: openshell_core::proposals::AgentProposals::default(), denial_tx: None, diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 30f404752a..b191655d1a 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -14,3 +14,15 @@ allocator implementation belongs to the VM crate. The VM dependency disables default features, so this instrumentation does not require the compute runtime or Unix-only sandbox library. It is a development dependency only. + +## Session inputs + +`run_networking` takes a `NetworkSession` with the sandbox identity, the +session's `GatewayEndpoint`, the SPIFFE Workload API socket for provider token +grants, the `policy.local` denial source, and proxy CA files. None of these +are read from the process environment, so one process can run several +sessions. Each session owns its provider token cache and HTTP client; a token +acquired for one sandbox is never served to another. Without a Workload API +socket, token grants fail closed. A process hosting several sessions serves +`/v1/denials` from a per-session `SessionDenialLog` instead of the shared +shorthand log files. diff --git a/crates/openshell-supervisor-network/src/host.rs b/crates/openshell-supervisor-network/src/host.rs index b07bc0f537..5f111acb21 100644 --- a/crates/openshell-supervisor-network/src/host.rs +++ b/crates/openshell-supervisor-network/src/host.rs @@ -48,7 +48,8 @@ pub struct HostProxyConfig { pub binary_path: PathBuf, pub sandbox_id: Option, pub sandbox_name: Option, - pub openshell_endpoint: Option, + /// Gateway access with this sandbox's credentials. + pub gateway: Option, pub provider_credentials: Option, /// Shared feature state for the policy.local agent proposal surface. pub agent_proposals: AgentProposals, @@ -99,7 +100,7 @@ pub async fn start_host_proxy(config: HostProxyConfig) -> Result Pin> + Send + 'a>>; } -/// Resolves provider tokens using the supervisor's authenticated grant flow. -#[derive(Default)] -pub struct SpiffeTokenGrantResolver; +/// Resolves provider tokens through one session's authenticated grant flow. +pub struct SpiffeTokenGrantResolver { + grants: crate::token_grant::SessionTokenGrants, +} + +impl SpiffeTokenGrantResolver { + #[must_use] + pub fn new(grants: crate::token_grant::SessionTokenGrants) -> Self { + Self { grants } + } +} impl TokenGrantResolver for SpiffeTokenGrantResolver { fn obtain<'a>( @@ -63,8 +71,8 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { request: TokenGrantRequest<'a>, ) -> Pin> + Send + 'a>> { Box::pin(async move { - crate::token_grant::obtain_provider_token( - crate::token_grant::ObtainProviderTokenRequest { + self.grants + .obtain_provider_token(crate::token_grant::ObtainProviderTokenRequest { provider_name: request.provider_key, token_endpoint: request.token_endpoint, jwt_svid_audience: request.jwt_svid_audience, @@ -75,18 +83,12 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { grant_type: request.grant_type, requested_token_type: request.requested_token_type, request_id: request.request_id, - }, - ) - .await + }) + .await }) } } -/// Construct the production SPIFFE-backed resolver. -pub fn default_resolver() -> Arc { - Arc::new(SpiffeTokenGrantResolver) -} - /// A refused request: the grant failure and the credential whose grant failed. /// /// The credential is the `provider:credential` name pair of the selected diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index c3bb88b4ed..a581800c15 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -3,7 +3,9 @@ //! Sandbox-local policy advisor HTTP API. +use crate::l7::token_grant_injection::TokenGrantResolver; use miette::{IntoDiagnostic, Result}; +use openshell_core::grpc_client::GatewayEndpoint; use openshell_core::proposals::AgentProposals; use openshell_core::proto::{ L7Allow, L7DenyRule, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, PolicyChunk, @@ -82,50 +84,153 @@ const SHORTHAND_LOG_PREFIX: &str = "openshell."; /// log entry (very long URL path, etc.) cannot blow up the response. const MAX_DENIAL_LINE_BYTES: usize = 4096; -#[derive(Debug)] +/// Most recent denial lines retained for one hosted session. +const MAX_SESSION_DENIAL_LINES: usize = 256; + +/// Where `/v1/denials` reads this sandbox's recent denials. +#[derive(Clone, Debug)] +pub enum DenialLogSource { + /// Shorthand log files written by a supervisor process that serves one + /// sandbox. + ShorthandFiles(PathBuf), + /// Denials recorded for this session alone. A process hosting several + /// sandboxes cannot expose its shared log files to any one of them. + Session(SessionDenialLog), +} + +impl Default for DenialLogSource { + fn default() -> Self { + Self::ShorthandFiles(PathBuf::from(LOG_DIR)) + } +} + +/// Bounded recent OCSF denial lines recorded for one hosted session. +#[derive(Clone, Default)] +pub struct SessionDenialLog { + lines: Arc>>, +} + +impl std::fmt::Debug for SessionDenialLog { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let retained = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len(); + formatter + .debug_struct("SessionDenialLog") + .field("retained", &retained) + .finish() + } +} + +impl SessionDenialLog { + /// Retain `line` when it is a rendered OCSF denial; ignore anything else. + pub fn record(&self, line: &str) { + let line = line.trim_end(); + if !is_ocsf_denial_line(line) { + return; + } + let mut lines = self + .lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if lines.len() == MAX_SESSION_DENIAL_LINES { + lines.pop_front(); + } + lines.push_back(line.to_string()); + } + + fn recent(&self, limit: usize) -> Vec { + self.lines + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .rev() + .take(limit) + .map(|line| surfaced_denial_line(line)) + .collect() + } +} + pub struct PolicyLocalContext { current_policy: Arc>>>, agent_proposals: AgentProposals, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, - shorthand_log_dir: PathBuf, + denials: DenialLogSource, + token_grants: Option>, workspace_rx: tokio::sync::watch::Receiver, } +impl std::fmt::Debug for PolicyLocalContext { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PolicyLocalContext") + .field("gateway", &self.gateway) + .field("sandbox_name", &self.sandbox_name) + .field("denials", &self.denials) + .field("token_grants", &self.token_grants.is_some()) + .finish_non_exhaustive() + } +} + impl PolicyLocalContext { pub fn new( current_policy: Option, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, ) -> Self { - Self::with_log_dir( - current_policy, - gateway_endpoint, - sandbox_name, + Self { + current_policy: Arc::new(RwLock::new(current_policy.map(Arc::new))), agent_proposals, - PathBuf::from(LOG_DIR), + gateway, + sandbox_name, + denials: DenialLogSource::default(), + token_grants: None, workspace_rx, - ) + } } + #[cfg(test)] fn with_log_dir( current_policy: Option, - gateway_endpoint: Option, + gateway: Option, sandbox_name: Option, agent_proposals: AgentProposals, shorthand_log_dir: PathBuf, workspace_rx: tokio::sync::watch::Receiver, ) -> Self { - Self { - current_policy: Arc::new(RwLock::new(current_policy.map(Arc::new))), - agent_proposals, - gateway_endpoint, + Self::new( + current_policy, + gateway, sandbox_name, - shorthand_log_dir, + agent_proposals, workspace_rx, - } + ) + .with_denial_log(DenialLogSource::ShorthandFiles(shorthand_log_dir)) + } + + /// Serve `/v1/denials` from `denials` instead of the process log files. + #[must_use] + pub fn with_denial_log(mut self, denials: DenialLogSource) -> Self { + self.denials = denials; + self + } + + /// Acquire provider token grants for this sandbox's HTTP relays. + #[must_use] + pub fn with_token_grants(mut self, resolver: Arc) -> Self { + self.token_grants = Some(resolver); + self + } + + /// The session's provider token grant resolver, absent when grants are + /// unavailable and must fail closed. + pub(crate) fn token_grant_resolver(&self) -> Option> { + self.token_grants.clone() } pub async fn set_current_policy(&self, policy: ProtoSandboxPolicy) { @@ -318,7 +423,18 @@ async fn recent_denials_response( query: &str, ) -> (u16, serde_json::Value) { let limit = parse_last_query(query).unwrap_or(DEFAULT_DENIALS_LIMIT); - let log_dir = ctx.shorthand_log_dir.clone(); + let log_dir = match &ctx.denials { + DenialLogSource::ShorthandFiles(log_dir) => log_dir.clone(), + DenialLogSource::Session(log) => { + return ( + 200, + serde_json::json!({ + "denials": log.recent(limit), + "log_available": true, + }), + ); + } + }; // Distinguish "shorthand log exists and no denials happened" from "no log // file yet, so we have nothing to read." Without this flag the agent sees @@ -398,9 +514,7 @@ fn read_recent_denial_lines(log_dir: &Path, limit: usize) -> Vec { // proxy.rs FORWARD path so the on-disk shorthand log itself is // clean — tracked separately). Redact first so truncation cannot // slice mid-secret. - let redacted = redact_query_strings(line); - let surfaced = truncate_at_char_boundary(&redacted, MAX_DENIAL_LINE_BYTES); - lines.push(surfaced); + lines.push(surfaced_denial_line(line)); if lines.len() >= limit { return lines; } @@ -409,6 +523,11 @@ fn read_recent_denial_lines(log_dir: &Path, limit: usize) -> Vec { lines } +fn surfaced_denial_line(line: &str) -> String { + let redacted = redact_query_strings(line); + truncate_at_char_boundary(&redacted, MAX_DENIAL_LINE_BYTES) +} + /// Replace any `?` substring with `?[redacted]` to keep query-string /// secrets out of the agent's view. Walks per Unicode scalar value so multi-byte /// content is safe. A query is everything from `?` until the next whitespace or @@ -486,7 +605,7 @@ fn collect_shorthand_log_files(log_dir: &Path, max_files: usize) -> std::io::Res } async fn submit_proposal(ctx: &PolicyLocalContext, body: &[u8]) -> (u16, serde_json::Value) { - let Some(endpoint) = ctx.gateway_endpoint.as_deref() else { + let Some(gateway) = ctx.gateway.as_ref() else { return ( 503, serde_json::json!({ @@ -526,7 +645,7 @@ async fn submit_proposal(ctx: &PolicyLocalContext, body: &[u8]) -> (u16, serde_j Err(error) => return (400, error_payload("invalid_proposal", error)), }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(gateway).await { Ok(client) => { client.set_workspace(workspace); client @@ -956,7 +1075,7 @@ struct LookupSession<'a> { async fn open_lookup_session( ctx: &PolicyLocalContext, ) -> std::result::Result, (u16, serde_json::Value)> { - let endpoint = ctx.gateway_endpoint.as_deref().ok_or_else(|| { + let gateway = ctx.gateway.as_ref().ok_or_else(|| { ( 503, error_payload( @@ -989,7 +1108,7 @@ async fn open_lookup_session( ), )); } - let client = openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint) + let client = openshell_core::grpc_client::CachedOpenShellClient::connect(gateway) .await .map_err(|e| (502, error_payload("gateway_connect_failed", e.to_string())))?; client.set_workspace(workspace); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index ffdb4674d1..bd93c78c57 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1227,11 +1227,11 @@ async fn handle_transparent_tcp_connection( &decision, None, None, - agent_proposals, // The transparent TCP path carries no PolicyLocalContext, so no // workspace is available here; matches the CONNECT path default when // policy-local context is absent. - String::new(), + None, + agent_proposals, relay::RelaySignals { activity: activity_tx.clone(), endpoint_observation: None, @@ -3019,16 +3019,12 @@ async fn handle_mediated_connection( // gate needs it) and drives the raw-tunnel branch below. // Build request-processing context shared by CONNECT and forward HTTP. - let workspace = policy_local_ctx - .as_ref() - .map(|ctx| ctx.workspace()) - .unwrap_or_default(); let mut ctx = relay::http_context( &decision, provider_credentials, secret_resolver.clone(), + policy_local_ctx.as_deref(), agent_proposals, - workspace, relay::RelaySignals { activity: activity_tx.clone(), endpoint_observation: endpoint_observation_tx, @@ -5673,16 +5669,12 @@ async fn handle_forward_proxy( .map_or_else(std::collections::HashMap::new, |query| { crate::l7::rest::parse_query_params(query).unwrap_or_default() }); - let workspace = policy_local_ctx - .as_ref() - .map(|ctx| ctx.workspace()) - .unwrap_or_default(); let mut l7_ctx = relay::http_context( &decision, provider_credentials, secret_resolver.clone(), + policy_local_ctx.as_deref(), agent_proposals, - workspace, relay::RelaySignals { activity: activity_tx.cloned(), endpoint_observation: endpoint_observation_tx, diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index dad4b12633..05f685e122 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -47,8 +47,8 @@ pub(super) fn http_context( decision: &EgressDecision, provider_credentials: Option, secret_resolver: Option>, + policy_local: Option<&crate::policy_local::PolicyLocalContext>, agent_proposals: openshell_core::proposals::AgentProposals, - workspace: String, signals: RelaySignals, ) -> L7EvalContext { // Provider-backed credentials must be acquired from the live state for @@ -58,9 +58,14 @@ pub(super) fn http_context( .is_none() .then_some(secret_resolver) .flatten(); + // Grants use the owning session's resolver; without one they fail closed. let token_grant_resolver = provider_credentials .as_ref() - .map(|_| crate::l7::token_grant_injection::default_resolver()); + .and(policy_local) + .and_then(crate::policy_local::PolicyLocalContext::token_grant_resolver); + let workspace = policy_local + .map(crate::policy_local::PolicyLocalContext::workspace) + .unwrap_or_default(); let policy_name = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.clone().unwrap_or_default(), NetworkAction::Deny { .. } => String::new(), diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index a7647d39ad..34f6e116d2 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -35,10 +35,13 @@ use crate::l7::tls::{ CertCache, ProxyTlsState, SandboxCa, build_upstream_client_config, read_system_ca_bundle, write_ca_files, }; +use crate::l7::token_grant_injection::SpiffeTokenGrantResolver; use crate::opa::OpaEngine; -use crate::policy_local::PolicyLocalContext; +use crate::policy_local::{DenialLogSource, PolicyLocalContext}; use crate::proxy::ProxyHandle; +use crate::token_grant::SessionTokenGrants; use openshell_core::endpoint_status::EndpointObservationSender; +use openshell_core::grpc_client::GatewayEndpoint; /// Concrete listener options supplied by trusted supervisor composition. /// This is not an isolation-backend capability or a serialized launch contract. @@ -189,6 +192,52 @@ fn append_additional_ca_bundle( Ok(()) } +/// Identity and authority of the one sandbox a [`run_networking`] call serves. +/// +/// A process hosting several sandboxes builds one per session; no value is +/// read from the process environment, so sessions cannot share gateway +/// credentials, provider token grants, denial history, or proxy CA files. +#[derive(Debug, Default)] +pub struct NetworkSession { + pub sandbox_id: Option, + pub sandbox_name: Option, + /// Gateway access with this session's credentials. + pub gateway: Option, + /// SPIFFE Workload API socket for provider token grants. Without one, + /// grants fail closed. + pub provider_workload_api_socket: Option, + /// Source served by `policy.local` `/v1/denials`. + pub denial_log: DenialLogSource, + /// Proxy interception CA and trust-file directory. + pub proxy_tls: ProxyTlsFiles, +} + +/// Provisioned proxy CA files and the directory that receives trust files. +/// +/// Without a provisioned CA the proxy generates an ephemeral one. Without a +/// directory it writes to the default container TLS root. +#[derive(Clone, Debug, Default)] +pub struct ProxyTlsFiles { + pub ca_certificate: Option, + pub ca_private_key: Option, + pub directory: Option, +} + +impl ProxyTlsFiles { + /// Files configured for a single-sandbox supervisor through its environment. + #[must_use] + pub fn from_environment() -> Self { + Self { + ca_certificate: std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT) + .map(std::path::PathBuf::from), + ca_private_key: std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY) + .map(std::path::PathBuf::from), + directory: std::env::var_os(openshell_core::sandbox_env::PROXY_TLS_DIR) + .map(std::path::PathBuf::from), + } + } +} + /// Handles and values produced by [`run_networking`] that the rest of /// `run_sandbox` consumes. /// @@ -235,9 +284,7 @@ pub async fn run_networking( entrypoint_pid: Arc, process_enabled: bool, provider_credentials: &ProviderCredentialState, - sandbox_id: Option<&str>, - sandbox_name: Option<&str>, - openshell_endpoint: Option<&str>, + session: NetworkSession, denial_tx: Option>, activity_tx: Option, endpoint_observation_tx: Option, @@ -245,7 +292,6 @@ pub async fn run_networking( workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, additional_ca_bundle: Option<&str>, - proxy_tls_dir: Option<&std::path::Path>, host_gateway_ip: Option, #[cfg(target_os = "linux")] transparent_runtime: Option, network_mediation_source: Option>, @@ -254,15 +300,30 @@ pub async fn run_networking( // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so // it can publish updated policy snapshots after a successful reload. - let policy_local_ctx = Arc::new(PolicyLocalContext::new( - retained_proto.cloned(), - openshell_endpoint.map(str::to_string), - sandbox_name - .map(str::to_string) - .or_else(|| sandbox_id.map(str::to_string)), - agent_proposals.clone(), - workspace_rx, + let NetworkSession { + sandbox_id, + sandbox_name, + gateway, + provider_workload_api_socket, + denial_log, + proxy_tls, + } = session; + let token_grants = SpiffeTokenGrantResolver::new(SessionTokenGrants::new( + gateway.clone(), + sandbox_id.clone(), + provider_workload_api_socket, )); + let policy_local_ctx = Arc::new( + PolicyLocalContext::new( + retained_proto.cloned(), + gateway, + sandbox_name.or(sandbox_id), + agent_proposals.clone(), + workspace_rx, + ) + .with_denial_log(denial_log) + .with_token_grants(Arc::new(token_grants)), + ); // Readiness signal for the proxy accept loop: the proxy binds the TCP // listener immediately (so the OS backlog queues early SYN packets) but @@ -370,14 +431,10 @@ pub async fn run_networking( // process; otherwise generate an ephemeral CA. // The CA cert is written to disk so sandbox processes can trust it. let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - let configured_ca = match ( - std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT), - std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY), - ) { - (Some(certificate), Some(private_key)) => Some(SandboxCa::load_from_paths( - std::path::Path::new(&certificate), - std::path::Path::new(&private_key), - )?), + let configured_ca = match (&proxy_tls.ca_certificate, &proxy_tls.ca_private_key) { + (Some(certificate), Some(private_key)) => { + Some(SandboxCa::load_from_paths(certificate, private_key)?) + } (None, None) => None, _ => { return Err(miette::miette!( @@ -390,14 +447,9 @@ pub async fn run_networking( let durable_ca = configured_ca.is_some(); match configured_ca.map_or_else(SandboxCa::generate, Ok) { Ok(ca) => { - let configured_tls_dir = - std::env::var_os(openshell_core::sandbox_env::PROXY_TLS_DIR) - .map(std::path::PathBuf::from); - let tls_dir = proxy_tls_dir - .or(configured_tls_dir.as_deref()) - .unwrap_or_else(|| { - std::path::Path::new(openshell_core::container_paths::TLS_ROOT) - }); + let tls_dir = proxy_tls.directory.as_deref().unwrap_or_else(|| { + std::path::Path::new(openshell_core::container_paths::TLS_ROOT) + }); let mut system_ca_bundle = read_system_ca_bundle(); // A TLS-intercepting corporate proxy (issue #1792) re-signs // tunneled server certificates with the corporate CA, so the diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index e165754105..05331e128d 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -27,32 +27,30 @@ //! - `scopes` — `OAuth2` scopes to request (optional) //! - `cache_ttl` — Optional exact cache override; zero disables caching //! -//! ## Environment +//! ## Session scope //! -//! Requires `OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET` to be set (path to -//! the SPIFFE Workload API socket). +//! Each supervised sandbox owns a [`SessionTokenGrants`]: its token cache, +//! HTTP client, gateway credentials, sandbox ID, and SPIFFE Workload API +//! socket. A token is never served to another sandbox. Without a configured +//! Workload API socket, grants fail closed with a configuration failure. use std::collections::HashMap; use std::future::Future; -use std::sync::{Arc, LazyLock, RwLock}; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use openshell_core::grpc_client::GatewayEndpoint; use openshell_core::oauth::{ self, ACCESS_TOKEN_TYPE, OAuthTokenResponse, TokenExchangeParams, TokenGrantClient, TokenGrantParams, effective_client_assertion_type, effective_token_type, }; use openshell_core::proto::ProviderCredentialTokenGrantType; -use openshell_core::sandbox_env; use openshell_core::token_grant_failure::{ TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, }; use spiffe::{WorkloadApiClient, WorkloadApiError}; -/// Token cache shared across all provider token grants. -static TOKEN_CACHE: LazyLock = LazyLock::new(TokenCache::new); -// Initialization failures stay bounded and typed; credential acquisition must not panic. -static TOKEN_GRANT_HTTP_CLIENT: LazyLock> = - LazyLock::new(TokenGrantClient::new); const DEFAULT_TOKEN_CACHE_TTL_SECONDS: i64 = 300; const TOKEN_CACHE_EXPIRY_SKEW_SECONDS: i64 = 30; const MAX_TOKEN_EXPIRES_IN_SECONDS: i64 = 3600; @@ -102,33 +100,154 @@ impl TokenCache { } } -/// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. +/// Provider token grants owned by one supervised sandbox. /// -/// This function fetches the sandbox's SPIFFE JWT-SVID from the local SPIRE -/// agent, then exchanges it for an access token with a POST request to the provider's -/// token endpoint with the JWT client assertion grant flow (RFC 7523). -/// -/// Tokens are cached per provider name with TTL. Subsequent calls return the -/// cached token if it has not expired. -/// -/// # Arguments -/// -/// * `provider_name` — Unique provider identifier (used as cache key) -/// * `token_endpoint` — `OAuth2` token service URL -/// * `jwt_svid_audience` — Optional audience to request when fetching the JWT-SVID -/// * `client_assertion_type` — Optional `OAuth2` client assertion type -/// * `audience` — Resource audience to request in the token request -/// * `scopes` — `OAuth2` scopes to request (may be empty) -/// * `cache_ttl_override` — Exact cache TTL; absence uses `expires_in` and zero disables caching -/// -/// # Errors -/// -/// Returns an error if: -/// - SPIFFE Workload API socket is not configured -/// - SPIRE agent is unreachable -/// - JWT-SVID fetch fails -/// - Token service request fails -/// - Token response is invalid +/// The cache, HTTP connection pool, gateway credentials, sandbox ID, and +/// workload identity socket all belong to this session, so a token acquired +/// for one sandbox is never served to another. +pub struct SessionTokenGrants { + cache: TokenCache, + // Initialization failures stay bounded and typed; acquisition must not panic. + http_client: Result, + gateway: Option, + sandbox_id: Option, + workload_api_socket: Option, +} + +impl SessionTokenGrants { + /// Grants authenticated as `sandbox_id` through `gateway`. A missing + /// Workload API socket makes every grant fail closed. + #[must_use] + pub fn new( + gateway: Option, + sandbox_id: Option, + workload_api_socket: Option, + ) -> Self { + Self { + cache: TokenCache::new(), + http_client: TokenGrantClient::new(), + gateway, + sandbox_id: sandbox_id.filter(|value| !value.trim().is_empty()), + workload_api_socket: workload_api_socket.filter(|path| !path.as_os_str().is_empty()), + } + } + + /// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. + /// + /// This fetches the sandbox's SPIFFE JWT-SVID from the session's Workload + /// API socket, then exchanges it for an access token with a POST request to + /// the provider's token endpoint with the JWT client assertion grant flow + /// (RFC 7523). + /// + /// Tokens are cached per provider name with TTL in this session's cache. + /// Subsequent calls return the cached token if it has not expired. + /// + /// # Errors + /// + /// Returns an error if: + /// - SPIFFE Workload API socket is not configured + /// - SPIRE agent is unreachable + /// - JWT-SVID fetch fails + /// - Token service request fails + /// - Token response is invalid + pub async fn obtain_provider_token( + &self, + request: ObtainProviderTokenRequest<'_>, + ) -> Result { + let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) + .map_err(|_| configuration_failure())?; + obtain_provider_token_with_grant( + ObtainProviderTokenInput { + cache: &self.cache, + provider_name: request.provider_name, + token_endpoint: request.token_endpoint, + jwt_svid_audience: request.jwt_svid_audience, + client_assertion_type: request.client_assertion_type, + audience: request.audience, + scopes: request.scopes, + cache_ttl_override: request.cache_ttl_override, + grant_type, + requested_token_type: request.requested_token_type, + }, + |jwt_audience| async move { + // Resolve local exchange prerequisites before fetching any workload credential. + let exchange_context = match grant_type { + ProviderCredentialTokenGrantType::ClientCredentials + | ProviderCredentialTokenGrantType::Unspecified => None, + ProviderCredentialTokenGrantType::TokenExchange => { + let (provider, credential_key) = + parse_provider_credential_key(request.provider_name)?; + let gateway = self.gateway.as_ref().ok_or_else(configuration_failure)?; + openshell_core::grpc_client::token_grant_gateway_endpoint(gateway.url())?; + let sandbox_id = self + .sandbox_id + .as_deref() + .ok_or_else(configuration_failure)?; + Some((provider, credential_key, gateway, sandbox_id)) + } + }; + let http_client = self.http_client.as_ref().map_err(|failure| *failure)?; + let socket_path = self + .workload_api_socket + .as_deref() + .ok_or_else(configuration_failure)?; + // Fetch JWT-SVID with authorization server as audience + // For RFC 7523, the JWT assertion's aud claim identifies the issuer/realm + let jwt_svid = fetch_jwt_svid_from_socket(socket_path, &jwt_audience).await?; + + match exchange_context { + None => { + // Perform OAuth2 JWT client assertion grant. The audience + // parameter in the token request specifies the resource server. + perform_token_grant( + http_client, + request.token_endpoint, + &jwt_svid, + request.client_assertion_type, + request.audience, + request.scopes, + request.request_id, + ) + .await + } + Some((provider, credential_key, gateway, sandbox_id)) => { + let intermediate = + openshell_core::grpc_client::exchange_provider_subject_token( + gateway, + sandbox_id, + provider, + credential_key, + &jwt_svid, + Some(request.request_id), + ) + .await?; + validate_access_token(&intermediate.access_token).map_err(|failure| { + failure.at_stage(TokenGrantStage::IntermediateExchange) + })?; + perform_token_exchange( + http_client, + request.token_endpoint, + &TokenExchangeParams { + client_assertion: &jwt_svid, + client_assertion_type: request.client_assertion_type, + subject_token: &intermediate.access_token, + subject_token_type: ACCESS_TOKEN_TYPE, + audience: request.audience, + scopes: request.scopes, + requested_token_type: request.requested_token_type, + }, + request.request_id, + ) + .await + } + } + }, + ) + .await + } +} + +/// One provider token request; see [`SessionTokenGrants::obtain_provider_token`]. pub struct ObtainProviderTokenRequest<'a> { pub provider_name: &'a str, pub token_endpoint: &'a str, @@ -144,96 +263,6 @@ pub struct ObtainProviderTokenRequest<'a> { pub request_id: &'a str, } -/// Obtain and cache a provider token, retaining only bounded grant failures. -pub async fn obtain_provider_token( - request: ObtainProviderTokenRequest<'_>, -) -> Result { - let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) - .map_err(|_| configuration_failure())?; - obtain_provider_token_with_grant( - ObtainProviderTokenInput { - cache: &TOKEN_CACHE, - provider_name: request.provider_name, - token_endpoint: request.token_endpoint, - jwt_svid_audience: request.jwt_svid_audience, - client_assertion_type: request.client_assertion_type, - audience: request.audience, - scopes: request.scopes, - cache_ttl_override: request.cache_ttl_override, - grant_type, - requested_token_type: request.requested_token_type, - }, - |jwt_audience| async move { - // Resolve local exchange prerequisites before fetching any workload credential. - let exchange_context = match grant_type { - ProviderCredentialTokenGrantType::ClientCredentials - | ProviderCredentialTokenGrantType::Unspecified => None, - ProviderCredentialTokenGrantType::TokenExchange => { - let (provider, credential_key) = - parse_provider_credential_key(request.provider_name)?; - let endpoint = supervisor_gateway_endpoint_from_env()?; - openshell_core::grpc_client::token_grant_gateway_endpoint(&endpoint)?; - Some(( - provider, - credential_key, - endpoint, - supervisor_sandbox_id_from_env()?, - )) - } - }; - // Fetch JWT-SVID with authorization server as audience - // For RFC 7523, the JWT assertion's aud claim identifies the issuer/realm - let jwt_svid = fetch_jwt_svid_for_token_grant(&jwt_audience).await?; - - match exchange_context { - None => { - // Perform OAuth2 JWT client assertion grant. The audience - // parameter in the token request specifies the resource server. - perform_token_grant( - request.token_endpoint, - &jwt_svid, - request.client_assertion_type, - request.audience, - request.scopes, - request.request_id, - ) - .await - } - Some((provider, credential_key, endpoint, sandbox_id)) => { - let intermediate = - openshell_core::grpc_client::exchange_provider_subject_token( - &endpoint, - &sandbox_id, - provider, - credential_key, - &jwt_svid, - Some(request.request_id), - ) - .await?; - validate_access_token(&intermediate.access_token).map_err(|failure| { - failure.at_stage(TokenGrantStage::IntermediateExchange) - })?; - perform_token_exchange( - request.token_endpoint, - &TokenExchangeParams { - client_assertion: &jwt_svid, - client_assertion_type: request.client_assertion_type, - subject_token: &intermediate.access_token, - subject_token_type: ACCESS_TOKEN_TYPE, - audience: request.audience, - scopes: request.scopes, - requested_token_type: request.requested_token_type, - }, - request.request_id, - ) - .await - } - } - }, - ) - .await -} - struct ObtainProviderTokenInput<'a> { cache: &'a TokenCache, provider_name: &'a str, @@ -297,21 +326,14 @@ where Ok(token_response.access_token) } -/// Fetch JWT-SVID from SPIRE agent for token grant authentication. +/// Fetch a JWT-SVID from the SPIRE agent for token grant authentication. /// -/// This function connects to the local SPIRE agent via the Workload API and -/// requests a JWT-SVID with the specified audience. The JWT-SVID is used as -/// the client assertion in the `OAuth2` grant request. -async fn fetch_jwt_svid_for_token_grant(audience: &str) -> Result { - let socket_path = provider_spiffe_workload_api_socket_from_env()?; - fetch_jwt_svid_from_socket(&socket_path, audience).await -} - +/// The JWT-SVID is used as the client assertion in the `OAuth2` grant request. async fn fetch_jwt_svid_from_socket( - socket_path: &str, + socket_path: &std::path::Path, audience: &str, ) -> Result { - let endpoint = openshell_core::spiffe::workload_api_endpoint(std::path::Path::new(socket_path)); + let endpoint = openshell_core::spiffe::workload_api_endpoint(socket_path); // Connect to SPIRE agent let client = WorkloadApiClient::connect_to(&endpoint) @@ -357,14 +379,8 @@ fn configuration_failure() -> TokenGrantFailure { ) } -fn provider_spiffe_workload_api_socket_from_env() -> Result { - std::env::var(sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - async fn perform_token_grant( + http_client: &TokenGrantClient, token_endpoint: &str, jwt_svid: &str, client_assertion_type: &str, @@ -373,9 +389,7 @@ async fn perform_token_grant( request_id: &str, ) -> Result { oauth::post_oauth_token_grant( - TOKEN_GRANT_HTTP_CLIENT - .as_ref() - .map_err(|failure| *failure)?, + http_client, token_endpoint, &TokenGrantParams { client_assertion: jwt_svid, @@ -389,19 +403,12 @@ async fn perform_token_grant( } async fn perform_token_exchange( + http_client: &TokenGrantClient, token_endpoint: &str, params: &TokenExchangeParams<'_>, request_id: &str, ) -> Result { - oauth::post_oauth_token_exchange( - TOKEN_GRANT_HTTP_CLIENT - .as_ref() - .map_err(|failure| *failure)?, - token_endpoint, - params, - Some(request_id), - ) - .await + oauth::post_oauth_token_exchange(http_client, token_endpoint, params, Some(request_id)).await } pub use oauth::validate_access_token; @@ -457,20 +464,6 @@ fn effective_jwt_svid_audience(token_endpoint: &str, jwt_svid_audience: &str) -> } } -fn supervisor_gateway_endpoint_from_env() -> Result { - std::env::var(sandbox_env::ENDPOINT) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - -fn supervisor_sandbox_id_from_env() -> Result { - std::env::var(sandbox_env::SANDBOX_ID) - .ok() - .filter(|value| !value.trim().is_empty()) - .ok_or_else(configuration_failure) -} - fn parse_provider_credential_key(key: &str) -> Result<(&str, &str), TokenGrantFailure> { openshell_core::dynamic_credential_key::credential_identity(key) .and_then(|identity| identity.split_once(':')) @@ -670,6 +663,10 @@ mod tests { const TEST_REQUEST_ID: &str = "4f1c2b3a-9d8e-4c7b-a6f5-e4d3c2b1a098"; + fn test_http_client() -> TokenGrantClient { + TokenGrantClient::new().expect("token grant HTTP client") + } + #[derive(Debug)] struct CapturedTokenRequest { request_line: String, @@ -1415,6 +1412,7 @@ mod tests { let scopes = vec!["read".to_string(), "write".to_string()]; let response = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "", @@ -1475,6 +1473,7 @@ mod tests { token_endpoint_once("200 OK", r#"{"access_token":"access-123"}"#).await; let response = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "urn:ietf:params:oauth:client-assertion-type:jwt-spiffe", @@ -1504,9 +1503,17 @@ mod tests { ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("malformed access token should fail closed"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("malformed access token should fail closed"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(200)); @@ -1516,9 +1523,17 @@ mod tests { async fn perform_token_grant_does_not_follow_redirects() { let (endpoint, request) = token_endpoint_redirect_once("http://127.0.0.1:1/stolen").await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("redirect response should fail closed"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("redirect response should fail closed"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(302)); @@ -1529,10 +1544,17 @@ mod tests { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":"access-123"}"#).await; - let response = - perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect("token grant should succeed without audience or scopes"); + let response = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect("token grant should succeed without audience or scopes"); let request = request.await.expect("token endpoint task should finish"); assert_eq!(response.access_token, "access-123"); @@ -1553,6 +1575,7 @@ mod tests { .await; let err = perform_token_grant( + &test_http_client(), &endpoint, "jwt-svid-token", "", @@ -1580,9 +1603,17 @@ mod tests { ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("token grant should fail on server error"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("token grant should fail on server error"); let _request = request.await.expect("token endpoint task should finish"); let message = err.to_string(); @@ -1597,9 +1628,17 @@ mod tests { async fn perform_token_grant_reports_malformed_success_json() { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":42"#).await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) - .await - .expect_err("malformed token response should fail"); + let err = perform_token_grant( + &test_http_client(), + &endpoint, + "jwt-svid-token", + "", + "", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("malformed token response should fail"); let _request = request.await.expect("token endpoint task should finish"); assert_eq!(err.upstream_status(), Some(200)); diff --git a/crates/openshell-supervisor-process/src/debug_rpc.rs b/crates/openshell-supervisor-process/src/debug_rpc.rs index 04cd1e9ba7..b4370c8954 100644 --- a/crates/openshell-supervisor-process/src/debug_rpc.rs +++ b/crates/openshell-supervisor-process/src/debug_rpc.rs @@ -24,7 +24,9 @@ use openshell_core::proto::{ }; use sha2::{Digest, Sha256}; -use openshell_core::grpc_client::{AuthedChannel, connect_channel_pub}; +use openshell_core::grpc_client::{ + AuthedChannel, GatewayCredentials, GatewayEndpoint, connect_channel_pub, +}; /// Entry point for the `debug-rpc` subcommand. Returns the process exit /// code; `main` propagates it. @@ -66,7 +68,8 @@ async fn open_client() -> Result> { let endpoint = std::env::var(openshell_core::sandbox_env::ENDPOINT) .into_diagnostic() .wrap_err("OPENSHELL_ENDPOINT must be set")?; - let channel = connect_channel_pub(&endpoint).await?; + let credentials = GatewayCredentials::from_environment(&endpoint).await?; + let channel = connect_channel_pub(&GatewayEndpoint::new(endpoint, credentials)).await?; Ok(OpenShellClient::new(channel)) } diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 91e1fa5337..32e3fd4433 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -82,7 +82,7 @@ impl Drop for BoundaryAccess { #[allow(clippy::too_many_arguments)] pub async fn start_boundary_access( sandbox_id: Option<&str>, - openshell_endpoint: Option<&str>, + gateway: Option<&openshell_core::grpc_client::GatewayEndpoint>, ssh_socket_path: Option<&str>, shared_ssh_socket: bool, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, @@ -116,10 +116,10 @@ pub async fn start_boundary_access( // Gateway authentication, forwarding, and readiness do not depend on SSH // or the host OS. Acceptance remains false until the gateway authenticates // this supervisor, and reconnects retain main's retry behavior. - let (session_task, session_readiness) = match (openshell_endpoint, sandbox_id) { - (Some(endpoint), Some(id)) => { + let (session_task, session_readiness) = match (gateway, sandbox_id) { + (Some(gateway), Some(id)) => { let (task, accepted) = crate::supervisor_session::spawn_with_readiness( - endpoint.to_string(), + gateway.clone(), id.to_string(), ssh_socket_path.unwrap_or_default(), port_forward, @@ -223,7 +223,7 @@ async fn start_optional_ssh( /// Report the canonical process exit until the gateway acknowledges it. pub async fn report_main_process_exit( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, exit_code: i32, @@ -231,7 +231,7 @@ pub async fn report_main_process_exit( let mut delay = Duration::from_millis(250); loop { match crate::supervisor_session::report_main_process_exit( - endpoint, + gateway, sandbox_id, instance_id, exit_code, @@ -249,11 +249,15 @@ pub async fn report_main_process_exit( } /// Finalize canonical process terminal delivery until acknowledged. -pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instance_id: &str) { +pub async fn finalize_main_process_exit( + gateway: &openshell_core::grpc_client::GatewayEndpoint, + sandbox_id: &str, + instance_id: &str, +) { let mut delay = Duration::from_millis(250); loop { match crate::supervisor_session::finalize_main_process_exit( - endpoint, + gateway, sandbox_id, instance_id, ) @@ -352,9 +356,14 @@ mod tests { #[tokio::test] async fn no_ssh_access_retains_main_attachment_and_session_readiness() { let boundary = Arc::new(AccessBoundary); + let gateway = openshell_core::grpc_client::GatewayEndpoint::new( + "http://127.0.0.1:1", + openshell_core::grpc_client::GatewayCredentials::from_gateway_token("test-token") + .expect("test credentials"), + ); let access = start_boundary_access( Some("sandbox"), - Some("http://127.0.0.1:1"), + Some(&gateway), None, false, None, diff --git a/crates/openshell-supervisor-process/src/log_push.rs b/crates/openshell-supervisor-process/src/log_push.rs index d12061e82e..930ef87d5d 100644 --- a/crates/openshell-supervisor-process/src/log_push.rs +++ b/crates/openshell-supervisor-process/src/log_push.rs @@ -7,32 +7,78 @@ //! channel to a background task. The task batches lines and streams them to //! the server using the `PushSandboxLogs` client-streaming RPC. -use openshell_core::grpc_client::CachedOpenShellClient; +use openshell_core::grpc_client::{CachedOpenShellClient, GatewayEndpoint}; use openshell_core::proto::{PushSandboxLogsRequest, SandboxLogLine}; +use std::cell::RefCell; +use std::sync::Arc; use tokio::sync::mpsc; use tracing::{Event, Subscriber}; use tracing_subscriber::Layer; use tracing_subscriber::layer::Context; +/// One sandbox's push queue. Lines are labelled with this sandbox and sent +/// only with the credentials of the task that drains the queue. +#[derive(Clone)] +pub struct LogSink { + sandbox_id: Arc, + tx: mpsc::Sender, +} + +impl LogSink { + #[must_use] + pub fn new(sandbox_id: impl Into>, tx: mpsc::Sender) -> Self { + Self { + sandbox_id: sandbox_id.into(), + tx, + } + } +} + +std::thread_local! { + /// Sink of the hosted session whose runtime owns this thread. + static THREAD_SINK: RefCell> = const { RefCell::new(None) }; +} + +/// Route events emitted on the calling thread to a hosted session's sink. +/// +/// Hosted sessions bind every thread of their runtime. Unbound threads push +/// nothing, so an event cannot reach another sandbox's log stream. +pub fn bind_thread_sink(sink: Option) { + THREAD_SINK.with(|current| *current.borrow_mut() = sink); +} + +#[derive(Clone)] +enum Route { + /// The process supervises one sandbox; every event belongs to it. + Fixed(LogSink), + /// Events belong to the session bound to the emitting thread, if any. + ThreadBound, +} + /// Tracing layer that pushes log events to the `OpenShell` server. /// /// Events are sent best-effort via `try_send` — if the channel is full the /// event is dropped. Logging must never block the sandbox. #[derive(Clone)] pub struct LogPushLayer { - sandbox_id: String, - tx: mpsc::Sender, + route: Route, max_level: tracing::Level, } impl LogPushLayer { pub fn new(sandbox_id: String, tx: mpsc::Sender) -> Self { + Self::with_route(Route::Fixed(LogSink::new(sandbox_id, tx))) + } + + /// Push each event to the sink bound to its emitting thread. + #[must_use] + pub fn thread_bound() -> Self { + Self::with_route(Route::ThreadBound) + } + + fn with_route(route: Route) -> Self { let max_level = parse_max_level(std::env::var("OPENSHELL_LOG_PUSH_LEVEL").ok().as_deref()); - Self { - sandbox_id, - tx, - max_level, - } + Self { route, max_level } } } @@ -50,62 +96,83 @@ impl Layer for LogPushLayer { if *meta.level() > self.max_level { return; } + match &self.route { + Route::Fixed(sink) => push_event(sink, event), + Route::ThreadBound => THREAD_SINK.with(|sink| { + if let Some(sink) = sink.borrow().as_ref() { + push_event(sink, event); + } + }), + } + } +} - // OCSF events carry their payload in a thread-local; extract the - // shorthand representation for the push message. Non-OCSF events - // use the original visitor-based extraction. - let (msg, fields) = if meta.target() == openshell_ocsf::OCSF_TARGET { - if let Some(ocsf_event) = openshell_ocsf::clone_current_event() { - ( - ocsf_event.format_shorthand(), - std::collections::HashMap::new(), - ) - } else { - return; - } +fn push_event(sink: &LogSink, event: &Event<'_>) { + let meta = event.metadata(); + // OCSF events carry their payload in a thread-local; extract the + // shorthand representation for the push message. Non-OCSF events + // use the original visitor-based extraction. + let (msg, fields) = if meta.target() == openshell_ocsf::OCSF_TARGET { + if let Some(ocsf_event) = openshell_ocsf::clone_current_event() { + ( + ocsf_event.format_shorthand(), + std::collections::HashMap::new(), + ) } else { - let mut visitor = LogVisitor::default(); - event.record(&mut visitor); - visitor.into_parts(meta.name()) - }; + return; + } + } else { + let mut visitor = LogVisitor::default(); + event.record(&mut visitor); + visitor.into_parts(meta.name()) + }; - let ts = openshell_core::time::now_ms(); + let ts = openshell_core::time::now_ms(); - let is_ocsf = meta.target() == openshell_ocsf::OCSF_TARGET; + let is_ocsf = meta.target() == openshell_ocsf::OCSF_TARGET; - let log = SandboxLogLine { - sandbox_id: self.sandbox_id.clone(), - event_time: openshell_core::time::timestamp_from_millis(ts).ok(), - level: if is_ocsf { - "OCSF".to_string() - } else { - meta.level().to_string() - }, - target: meta.target().to_string(), - message: msg, - source: "sandbox".to_string(), - fields, - }; + let log = SandboxLogLine { + sandbox_id: sink.sandbox_id.to_string(), + event_time: openshell_core::time::timestamp_from_millis(ts).ok(), + level: if is_ocsf { + "OCSF".to_string() + } else { + meta.level().to_string() + }, + target: meta.target().to_string(), + message: msg, + source: "sandbox".to_string(), + fields, + }; - // Best-effort: drop if the channel is full (don't block tracing). - let _ = self.tx.try_send(log); - } + // Best-effort: drop if the channel is full (don't block tracing). + let _ = sink.tx.try_send(log); } +/// Lines a sandbox's push queue holds before the layer starts dropping them. +pub const LOG_PUSH_QUEUE_LINES: usize = 1024; + /// Spawn a background task that batches and pushes log lines to the server. /// /// Returns the sender half of the channel (for the [`LogPushLayer`]) and the /// task handle. The task runs until the sender is dropped or the gRPC stream /// breaks. pub fn spawn_log_push_task( - endpoint: String, + gateway: GatewayEndpoint, sandbox_id: String, ) -> (mpsc::Sender, tokio::task::JoinHandle<()>) { - let (tx, rx) = mpsc::channel::(1024); - - let handle = tokio::spawn(run_push_loop(endpoint, sandbox_id, rx)); + let (tx, rx) = mpsc::channel::(LOG_PUSH_QUEUE_LINES); + (tx, start_log_push(gateway, sandbox_id, rx)) +} - (tx, handle) +/// Push lines from a queue created before the caller's runtime, with the +/// owning sandbox's gateway credentials. +pub fn start_log_push( + gateway: GatewayEndpoint, + sandbox_id: String, + lines: mpsc::Receiver, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(run_push_loop(gateway, sandbox_id, lines)) } /// Maximum backoff delay between reconnection attempts. @@ -114,7 +181,7 @@ const MAX_BACKOFF: tokio::time::Duration = tokio::time::Duration::from_secs(30); const INITIAL_BACKOFF: tokio::time::Duration = tokio::time::Duration::from_secs(1); async fn run_push_loop( - endpoint: String, + gateway: GatewayEndpoint, sandbox_id: String, mut rx: mpsc::Receiver, ) { @@ -127,7 +194,7 @@ async fn run_push_loop( attempt += 1; // --- Connect --- - let client = match CachedOpenShellClient::connect(&endpoint).await { + let client = match CachedOpenShellClient::connect(&gateway).await { Ok(c) => { if attempt > 1 { eprintln!("openshell: log push reconnected (attempt {attempt})"); @@ -339,8 +406,7 @@ mod tests { fn capture(capacity: usize, f: impl FnOnce()) -> Vec { let (tx, mut rx) = mpsc::channel::(capacity); let layer = LogPushLayer { - sandbox_id: "sb-test".to_string(), - tx, + route: Route::Fixed(LogSink::new("sb-test", tx)), max_level: tracing::Level::INFO, }; let subscriber = tracing_subscriber::registry().with(layer); diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 77620a94d1..71313e19c0 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -381,7 +381,7 @@ fn map_session_stream_message( /// The task runs for the lifetime of the sandbox process, reconnecting with /// exponential backoff on failures. pub fn spawn( - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -390,7 +390,7 @@ pub fn spawn( runtime: SessionRuntimeContext, ) -> tokio::task::JoinHandle<()> { spawn_with_readiness( - endpoint, + gateway, sandbox_id, ssh_socket_path, port_forward, @@ -403,7 +403,7 @@ pub fn spawn( /// Spawn the supervisor session and expose when the gateway has accepted it. pub fn spawn_with_readiness( - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -413,7 +413,7 @@ pub fn spawn_with_readiness( ) -> (tokio::task::JoinHandle<()>, watch::Receiver) { let (ready_tx, ready_rx) = watch::channel(false); let config = SessionConfig { - endpoint, + gateway, sandbox_id, ssh_socket_path, port_forward, @@ -429,7 +429,7 @@ pub fn spawn_with_readiness( } struct SessionConfig { - endpoint: String, + gateway: grpc_client::GatewayEndpoint, sandbox_id: String, ssh_socket_path: std::path::PathBuf, port_forward: Arc, @@ -453,7 +453,7 @@ async fn run_session_loop(config: SessionConfig) { // dial `target` for the next attempt and mark it as redirected so that // replica serves us rather than redirecting again. Any failure sends us // back to the configured address, which load-balances across replicas. - let mut target = config.endpoint.clone(); + let mut target = config.gateway.clone(); let mut redirected = false; loop { @@ -466,8 +466,11 @@ async fn run_session_loop(config: SessionConfig) { match result { Ok(SessionOutcome::Closed) => { config.ready_tx.send_replace(false); - let event = - session_closed_event(&openshell_ocsf::ctx::ctx(), &target, &config.sandbox_id); + let event = session_closed_event( + &openshell_ocsf::ctx::ctx(), + target.url(), + &config.sandbox_id, + ); ocsf_emit!(event); break; } @@ -482,7 +485,7 @@ async fn run_session_loop(config: SessionConfig) { peer_endpoint = %peer_endpoint, "supervisor session: following gateway redirect" ); - target = peer_endpoint; + target = config.gateway.with_url(peer_endpoint); redirected = true; // No backoff: this is an expected handoff, not a failure. } @@ -490,15 +493,15 @@ async fn run_session_loop(config: SessionConfig) { let accepted = config.ready_tx.send_replace(false); let event = session_failed_event( &openshell_ocsf::ctx::ctx(), - &target, + target.url(), attempt, &e.to_string(), ); ocsf_emit!(event); // Fall back to the configured gateway address so a redirect // to a replica that is going away cannot strand the sandbox. - let failed_redirect_target = target != config.endpoint && !accepted; - target.clone_from(&config.endpoint); + let failed_redirect_target = target.url() != config.gateway.url() && !accepted; + target.clone_from(&config.gateway); redirected = redirect_survives_failure(redirected, accepted); if let Some(delay) = backoff.on_failure(accepted, failed_redirect_target) { tokio::time::sleep(delay).await; @@ -580,7 +583,7 @@ enum SessionOutcome { async fn run_single_session( config: &SessionConfig, - target: &str, + target: &grpc_client::GatewayEndpoint, redirected: bool, connection_epoch: u64, ) -> Result> { @@ -668,7 +671,7 @@ async fn run_single_session( } let event = session_established_event( &openshell_ocsf::ctx::ctx(), - target, + target.url(), &accepted.session_id, u32::try_from(heartbeat_secs).unwrap_or(u32::MAX), ); @@ -807,12 +810,12 @@ async fn apply_update( /// Report the canonical process result and wait for durable handling. pub async fn report_main_process_exit( - endpoint: &str, + gateway: &grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, exit_code: i32, ) -> Result<(), Box> { - let channel = grpc_client::connect_channel_pub(endpoint) + let channel = grpc_client::connect_channel_pub(gateway) .await .map_err(|error| format!("connect failed: {error}"))?; let mut client = OpenShellClient::new(channel); @@ -844,11 +847,11 @@ pub(crate) async fn test_bridge_ssh_relay( /// Confirm terminal delivery and permit ephemeral cleanup. pub async fn finalize_main_process_exit( - endpoint: &str, + gateway: &grpc_client::GatewayEndpoint, sandbox_id: &str, instance_id: &str, ) -> Result<(), Box> { - let channel = grpc_client::connect_channel_pub(endpoint) + let channel = grpc_client::connect_channel_pub(gateway) .await .map_err(|error| format!("connect failed: {error}"))?; let mut client = OpenShellClient::new(channel); diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 8108c6ea03..864a24a9d9 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -22,6 +22,24 @@ only after boundary attachment and confirmation; the supervisor owns policy evaluation, live credentials, and listener lifetime. These settings do not add a proxy hook to the isolation interface or the shared Sandbox Protocol descriptor. +## Session ownership + +`run_session` supervises one sandbox generation from a `SessionConfig` and +`SessionControls`. Everything that belongs to that generation is an input or +is created by the session: gateway credentials and their renewal, the Sandbox +Protocol credential, policy, provider credentials and token grants, middleware +credentials, OCSF JSON settings, proxy trust files, the denial history served +by `policy.local`, the backend connection, access plane, readiness, and stop +signal. Gateway clients receive a `GatewayEndpoint`, which pairs the URL with +that session's `GatewayCredentials`; nothing is installed process-wide. + +`run_sandbox_with_backend` is the dedicated wrapper. It keeps the existing +process contract: it sets the process-wide OCSF context, reads launch inputs +from the environment, binds the readiness socket and port to session +readiness, and stops the session on SIGTERM or SIGINT. The binary shares one +`GatewayCredentials` between log push and supervision, so renewal by either +keeps both current. + ## Console logging Diagnostics and OCSF shorthand share a bounded, nonblocking stderr writer. With `ocsf_json_enabled=true`, the same writer also receives timestamped `OCSF-JSON` records. Each formatter submits a complete line in one write so concurrent producers cannot interleave records in the queue. The 1,024-line queue drops new lines when full rather than waiting for stderr. diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index c291b42ef0..b9b5c1cdc9 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -702,6 +702,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { additional_ca_bundle: None, backend_descriptor: TestSetup::descriptor(), auth_bundle: setup.auth.clone(), + gateway_credentials: None, admitted_isolation_backend: Some(TEST_BACKEND.into()), main_exit_marker: Some(marker.clone()), }, diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index 62e7ffb6d4..1a83fd0887 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -738,7 +738,7 @@ impl ConfigRuntime { ); let provider = match client .fetch_provider_environment( - &ctx.endpoint, + &ctx.gateway, &ctx.sandbox_id, ctx.provider_credentials.requires_stable_placeholders(), ) @@ -1159,7 +1159,7 @@ impl ConfigRuntime { pub async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { let client = openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - &ctx.endpoint, + &ctx.gateway, ctx.extension_credentials.clone(), ) .await?; @@ -1198,7 +1198,7 @@ async fn report_runtime_configuration( ); } openshell_core::grpc_client::report_sandbox_configuration( - &ctx.endpoint, + &ctx.gateway, &ctx.sandbox_id, instance_id, Some(snapshot), @@ -1618,6 +1618,7 @@ mod tests { use super::*; use crate::tests::{ proto_policy_fixture, proto_tcp_policy_fixture, settings_poll_result, startup_provider, + test_gateway, }; use crate::*; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1779,7 +1780,7 @@ mod tests { async fn fetch_provider_environment( &self, - _endpoint: &str, + _gateway: &openshell_core::grpc_client::GatewayEndpoint, _sandbox_id: &str, _requires_stable_placeholders: bool, ) -> Result { @@ -1826,12 +1827,12 @@ mod tests { async fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { self.inner - .fetch_provider_environment(endpoint, sandbox_id, requires_stable_placeholders) + .fetch_provider_environment(gateway, sandbox_id, requires_stable_placeholders) .await } @@ -2074,7 +2075,7 @@ mod tests { async fn fetch_provider_environment( &self, - _endpoint: &str, + _gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { @@ -2212,12 +2213,12 @@ mod tests { async fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> Result { self.inner - .fetch_provider_environment(endpoint, sandbox_id, requires_stable_placeholders) + .fetch_provider_environment(gateway, sandbox_id, requires_stable_placeholders) .await } @@ -2709,7 +2710,7 @@ mod tests { ); } PolicyPollLoopContext { - endpoint: String::new(), + gateway: test_gateway(), sandbox_id: "sandbox-test".to_string(), sandbox: "sandbox-test-name".to_string(), opa_engine, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index d78c4c04b9..e27bd23455 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -303,6 +303,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl payload: Vec::new(), }, auth_bundle, + gateway_credentials: None, admitted_isolation_backend: Some("test-held".into()), main_exit_marker: None, }, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 9628d811a0..ab05a5e1fb 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -601,6 +601,8 @@ pub async fn run_network_proxy( ); let (_, workspace_rx) = tokio::sync::watch::channel(String::new()); let tls_dir = prepare_network_proxy_tls_dir(tls_dir)?; + let mut proxy_tls = openshell_supervisor_network::run::ProxyTlsFiles::from_environment(); + proxy_tls.directory = Some(tls_dir.path.clone()); let mut networking = openshell_supervisor_network::run::run_networking( &policy, None, @@ -609,9 +611,11 @@ pub async fn run_network_proxy( Arc::new(AtomicU32::new(0)), false, &provider_credentials, - None, - Some("network-proxy"), - None, + openshell_supervisor_network::run::NetworkSession { + sandbox_name: Some("network-proxy".to_string()), + proxy_tls, + ..Default::default() + }, None, None, None, @@ -619,7 +623,6 @@ pub async fn run_network_proxy( workspace_rx, &upstream_proxy_args, additional_ca_bundle.as_deref(), - Some(&tls_dir.path), None, #[cfg(target_os = "linux")] None, @@ -714,6 +717,7 @@ pub async fn run_sandbox( additional_ca_bundle, backend_descriptor, auth_bundle, + gateway_credentials: None, admitted_isolation_backend, main_exit_marker, }, @@ -742,10 +746,19 @@ pub struct SandboxRunConfig { pub additional_ca_bundle: Option, pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, + /// Credentials issued from `auth_bundle` and shared with other gateway + /// clients of this process, such as log push. Created when absent. + pub gateway_credentials: Option, pub admitted_isolation_backend: Option, pub main_exit_marker: Option, } +/// Supervise the one sandbox generation this process was launched for. +/// +/// The process-wide OCSF context, SIGTERM/SIGINT handling, readiness +/// endpoints, and the environment-provided launch inputs belong to the +/// dedicated process; [`run_session`] owns everything else. +/// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. #[allow(clippy::similar_names)] @@ -773,9 +786,260 @@ pub async fn run_sandbox_with_backend( additional_ca_bundle, backend_descriptor, auth_bundle, + gateway_credentials, admitted_isolation_backend, main_exit_marker, } = config; + + // Initialize the process-wide OCSF context early so that events emitted + // during policy loading (filesystem config, validation) have a context. + // Proxy IP/port use defaults here; the boundary mediation source carries + // workload-side connection metadata. + if !openshell_ocsf::ctx::set_ctx(session_event_context( + sandbox_id.as_deref(), + sandbox.as_deref(), + std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + )) { + debug!("OCSF context already initialized, keeping existing"); + } + + let (readiness, session_readiness) = tokio::sync::watch::channel(false); + let control_readiness = health_socket_path + .map(ReadinessEndpoint::Unix) + .into_iter() + .chain(health_port.map(ReadinessEndpoint::Tcp)) + .map(|endpoint| ControlReadiness::start(endpoint, Some(session_readiness.clone()))) + .collect::>>()?; + let result = Box::pin(run_session( + backend_setup, + SessionConfig { + command, + workdir, + timeout_secs, + interactive, + await_main_process_attachment, + sandbox_id, + sandbox, + openshell_endpoint, + policy_rules, + policy_data, + ssh_socket_path, + shared_ssh_socket: shared_ssh_socket_from_env(), + ocsf_json: openshell_ocsf::OcsfJsonSettings { + enabled: ocsf_enabled, + target_version: ocsf_schema_version, + }, + upstream_proxy_args, + additional_ca_bundle, + backend_descriptor, + auth_bundle, + gateway_credentials, + admitted_isolation_backend, + main_exit_marker, + trace_context: TraceContext::from_environment(), + provider_workload_api_socket: std::env::var_os( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ) + .filter(|path| !path.is_empty()) + .map(std::path::PathBuf::from), + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), + }, + SessionControls { + shutdown: Box::pin(wait_for_control_shutdown_signal()), + readiness, + }, + )) + .await; + drop(control_readiness); + result +} + +/// OCSF identity of one supervised sandbox. +#[must_use] +pub fn session_event_context( + sandbox_id: Option<&str>, + sandbox: Option<&str>, + container_image: String, +) -> EventContext { + let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( + |_| "openshell-sandbox".to_string(), + |s| s.trim().to_string(), + ); + EventContext { + sandbox_id: sandbox_id.unwrap_or_default().to_string(), + sandbox_name: sandbox.unwrap_or_default().to_string(), + container_image, + hostname, + product_version: openshell_core::VERSION.to_string(), + proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), + proxy_port: 3128, + origin: openshell_ocsf::EventOrigin::Supervisor, + } +} + +/// W3C trace context of the operation that launched a sandbox generation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TraceContext { + pub traceparent: String, + pub tracestate: Option, +} + +impl TraceContext { + /// The context a driver passed to a dedicated supervisor process. + #[must_use] + pub fn from_environment() -> Option { + let traceparent = std::env::var(openshell_otel::TRACEPARENT_ENV) + .ok() + .filter(|value| !value.is_empty())?; + Some(Self { + traceparent, + tracestate: std::env::var(openshell_otel::TRACESTATE_ENV) + .ok() + .filter(|value| !value.is_empty()), + }) + } +} + +/// Trusted inputs owned by one supervised sandbox generation. +/// +/// Nothing in a session reads process-wide sandbox state: identity, +/// credentials, policy inputs, logging settings, and file locations all come +/// from here, so one process can host several sessions. +pub struct SessionConfig { + pub command: Vec, + pub workdir: Option, + pub timeout_secs: u64, + pub interactive: bool, + pub await_main_process_attachment: bool, + pub sandbox_id: Option, + pub sandbox: Option, + pub openshell_endpoint: Option, + pub policy_rules: Option, + pub policy_data: Option, + pub ssh_socket_path: Option, + /// Grant the SSH socket to the supervisor's group rather than its owner. + pub shared_ssh_socket: bool, + /// OCSF JSON output settings this session's gateway configuration controls. + pub ocsf_json: openshell_ocsf::OcsfJsonSettings, + pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + pub additional_ca_bundle: Option, + pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, + pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, + /// Credentials issued from `auth_bundle`; created from it when absent. + pub gateway_credentials: Option, + pub admitted_isolation_backend: Option, + pub main_exit_marker: Option, + /// Parent of the startup span. + pub trace_context: Option, + /// SPIFFE Workload API socket for provider token grants; absent fails closed. + pub provider_workload_api_socket: Option, + pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, + /// Source of `policy.local` recent denials. + pub denial_log: openshell_supervisor_network::policy_local::DenialLogSource, +} + +/// How the owner of a session stops it and observes its readiness. +pub struct SessionControls { + /// Completes when the owner asks the session to stop its workload. It is + /// first polled once the workload has started. + pub shutdown: Pin + Send>>, + /// True while the gateway has accepted this session's supervision and its + /// access plane is retained; false otherwise. + pub readiness: tokio::sync::watch::Sender, +} + +/// Mirrors gateway session acceptance into the owner's readiness until the +/// access plane is released. +struct SessionReadiness { + readiness: tokio::sync::watch::Sender, + forward: Option>, +} + +impl SessionReadiness { + fn start( + readiness: tokio::sync::watch::Sender, + accepted: Option>, + ) -> Self { + // Without a gateway session, an attached access plane is ready. + let Some(mut accepted) = accepted else { + readiness.send_replace(true); + return Self { + readiness, + forward: None, + }; + }; + let sender = readiness.clone(); + let forward = tokio::spawn(async move { + loop { + sender.send_replace(*accepted.borrow_and_update()); + if accepted.changed().await.is_err() { + sender.send_replace(false); + return; + } + } + }); + Self { + readiness, + forward: Some(forward), + } + } + + fn release(&mut self) { + if let Some(forward) = self.forward.take() { + forward.abort(); + } + self.readiness.send_replace(false); + } +} + +impl Drop for SessionReadiness { + fn drop(&mut self) { + self.release(); + } +} + +/// Supervise one sandbox generation until its workload exits or the owner +/// stops it. +/// +/// Trusted composition chooses the setup before shared admission, policy, and +/// lifecycle handling. Payload contents never select a backend implementation. +#[allow(clippy::similar_names)] +pub async fn run_session( + backend_setup: &dyn backend_setup::BackendSetup, + config: SessionConfig, + controls: SessionControls, +) -> Result { + let SessionConfig { + command, + workdir, + timeout_secs, + interactive, + await_main_process_attachment, + sandbox_id, + sandbox, + openshell_endpoint, + policy_rules, + policy_data, + ssh_socket_path, + shared_ssh_socket, + ocsf_json, + upstream_proxy_args, + additional_ca_bundle, + backend_descriptor, + auth_bundle, + gateway_credentials, + admitted_isolation_backend, + main_exit_marker, + trace_context, + provider_workload_api_socket, + proxy_tls, + denial_log, + } = config; + let SessionControls { + shutdown: shutdown_requested, + readiness, + } = controls; // The current client attachment adapter is SSH; retaining a process's I/O // internally does not make it reachable by a foreground client. if await_main_process_attachment && (!cfg!(unix) || ssh_socket_path.is_none()) { @@ -791,33 +1055,10 @@ pub async fn run_sandbox_with_backend( |(program, args)| (program.clone(), args.to_vec()), ); - // Initialize the process-wide OCSF context early so that events emitted - // during policy loading (filesystem config, validation) have a context. - // Proxy IP/port use defaults here; the boundary mediation source carries - // workload-side connection metadata. - { - let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( - |_| "openshell-sandbox".to_string(), - |s| s.trim().to_string(), - ); - - if !openshell_ocsf::ctx::set_ctx(EventContext { - sandbox_id: sandbox_id.clone().unwrap_or_default(), - sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), - container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - hostname, - product_version: openshell_core::VERSION.to_string(), - proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), - proxy_port: 3128, - origin: openshell_ocsf::EventOrigin::Supervisor, - }) { - debug!("OCSF context already initialized, keeping existing"); - } - } - - // Extension credentials are owned by this supervisor and shared by every - // gateway connection it opens, so the middleware registry's bearer slots - // and the policy poll loop that rotates them stay the same objects. + // Extension credentials are owned by this supervisor session and shared + // by every gateway connection it opens, so the middleware registry's + // bearer slots and the policy poll loop that rotates them stay the same + // objects. let extension_credentials = openshell_extension_core::ExtensionCredentialStore::new(); let selected_backend = backend_setup::SelectedBackend::select( @@ -827,7 +1068,17 @@ pub async fn run_sandbox_with_backend( sandbox_id.as_deref(), &auth_bundle, )?; - let sandbox_bearer = openshell_core::grpc_client::install_supervisor_auth_bundle(&auth_bundle)?; + let gateway_credentials = match gateway_credentials { + Some(credentials) => credentials, + None => openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&auth_bundle)?, + }; + let sandbox_bearer = gateway_credentials.sandbox_bearer().ok_or_else(|| { + miette::miette!("supervisor gateway credentials carry no Sandbox Protocol credential") + })?; + // Every gateway client of this session uses its own credentials. + let gateway = openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new(endpoint, gateway_credentials.clone()) + }); #[cfg(unix)] let ssh_host_key = if ssh_socket_path.is_some() { Some(openshell_supervisor_process::ssh::parse_host_key( @@ -852,7 +1103,13 @@ pub async fn run_sandbox_with_backend( sandbox.id = sandbox_id.as_deref().unwrap_or_default(), otel.status_code = tracing::field::Empty, ); - openshell_otel::set_parent_from_environment(&startup); + if let Some(trace_context) = &trace_context { + openshell_otel::set_parent_from_trace_context( + &startup, + Some(&trace_context.traceparent), + trace_context.tracestate.as_deref(), + ); + } let startup_status = startup.in_scope(openshell_otel::ErrorStatusGuard::current); let (image_yaml, invalid_image) = selected_backend .discover_policy(sandbox_bearer.clone()) @@ -872,7 +1129,6 @@ pub async fn run_sandbox_with_backend( let vm_policy_identity = selected_backend.vm_policy_identity(); // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); let ( policy, @@ -886,7 +1142,7 @@ pub async fn run_sandbox_with_backend( ) = load_policy_with_gateway( sandbox_id.clone(), sandbox.clone(), - openshell_endpoint.clone(), + gateway.clone(), policy_rules, policy_data, &extension_credentials, @@ -894,7 +1150,7 @@ pub async fn run_sandbox_with_backend( vm_policy_identity, Some(image_discovery), &RemoteStartupGateway { - endpoint: openshell_endpoint.clone().unwrap_or_default(), + gateway: gateway.clone(), }, ) .instrument(tracing::info_span!(parent: &startup, "supervisor.policy.load")) @@ -915,8 +1171,8 @@ pub async fn run_sandbox_with_backend( // Fetch provider environment variables from the server. // This is done after loading the policy so the sandbox can still start // even if provider env fetch fails (graceful degradation). - let environment = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id) + let environment = if let (Some(id), Some(gateway)) = (&sandbox_id, &gateway) { + match openshell_core::grpc_client::fetch_provider_environment(gateway, id) .instrument(startup.clone()) .await { @@ -1109,9 +1365,14 @@ pub async fn run_sandbox_with_backend( // unnecessarily delay DNS and network readiness. false, &provider_credentials, - sandbox_id.as_deref(), - sandbox_name_for_agg.as_deref(), - openshell_endpoint_for_proxy.as_deref(), + openshell_supervisor_network::run::NetworkSession { + sandbox_id: sandbox_id.clone(), + sandbox_name: sandbox_name_for_agg.clone(), + gateway: gateway.clone(), + provider_workload_api_socket, + denial_log, + proxy_tls, + }, denial_tx, activity_tx, endpoint_observation_tx.clone(), @@ -1119,7 +1380,6 @@ pub async fn run_sandbox_with_backend( workspace_rx.clone(), &upstream_proxy_args, additional_ca_bundle.as_deref(), - None, remote_host_gateway_ip, #[cfg(target_os = "linux")] None, @@ -1142,14 +1402,14 @@ pub async fn run_sandbox_with_backend( // Spawn the denial-aggregator flush task. The aggregator drains proxy // denial events, batches them, and ships summaries to the gateway via // `SubmitPolicyAnalysis`. - if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { + if let (Some(rx), Some(gateway)) = (denial_rx, gateway.as_ref()) { // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall // back to the ID when the name isn't set. let agg_name = sandbox_name_for_agg .clone() .or_else(|| sandbox_id.clone()) .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); + let agg_gateway = gateway.clone(); let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") .ok() .and_then(|v| v.parse().ok()) @@ -1163,12 +1423,12 @@ pub async fn run_sandbox_with_backend( aggregator .run( |summaries| { - let endpoint = agg_endpoint.clone(); + let gateway = agg_gateway.clone(); let sandbox_name = agg_name.clone(); let workspace = denial_workspace_rx.borrow().clone(); async move { if let Err(e) = flush_proposals_to_gateway( - &endpoint, + &gateway, &sandbox_name, &workspace, summaries, @@ -1188,12 +1448,12 @@ pub async fn run_sandbox_with_backend( // Spawn the activity-aggregator flush task. The aggregator drains // anonymous activity events from the proxy, sanitizes deny groups, // and ships periodic summaries to the gateway. - if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { + if let (Some(rx), Some(gateway)) = (activity_rx, gateway.as_ref()) { let agg_name = sandbox_name_for_agg .clone() .or_else(|| sandbox_id.clone()) .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); + let agg_gateway = gateway.clone(); let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") .ok() @@ -1208,12 +1468,12 @@ pub async fn run_sandbox_with_backend( aggregator .run( move |summary| { - let endpoint = agg_endpoint.clone(); + let gateway = agg_gateway.clone(); let sandbox_name = agg_name.clone(); let workspace = activity_workspace_rx.borrow().clone(); async move { if let Err(e) = flush_activity_to_gateway( - &endpoint, + &gateway, &sandbox_name, &workspace, summary, @@ -1231,18 +1491,18 @@ pub async fn run_sandbox_with_backend( } // Spawn background policy poll task (gRPC mode only). - if let (Some(id), Some(sandbox), Some(endpoint), Some(engine)) = ( + if let (Some(id), Some(sandbox), Some(gateway), Some(engine)) = ( sandbox_id.as_deref(), sandbox.as_deref(), - openshell_endpoint.as_deref(), + gateway.as_ref(), opa_engine.as_ref(), ) { let poll_id = id.to_string(); let poll_sandbox = sandbox.to_string(); - let poll_endpoint = endpoint.to_string(); + let poll_gateway = gateway.clone(); let poll_engine = engine.clone(); - let poll_ocsf_enabled = ocsf_enabled.clone(); - let poll_ocsf_schema_version = ocsf_schema_version.clone(); + let poll_ocsf_enabled = ocsf_json.enabled.clone(); + let poll_ocsf_schema_version = ocsf_json.target_version.clone(); let poll_pid = entrypoint_pid.clone(); let poll_provider_credentials = provider_credentials.clone(); let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); @@ -1255,7 +1515,7 @@ pub async fn run_sandbox_with_backend( .and_then(|v| v.parse().ok()) .unwrap_or(10); let poll_ctx = PolicyPollLoopContext { - endpoint: poll_endpoint, + gateway: poll_gateway, sandbox_id: poll_id, sandbox: poll_sandbox, opa_engine: poll_engine, @@ -1326,9 +1586,9 @@ pub async fn run_sandbox_with_backend( let agent = running.agent(); let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( sandbox_id.as_deref(), - openshell_endpoint.as_deref(), + gateway.as_ref(), ssh_socket_path.as_deref(), - shared_ssh_socket_from_env(), + shared_ssh_socket, networking .as_ref() .and_then(|runtime| runtime.ca_file_paths.clone()), @@ -1345,29 +1605,23 @@ pub async fn run_sandbox_with_backend( info!(backend = %backend_name, "Control-mode access plane started"); startup_status.finish(Ok::<_, ()>(())).ok(); drop(startup); - let _provider_reporter = - sandbox_id - .as_ref() - .zip(openshell_endpoint.as_ref()) - .map(|(id, endpoint)| { - provider_readiness.start_reporter( - endpoint.clone(), - id.clone(), - provider_credentials.clone(), - supervisor_session_id.clone(), - running.exec(), - ) - }); - let mut control_readiness = health_socket_path - .map(ReadinessEndpoint::Unix) - .into_iter() - .chain(health_port.map(ReadinessEndpoint::Tcp)) - .map(|endpoint| ControlReadiness::start(endpoint, boundary_access.session_readiness())) - .collect::>>()?; + let _provider_reporter = sandbox_id + .as_ref() + .zip(gateway.as_ref()) + .map(|(id, gateway)| { + provider_readiness.start_reporter( + gateway.clone(), + id.clone(), + provider_credentials.clone(), + supervisor_session_id.clone(), + running.exec(), + ) + }); + let mut session_readiness = + SessionReadiness::start(readiness, boundary_access.session_readiness()); let instance_id = boundary_access.instance_id().to_string(); let wait_agent = agent.clone(); - let shutdown_requested = wait_for_control_shutdown_signal(); - tokio::pin!(shutdown_requested); + let mut shutdown_requested = shutdown_requested; let wait = async move { wait_agent .wait() @@ -1417,7 +1671,7 @@ pub async fn run_sandbox_with_backend( } }; if !retain_access { - control_readiness.clear(); + session_readiness.release(); } boundary_access .publish_main_exit(exit_code, await_main_process_attachment) @@ -1431,11 +1685,10 @@ pub async fn run_sandbox_with_backend( .wrap_err("persist canonical-process completion marker")?; } if !completion_cancelled - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + && let (Some(gateway), Some(id)) = (gateway.as_ref(), sandbox_id.as_deref()) { let report = openshell_supervisor_process::delegated::report_main_process_exit( - endpoint, + gateway, id, &instance_id, exit_code, @@ -1449,11 +1702,10 @@ pub async fn run_sandbox_with_backend( completion_phase_or_shutdown(drain, shutdown_requested.as_mut()).await; } if !completion_cancelled - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + && let (Some(gateway), Some(id)) = (gateway.as_ref(), sandbox_id.as_deref()) { let finalize = openshell_supervisor_process::delegated::finalize_main_process_exit( - endpoint, + gateway, id, &instance_id, ); @@ -1462,13 +1714,13 @@ pub async fn run_sandbox_with_backend( } if completion_cancelled { retain_access = false; - control_readiness.clear(); + session_readiness.release(); } if retain_access { info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; } - drop(control_readiness); + drop(session_readiness); drop(running); drop(boundary_access); exit_code @@ -1521,7 +1773,7 @@ fn persist_main_exit_marker(path: &std::path::Path, exit_code: i32) -> std::io:: /// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. async fn flush_proposals_to_gateway( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_name: &str, workspace: &str, summaries: Vec, @@ -1529,7 +1781,7 @@ async fn flush_proposals_to_gateway( use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::{DenialSummary, L7RequestSample}; - let client = CachedOpenShellClient::connect(endpoint).await?; + let client = CachedOpenShellClient::connect(gateway).await?; client.set_workspace(workspace.to_string()); let proto_summaries: Vec = summaries @@ -1591,7 +1843,7 @@ async fn flush_proposals_to_gateway( /// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. async fn flush_activity_to_gateway( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_name: &str, workspace: &str, summary: activity_aggregator::FlushableActivitySummary, @@ -1599,7 +1851,7 @@ async fn flush_activity_to_gateway( use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; - let client = CachedOpenShellClient::connect(endpoint).await?; + let client = CachedOpenShellClient::connect(gateway).await?; client.set_workspace(workspace.to_string()); let proto_summary = NetworkActivitySummary { @@ -2153,7 +2405,15 @@ trait StartupGateway: Send + Sync { } struct RemoteStartupGateway { - endpoint: String, + gateway: Option, +} + +impl RemoteStartupGateway { + fn gateway(&self) -> Result<&openshell_core::grpc_client::GatewayEndpoint> { + self.gateway + .as_ref() + .ok_or_else(|| miette::miette!("supervisor has no gateway endpoint")) + } } impl StartupGateway for RemoteStartupGateway { @@ -2161,13 +2421,13 @@ impl StartupGateway for RemoteStartupGateway { &self, sandbox: &str, ) -> Result { - openshell_core::grpc_client::fetch_settings_snapshot(&self.endpoint, sandbox).await + openshell_core::grpc_client::fetch_settings_snapshot(self.gateway()?, sandbox).await } async fn provider( &self, id: &str, ) -> Result { - openshell_core::grpc_client::fetch_provider_environment(&self.endpoint, id).await + openshell_core::grpc_client::fetch_provider_environment(self.gateway()?, id).await } async fn sync( &self, @@ -2176,7 +2436,7 @@ impl StartupGateway for RemoteStartupGateway { workspace: &str, ) -> Result { openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - &self.endpoint, + self.gateway()?, sandbox, policy, workspace, @@ -2192,7 +2452,7 @@ impl StartupGateway for RemoteStartupGateway { error: &str, ) -> Result<()> { openshell_core::grpc_client::report_sandbox_configuration( - &self.endpoint, + self.gateway()?, id, instance_id, snapshot, @@ -2291,7 +2551,7 @@ impl CapturedProviderEnvironment { async fn load_policy( sandbox_id: Option, sandbox: Option, - openshell_endpoint: Option, + openshell_gateway: Option, policy_rules: Option, policy_data: Option, extension_credentials: &openshell_extension_core::ExtensionCredentialStore, @@ -2309,7 +2569,7 @@ async fn load_policy( load_policy_with_gateway( sandbox_id, sandbox, - openshell_endpoint.clone(), + openshell_gateway.clone(), policy_rules, policy_data, extension_credentials, @@ -2317,7 +2577,7 @@ async fn load_policy( None, None, &RemoteStartupGateway { - endpoint: openshell_endpoint.unwrap_or_default(), + gateway: openshell_gateway, }, ) .await @@ -2330,7 +2590,7 @@ async fn load_policy( async fn load_policy_with_gateway( sandbox_id: Option, sandbox: Option, - openshell_endpoint: Option, + openshell_gateway: Option, policy_rules: Option, policy_data: Option, extension_credentials: &openshell_extension_core::ExtensionCredentialStore, @@ -2351,7 +2611,7 @@ async fn load_policy_with_gateway( use openshell_core::proto::ConfigurationAdmissionState; // File mode: load OPA engine from rego rules + YAML data (dev override) if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { - if sandbox_id.is_some() && openshell_endpoint.is_some() { + if sandbox_id.is_some() && openshell_gateway.is_some() { return Err(miette::miette!( "Local policy overrides cannot be combined with gateway-managed activation; replace the sandbox policy through the gateway" )); @@ -2417,12 +2677,11 @@ async fn load_policy_with_gateway( } // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data - if let (Some(id), Some(sandbox), Some(endpoint)) = (&sandbox_id, &sandbox, &openshell_endpoint) - { + if let (Some(id), Some(sandbox), Some(endpoint)) = (&sandbox_id, &sandbox, &openshell_gateway) { info!( sandbox_id = %id, sandbox = %sandbox, - endpoint = %endpoint, + endpoint = %endpoint.url(), "Fetching sandbox policy via gRPC" ); let instance_id = uuid::Uuid::new_v4().to_string(); @@ -3511,14 +3770,14 @@ trait PolicyGatewayClient: Clone + Send + Sync + 'static { /// Refresh credentials while preserving the activated delivery requirement. fn fetch_provider_environment( &self, - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, requires_stable_placeholders: bool, ) -> impl Future> + Send { async move { openshell_core::grpc_client::fetch_provider_environment_with_stable_placeholders( - endpoint, + gateway, sandbox_id, requires_stable_placeholders, ) @@ -3706,7 +3965,7 @@ fn enqueue_policy_status(sender: &UnboundedSender, update: P /// and preserves the original construction error as the reported message. A /// delivery failure here is swallowed so it can never mask that error. async fn report_initial_policy_failure( - endpoint: &str, + gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, revision: Option<&LoadedPolicyRevision>, error: &miette::Report, @@ -3717,7 +3976,7 @@ async fn report_initial_policy_failure( }) else { return; }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(gateway).await { Ok(client) => client, Err(e) => { warn!(error = %e, "Failed to connect to report initial policy failure"); @@ -3749,7 +4008,7 @@ async fn report_initial_policy_failure( /// When the entrypoint PID is available, policy reloads include symlink /// resolution for binary paths via the container filesystem. struct PolicyPollLoopContext { - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, /// Canonical sandbox reference used by name-scoped configuration APIs. sandbox: String, @@ -4141,6 +4400,14 @@ fn emit_policy_validation_failure( )] mod tests { use super::*; + + pub fn test_gateway() -> openshell_core::grpc_client::GatewayEndpoint { + openshell_core::grpc_client::GatewayEndpoint::new( + "http://unused.invalid", + openshell_core::grpc_client::GatewayCredentials::from_gateway_token("test-token") + .expect("test gateway credentials"), + ) + } use std::sync::atomic::{AtomicUsize, Ordering}; /// Admission must reject this input before any backend decodes its payload @@ -4745,7 +5012,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".into()), Some("sandbox".into()), - Some("http://unused.invalid".into()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -4884,7 +5151,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -4961,7 +5228,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5007,7 +5274,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5340,7 +5607,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5489,7 +5756,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5815,7 +6082,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), @@ -5910,7 +6177,7 @@ network_policies: load_policy_with_gateway( Some("sandbox-id".to_string()), Some("sandbox".to_string()), - Some("http://unused.invalid".to_string()), + Some(test_gateway()), None, None, &openshell_extension_core::ExtensionCredentialStore::new(), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 0442fd10a2..daafbc364b 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -413,11 +413,10 @@ fn main() -> Result<()> { let setup = backend_setup(&args, &admitted_backend)?; let descriptor = backend_descriptor(&args, &admitted_backend)?; let auth = auth_bundle(&args)?; - // Install the driver-provisioned session before starting log push or - // any other gateway client. `run_sandbox` obtains the same Sandbox - // Protocol bearer slot after validating the descriptor binding. - let _ = openshell_core::grpc_client::install_supervisor_auth_bundle(&auth)?; - Some((setup, descriptor, auth, admitted_backend)) + // Log push and supervision share these credentials, so either one's + // renewal keeps the other's gateway token current. + let credentials = openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&auth)?; + Some((setup, descriptor, auth, admitted_backend, credentials)) } else { None }; @@ -443,7 +442,7 @@ fn main() -> Result<()> { }); let ocsf_headers = isolation_inputs .as_ref() - .and_then(|(_, _, auth, _)| auth.ocsf_otlp_headers.as_ref()) + .and_then(|(_, _, auth, _, _)| auth.ocsf_otlp_headers.as_ref()) .map(openshell_core::jwt::SecretOtlpHeaders::expose_secret); let (ocsf_export, ocsf_setup_error) = runtime.block_on(async { start_ocsf_export( @@ -457,11 +456,18 @@ fn main() -> Result<()> { let exit_code = runtime.block_on(async move { openshell_crypto::tls::ensure_default_provider(); - let log_push_state = if args.role == SupervisorRole::IsolationBackend - && let (Some(sandbox_id), Some(endpoint)) = (&args.sandbox_id, &args.openshell_endpoint) - { + let log_push_state = if let (Some(sandbox_id), Some(endpoint), Some(credentials)) = ( + &args.sandbox_id, + &args.openshell_endpoint, + isolation_inputs + .as_ref() + .map(|(_, _, _, _, credentials)| credentials), + ) { let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( - endpoint.clone(), + openshell_core::grpc_client::GatewayEndpoint::new( + endpoint.as_str(), + credentials.clone(), + ), sandbox_id.clone(), ); let layer = @@ -538,8 +544,13 @@ fn main() -> Result<()> { match args.role { SupervisorRole::IsolationBackend => { info!(command = ?command, "Starting sandbox supervision"); - let Some((backend_setup, backend_descriptor, auth_bundle, admitted_backend)) = - isolation_inputs + let Some(( + backend_setup, + backend_descriptor, + auth_bundle, + admitted_backend, + gateway_credentials, + )) = isolation_inputs else { return Err(miette::miette!( "isolation-backend role started without validated runtime inputs" @@ -567,6 +578,7 @@ fn main() -> Result<()> { additional_ca_bundle: args.additional_ca_bundle, backend_descriptor, auth_bundle, + gateway_credentials: Some(gateway_credentials), admitted_isolation_backend: Some(admitted_backend), main_exit_marker: args.main_exit_marker, }, diff --git a/crates/openshell-supervisor/src/provider_readiness.rs b/crates/openshell-supervisor/src/provider_readiness.rs index af883dc265..347f7ca3e9 100644 --- a/crates/openshell-supervisor/src/provider_readiness.rs +++ b/crates/openshell-supervisor/src/provider_readiness.rs @@ -283,7 +283,7 @@ impl Tracker { /// Report while the owner holds this handle; reconnects use the accepted session. pub(crate) fn start_reporter( &self, - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, credentials: ProviderCredentialState, sessions: watch::Receiver>, @@ -297,7 +297,7 @@ impl Tracker { sessions, boundary, GatewayReporter { - endpoint, + gateway, sandbox_id, }, ) @@ -399,7 +399,7 @@ trait ReportClient: Send + Sync { } struct GatewayReporter { - endpoint: String, + gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, } @@ -409,7 +409,7 @@ impl ReportClient for GatewayReporter { observation: ProviderReadinessObservation, ) -> miette::Result { openshell_core::grpc_client::report_provider_readiness( - &self.endpoint, + &self.gateway, &self.sandbox_id, observation, ) From 1c539421d601ecdfccf65be39bf0ce410bc25f78 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 18:30:10 -0700 Subject: [PATCH 09/25] docs: describe the experimental shared supervisor Document the gateway supervisor topology, its chart-owned connection settings and limits, and how to debug and test it. Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index d924bf1ece..9082bf2d76 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -68,15 +68,16 @@ gatewayConfig: supervisor_topology: gateway ``` -The gateway chart creates a separate, single-replica shared-host placeholder Deployment -in the release namespace, regardless of shared/managed/operator workspace mode. -Use a supervisor image built from this branch: `--mode=shared` is new. -The chart sets `--gateway-endpoint` from the gateway's gRPC endpoint and -`--address` to `0.0.0.0:5501`. Check its rollout and TCP health probes on this -listener (container port `supervisor`, 5501). The outbound gateway connection and -session RPCs are not implemented yet; sandboxes -still use dedicated supervisors. `workspace` is reserved and rejected. Omit the -field to disable it. +The gateway chart creates a single-replica shared supervisor Deployment, +Service, and NetworkPolicy in the release namespace, regardless of +shared/managed/operator workspace mode, and the certgen hook issues its +session-control Secrets. New sandbox generations run as sessions in that +process. Build the gateway, supervisor, and sandbox images from the same branch: +all three must support the routed runtime connection. `workspace` is reserved +and rejected. Omit the field to disable it. `mise run +e2e:kubernetes:shared-supervisor` runs the focused e2e +(`ci/values-supervisor.yaml`); layer a workspace-mode overlay +through `OPENSHELL_E2E_KUBE_EXTRA_VALUES` to cover managed namespaces. ### 2. Deploy OpenShell From bce3e09f0773f579b15e8c3cc52a7b3316fb75db Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 17:06:21 -0700 Subject: [PATCH 10/25] feat(supervisor): host concurrent sessions on session-bound runtimes SessionHost runs several supervisor sessions in one process. Each session gets its own Tokio runtime, and every thread that runtime starts, plus the thread driving it, is bound to the session's OCSF context, OCSF JSON settings, log sink, and denial log. Tasks, blocking work, reconnects, and credential renewal spawned by a session therefore keep its identity, while threads outside any session carry none and push no logs. Gateway credentials stay explicit per session. Removing a session stops its workload gracefully, or cancels it at once while it is still starting, then shuts down only its runtime and revokes its gateway credentials. The key stays registered until the session thread has finished, so the same generation cannot gain a second owner while one is stopping. A panicking session reports a failure, clears its readiness, and is revoked the same way. Hosted sessions get a private proxy trust directory and serve policy.local denials from their own log. A focused test runs two sessions concurrently and checks that nested and blocking tasks see their own identity, gateway requests carry their own bearer, log lines reach only their own stream, and removing one leaves the other supervising. Closes #4699 Signed-off-by: Drew Newberry --- Cargo.lock | 5 + crates/openshell-supervisor/Cargo.toml | 5 + crates/openshell-supervisor/README.md | 18 + crates/openshell-supervisor/src/lib.rs | 1 + .../openshell-supervisor/src/session_host.rs | 938 ++++++++++++++++++ 5 files changed, 967 insertions(+) create mode 100644 crates/openshell-supervisor/src/session_host.rs diff --git a/Cargo.lock b/Cargo.lock index 0a2fcb8c92..025ba026d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5093,8 +5093,13 @@ dependencies = [ name = "openshell-supervisor" version = "0.0.0" dependencies = [ + "bytes", "clap", "futures", + "http 1.4.0", + "http-body-util", + "hyper", + "hyper-util", "miette", "nix 0.29.0", "openshell-core", diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index 9e2e0ea8ad..a42c370404 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -58,6 +58,11 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } tonic-prost = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } tokio = { workspace = true, features = ["test-util"] } futures = { workspace = true } temp-env = "0.3" diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 864a24a9d9..12c08cebb0 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -40,6 +40,24 @@ readiness, and stops the session on SIGTERM or SIGINT. The binary shares one `GatewayCredentials` between log push and supervision, so renewal by either keeps both current. +`session_host::SessionHost` runs several sessions in one process. Each session +gets its own Tokio runtime; every thread of that runtime, and the thread +driving it, is bound to the session's OCSF context, OCSF JSON settings, log +sink, and denial log. Spawned tasks, blocking work, callbacks, reconnects, and +renewal therefore inherit the session's identity, and threads outside a +session carry none and push no logs. The OCSF context also supplies the +sandbox identity in middleware requests. + +Removing a session stops its workload, shuts down only its runtime, and +revokes its gateway credentials; the key stays registered until the session's +thread has finished. A session that panics reports a failure and is revoked +the same way. A session that is still starting is cancelled at once; the +sandbox runtime then loses supervision and fails closed, as it does when a +dedicated supervisor exits. A hosted session without an explicit proxy trust +directory receives a private one for its lifetime. Hosted processes must +install `LogPushLayer::thread_bound` and an `OcsfShorthandLayer` writing to +`SessionDenialWriter`. + ## Console logging Diagnostics and OCSF shorthand share a bounded, nonblocking stderr writer. With `ocsf_json_enabled=true`, the same writer also receives timestamped `OCSF-JSON` records. Each formatter submits a complete line in one write so concurrent producers cannot interleave records in the queue. The 1,024-line queue drops new lines when full rather than waiting for stderr. diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index ab05a5e1fb..bd402b5d38 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -24,6 +24,7 @@ mod denial_aggregator; mod endpoint_status; mod mechanistic_mapper; mod provider_readiness; +pub mod session_host; use config_runtime::{agent_proposals_enabled_from_settings, run_policy_poll_loop}; use miette::{IntoDiagnostic, Result, WrapErr}; diff --git a/crates/openshell-supervisor/src/session_host.rs b/crates/openshell-supervisor/src/session_host.rs new file mode 100644 index 0000000000..fcb4e989ff --- /dev/null +++ b/crates/openshell-supervisor/src/session_host.rs @@ -0,0 +1,938 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Several supervisor sessions in one process. +//! +//! Each session runs on its own Tokio runtime. Every thread that runtime +//! starts, and the thread driving it, is bound to the session's OCSF context, +//! OCSF JSON settings, log sink, and denial log, so tasks, callbacks, +//! reconnects, and credential renewal that the session spawns inherit its +//! identity. Unbound threads carry no sandbox identity and push no logs. +//! Gateway credentials are never bound to threads; each session passes its +//! own explicitly. Stopping a session shuts down only its runtime and revokes +//! its gateway credentials. + +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use miette::Result; +use openshell_core::grpc_client::GatewayCredentials; +use openshell_ocsf::{EventContext, OcsfJsonSettings}; +use openshell_supervisor_network::policy_local::SessionDenialLog; +use openshell_supervisor_process::log_push::LogSink; +use tokio::sync::watch; + +use crate::SessionControls; + +/// Worker threads in each session runtime. +const SESSION_WORKER_THREADS: usize = 2; +/// Bound on joining a session thread after its runtime is told to stop. +const RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Per-session values bound to every thread of the session's runtime. +#[derive(Clone)] +pub struct SessionBinding { + pub event_context: Arc, + pub ocsf_json: OcsfJsonSettings, + pub log_sink: Option, + pub denial_log: SessionDenialLog, +} + +impl SessionBinding { + fn bind(&self) { + openshell_ocsf::ctx::bind_thread_ctx(Some(self.event_context.clone())); + openshell_ocsf::bind_thread_json_settings(Some(self.ocsf_json.clone())); + openshell_supervisor_process::log_push::bind_thread_sink(self.log_sink.clone()); + bind_thread_denial_log(Some(self.denial_log.clone())); + } + + fn unbind() { + openshell_ocsf::ctx::bind_thread_ctx(None); + openshell_ocsf::bind_thread_json_settings(None); + openshell_supervisor_process::log_push::bind_thread_sink(None); + bind_thread_denial_log(None); + } +} + +std::thread_local! { + static THREAD_DENIAL_LOG: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +fn bind_thread_denial_log(log: Option) { + THREAD_DENIAL_LOG.with(|current| *current.borrow_mut() = log); +} + +/// Shorthand-log destination for hosted sessions' recent denials. +/// +/// Each line goes to the denial log of the session bound to the emitting +/// thread. Lines from unbound threads are dropped, so no session can read +/// another's denials through `policy.local`. +#[derive(Debug, Default, Clone, Copy)] +pub struct SessionDenialWriter; + +impl std::io::Write for SessionDenialWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + THREAD_DENIAL_LOG.with(|log| { + if let Some(log) = log.borrow().as_ref() { + for line in String::from_utf8_lossy(bytes).lines() { + log.record(line); + } + } + }); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Observable lifecycle of one hosted session. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SessionPhase { + /// Startup has not yet produced an accepted gateway session. + Starting, + /// The workload started and the gateway accepted supervision at least once. + Supervising, + /// The owner asked the session to stop. + Stopping, + /// Supervision ended with the workload's exit status. + Exited(i32), + /// Startup or supervision failed; the workload is no longer supervised. + Failed(String), +} + +impl SessionPhase { + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Exited(_) | Self::Failed(_)) + } +} + +/// Current state of one hosted session. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SessionStatus { + pub phase: SessionPhase, + /// Gateway supervision is currently accepted for this session. + pub ready: bool, +} + +/// Body of a hosted session, constructed on its own runtime. +pub type SessionBody = + Box Pin>>> + Send + 'static>; + +/// The hosted-session registration was rejected. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum SessionHostError { + #[error("a session with this key is already hosted")] + Duplicate, + #[error("start session thread: {0}")] + Thread(String), +} + +struct HostedSession { + /// Distinguishes this session from a later one that reuses its key. + serial: u64, + stop: watch::Sender, + abort: watch::Sender, + phase: watch::Receiver, + ready: watch::Receiver, + thread: Option>, + /// Final status of the removal in progress, shared by every remover. + removal: Option>>, +} + +impl HostedSession { + fn status(&self) -> SessionStatus { + let phase = self.phase.borrow().clone(); + // The runner always reports a terminal phase, even while unwinding; + // a closed channel without one still means supervision has ended. + if !phase.is_terminal() && self.phase.has_changed().is_err() { + return SessionStatus { + phase: SessionPhase::Failed("session thread ended unexpectedly".to_string()), + ready: false, + }; + } + SessionStatus { + phase, + ready: *self.ready.borrow(), + } + } +} + +type Sessions = Arc>>; + +fn lock(sessions: &Sessions) -> MutexGuard<'_, HashMap> { + sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// Runs and stops supervisor sessions, each on its own bound runtime. +#[derive(Default)] +pub struct SessionHost { + sessions: Sessions, + next_serial: AtomicU64, +} + +impl SessionHost { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Start `body` on a new runtime bound to `binding`. + /// + /// `credentials`, when present, are revoked once the session's runtime has + /// stopped, so nothing can reuse them after the session ends. + /// + /// # Errors + /// + /// Returns [`SessionHostError::Duplicate`] when `key` is already hosted, + /// including a session that ended but has not been removed. + pub fn start( + &self, + key: &str, + binding: SessionBinding, + credentials: Option, + body: SessionBody, + ) -> std::result::Result<(), SessionHostError> { + let mut sessions = lock(&self.sessions); + if sessions.contains_key(key) { + return Err(SessionHostError::Duplicate); + } + let (stop, stop_requested) = watch::channel(false); + let (abort, abort_requested) = watch::channel(false); + let (phase_tx, phase) = watch::channel(SessionPhase::Starting); + let (ready_tx, ready) = watch::channel(false); + let thread = std::thread::Builder::new() + .name("openshell-session".to_string()) + .spawn(move || { + run_hosted_session( + &binding, + credentials, + body, + SessionSignals { + stop_requested, + abort_requested, + phase: phase_tx, + ready: ready_tx, + }, + ); + }) + .map_err(|error| SessionHostError::Thread(error.to_string()))?; + sessions.insert( + key.to_string(), + HostedSession { + serial: self.next_serial.fetch_add(1, Ordering::Relaxed), + stop, + abort, + phase, + ready, + thread: Some(thread), + removal: None, + }, + ); + Ok(()) + } + + /// Status of a hosted session, including one that ended but has not been + /// removed. + #[must_use] + pub fn status(&self, key: &str) -> Option { + lock(&self.sessions).get(key).map(HostedSession::status) + } + + /// Keys of every hosted session. + #[must_use] + pub fn keys(&self) -> Vec { + lock(&self.sessions).keys().cloned().collect() + } + + /// Stop one session and forget it. Other sessions keep running. + /// + /// A supervising session terminates its workload gracefully within + /// `grace`; one still starting is cancelled at once, because startup has + /// no safe intermediate state to wind down. Removal runs on its own task, + /// so it completes even when the caller stops waiting, and concurrent + /// callers share it and its result. The key stays registered until the + /// session's thread has finished, so the same key cannot start a second + /// owner while this one is stopping. Returns the final status, or `None` + /// when `key` is not hosted. + pub async fn remove(&self, key: &str, grace: Duration) -> Option { + let (mut removed, phase) = self.begin_removal(key, grace)?; + // The removal task ends without a report only with its runtime. + removed + .wait_for(Option::is_some) + .await + .map_or_else(|_| Some(terminal_status(&phase)), |status| status.clone()) + } + + fn begin_removal( + &self, + key: &str, + grace: Duration, + ) -> Option<( + watch::Receiver>, + watch::Receiver, + )> { + let mut sessions = lock(&self.sessions); + let session = sessions.get_mut(key)?; + if let Some(removed) = &session.removal { + return Some((removed.clone(), session.phase.clone())); + } + session.stop.send_replace(true); + if *session.phase.borrow() == SessionPhase::Starting { + session.abort.send_replace(true); + } + let (report, removed) = watch::channel(None); + session.removal = Some(removed.clone()); + tokio::spawn( + Removal { + sessions: self.sessions.clone(), + key: key.to_string(), + serial: session.serial, + phase: session.phase.clone(), + thread: session.thread.take(), + grace, + report, + } + .run(), + ); + Some((removed, session.phase.clone())) + } +} + +fn terminal_status(phase: &watch::Receiver) -> SessionStatus { + SessionStatus { + phase: phase.borrow().clone(), + ready: false, + } +} + +/// The single owner of one session's removal. +struct Removal { + sessions: Sessions, + key: String, + serial: u64, + phase: watch::Receiver, + thread: Option>, + grace: Duration, + report: watch::Sender>, +} + +impl Removal { + async fn run(mut self) { + let finished = + tokio::time::timeout(self.grace, self.phase.wait_for(SessionPhase::is_terminal)) + .await + .is_ok(); + if !finished + && let Some(session) = lock(&self.sessions) + .get(&self.key) + .filter(|session| session.serial == self.serial) + { + session.abort.send_replace(true); + } + if let Some(thread) = self.thread.take() { + let _ = tokio::task::spawn_blocking(move || thread.join()).await; + } + let removed = { + let mut sessions = lock(&self.sessions); + match sessions.get(&self.key) { + Some(session) if session.serial == self.serial => sessions.remove(&self.key), + _ => None, + } + }; + let status = + removed.map_or_else(|| terminal_status(&self.phase), |session| session.status()); + self.report.send_replace(Some(status)); + } +} + +struct SessionSignals { + stop_requested: watch::Receiver, + abort_requested: watch::Receiver, + phase: watch::Sender, + ready: watch::Sender, +} + +/// Reports a hosted session's end and revokes its credentials, including when +/// the session thread unwinds. +struct SessionExit { + credentials: Option, + phase: watch::Sender, + ready: watch::Sender, + outcome: Option, +} + +impl Drop for SessionExit { + fn drop(&mut self) { + if let Some(credentials) = self.credentials.take() { + credentials.revoke(); + } + self.ready.send_replace(false); + let outcome = self.outcome.take().unwrap_or_else(|| { + SessionPhase::Failed("session thread ended without reporting a result".to_string()) + }); + self.phase.send_replace(outcome); + SessionBinding::unbind(); + } +} + +fn run_hosted_session( + binding: &SessionBinding, + credentials: Option, + body: SessionBody, + signals: SessionSignals, +) { + let SessionSignals { + stop_requested, + mut abort_requested, + phase, + ready, + } = signals; + let mut exit = SessionExit { + credentials, + phase: phase.clone(), + ready: ready.clone(), + outcome: None, + }; + binding.bind(); + let thread_binding = binding.clone(); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(SESSION_WORKER_THREADS) + .thread_name("openshell-session-worker") + .enable_all() + .on_thread_start(move || thread_binding.bind()) + .on_thread_stop(SessionBinding::unbind) + .build(); + let outcome = match runtime { + Ok(runtime) => { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + runtime.block_on(async { + let (readiness, mut accepted) = watch::channel(false); + let mirror_phase = phase.clone(); + let mirror_ready = ready.clone(); + let mirror = tokio::spawn(async move { + loop { + let is_ready = *accepted.borrow_and_update(); + mirror_ready.send_replace(is_ready); + if is_ready { + mirror_phase.send_if_modified(|current| { + let promote = *current == SessionPhase::Starting; + if promote { + *current = SessionPhase::Supervising; + } + promote + }); + } + if accepted.changed().await.is_err() { + mirror_ready.send_replace(false); + return; + } + } + }); + let mut stop = stop_requested.clone(); + let stop_phase = phase.clone(); + let shutdown = Box::pin(async move { + let _ = stop.wait_for(|requested| *requested).await; + stop_phase.send_replace(SessionPhase::Stopping); + }); + let session = body(SessionControls { + shutdown, + readiness, + }); + let outcome = tokio::select! { + result = session => result, + _ = abort_requested.wait_for(|requested| *requested) => { + Err(miette::miette!("session was cancelled before it finished")) + } + }; + mirror.abort(); + outcome + }) + })); + runtime.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT); + outcome.unwrap_or_else(|_| Err(miette::miette!("session panicked"))) + } + Err(error) => Err(miette::miette!("build session runtime: {error}")), + }; + exit.outcome = Some(match outcome { + Ok(code) => SessionPhase::Exited(code), + Err(error) => SessionPhase::Failed(error.to_string()), + }); +} + +/// Trusted inputs for one supervisor session hosted with others. +pub struct HostedSessionSpec { + /// Unique key for this generation within the host. + pub key: String, + pub backend_setup: Box, + pub config: crate::SessionConfig, + /// Workload image recorded in this session's OCSF events. + pub container_image: String, +} + +impl SessionHost { + /// Start a supervisor session with its own OCSF identity, OCSF JSON + /// settings, log push stream, denial log, and gateway credentials. + /// + /// # Errors + /// + /// Fails when the launch credentials are invalid or the key is hosted. + pub fn start_supervisor_session(&self, spec: HostedSessionSpec) -> Result<()> { + let HostedSessionSpec { + key, + backend_setup, + mut config, + container_image, + } = spec; + let credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; + // Sessions must never share proxy trust files; give one a private + // directory that lives exactly as long as the session. + let private_tls_dir = if config.proxy_tls.directory.is_none() { + let prepared = crate::prepare_network_proxy_tls_dir(None)?; + config.proxy_tls.directory = Some(prepared.path.clone()); + Some(prepared) + } else { + None + }; + let gateway = config.openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new(endpoint, credentials.clone()) + }); + let (log_tx, log_rx) = tokio::sync::mpsc::channel( + openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES, + ); + let log_push = config.sandbox_id.clone().zip(gateway); + let binding = SessionBinding { + event_context: Arc::new(crate::session_event_context( + config.sandbox_id.as_deref(), + config.sandbox.as_deref(), + container_image, + )), + ocsf_json: OcsfJsonSettings::default(), + log_sink: log_push + .as_ref() + .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), + denial_log: SessionDenialLog::default(), + }; + config.gateway_credentials = Some(credentials.clone()); + config.ocsf_json = binding.ocsf_json.clone(); + config.denial_log = openshell_supervisor_network::policy_local::DenialLogSource::Session( + binding.denial_log.clone(), + ); + let body: SessionBody = Box::new(move |controls| { + Box::pin(async move { + let _private_tls_dir = private_tls_dir; + let _log_push = log_push.map(|(sandbox_id, gateway)| { + openshell_supervisor_process::log_push::start_log_push( + gateway, sandbox_id, log_rx, + ) + }); + Box::pin(crate::run_session(backend_setup.as_ref(), config, controls)).await + }) + }); + self.start(&key, binding, Some(credentials), body) + .map_err(|error| miette::miette!("{error}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use http_body_util::BodyExt as _; + use openshell_core::grpc_client::GatewayEndpoint; + use openshell_core::proto::SandboxLogLine; + use tokio::sync::mpsc; + use tracing_subscriber::layer::SubscriberExt as _; + + type Captured = Arc)>>>; + + /// Records each gRPC request's bearer and body, then answers `UNIMPLEMENTED`. + async fn capture_gateway() -> (String, Captured) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let captured = Captured::default(); + let sink = captured.clone(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let sink = sink.clone(); + tokio::spawn(async move { + let service = hyper::service::service_fn( + move |request: http::Request| { + let sink = sink.clone(); + async move { + let authorization = request + .headers() + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let body = request + .into_body() + .collect() + .await + .map(|body| body.to_bytes().to_vec()) + .unwrap_or_default(); + sink.lock().unwrap().push((authorization, body)); + http::Response::builder() + .header("content-type", "application/grpc") + .header("grpc-status", "12") + .body(http_body_util::Empty::::new()) + } + }, + ); + let _ = hyper::server::conn::http2::Builder::new( + hyper_util::rt::TokioExecutor::new(), + ) + .serve_connection(hyper_util::rt::TokioIo::new(stream), service) + .await; + }); + } + }); + (format!("http://{address}"), captured) + } + + fn bearers_for(captured: &Captured, sandbox: &str) -> Vec { + captured + .lock() + .unwrap() + .iter() + .filter(|(_, body)| { + body.windows(sandbox.len()) + .any(|bytes| bytes == sandbox.as_bytes()) + }) + .map(|(authorization, _)| authorization.clone()) + .collect() + } + + struct Session { + identity: tokio::sync::oneshot::Receiver, + logs: mpsc::Receiver, + call_again: mpsc::Sender<()>, + alive: mpsc::Receiver<()>, + credentials: GatewayCredentials, + } + + fn start_session(host: &SessionHost, gateway_url: &str, sandbox: &'static str) -> Session { + let credentials = + GatewayCredentials::from_gateway_token(&format!("token-{sandbox}")).unwrap(); + let gateway = GatewayEndpoint::new(gateway_url, credentials.clone()); + let (log_tx, logs) = mpsc::channel(16); + let binding = SessionBinding { + event_context: Arc::new(crate::session_event_context( + Some(sandbox), + Some(sandbox), + String::new(), + )), + ocsf_json: OcsfJsonSettings::default(), + log_sink: Some(LogSink::new(sandbox, log_tx)), + denial_log: SessionDenialLog::default(), + }; + let (identity_tx, identity) = tokio::sync::oneshot::channel(); + let (call_again, mut calls) = mpsc::channel(1); + let (alive_tx, alive) = mpsc::channel::<()>(1); + let body: SessionBody = Box::new(move |controls| { + Box::pin(async move { + let SessionControls { + shutdown, + readiness, + } = controls; + // Nested tasks and the blocking pool inherit the session identity. + let observed = tokio::spawn(async { + tokio::task::spawn_blocking(|| openshell_ocsf::ctx::ctx().sandbox_id.clone()) + .await + .unwrap() + }) + .await + .unwrap(); + let _ = identity_tx.send(observed); + let _ = + openshell_core::grpc_client::fetch_settings_snapshot(&gateway, sandbox).await; + let dispatch = tracing::Dispatch::new( + tracing_subscriber::registry() + .with(openshell_supervisor_process::log_push::LogPushLayer::thread_bound()), + ); + tokio::spawn(async move { + tracing::dispatcher::with_default(&dispatch, || { + tracing::info!(target: "session_test", "emitted by {sandbox}"); + }); + }) + .await + .unwrap(); + // A long-lived task the session never stops itself. + tokio::spawn(async move { + let _alive = alive_tx; + std::future::pending::<()>().await; + }); + readiness.send_replace(true); + tokio::pin!(shutdown); + loop { + tokio::select! { + () = &mut shutdown => return Ok(0), + Some(()) = calls.recv() => { + let _ = openshell_core::grpc_client::fetch_settings_snapshot( + &gateway, sandbox, + ) + .await; + } + } + } + }) + }); + host.start(sandbox, binding, Some(credentials.clone()), body) + .unwrap(); + Session { + identity, + logs, + call_again, + alive, + credentials, + } + } + + async fn wait_supervising(host: &SessionHost, key: &str) { + tokio::time::timeout(Duration::from_secs(10), async { + while host.status(key).map(|status| status.phase) != Some(SessionPhase::Supervising) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("session reaches supervision"); + } + + fn unbound() -> SessionBinding { + SessionBinding { + event_context: Arc::new(crate::session_event_context(None, None, String::new())), + ocsf_json: OcsfJsonSettings::default(), + log_sink: None, + denial_log: SessionDenialLog::default(), + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn stopping_and_panicking_sessions_keep_their_key_and_fail_closed() { + let host = Arc::new(SessionHost::new()); + let body: SessionBody = Box::new(|controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + controls.shutdown.await; + // Winding down takes time; the key must stay taken meanwhile. + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(0) + }) + }); + host.start("generation", unbound(), None, body).unwrap(); + wait_supervising(&host, "generation").await; + let removal = tokio::spawn({ + let host = host.clone(); + async move { host.remove("generation", Duration::from_secs(10)).await } + }); + tokio::time::timeout(Duration::from_secs(10), async { + while host.status("generation").map(|status| status.phase) + != Some(SessionPhase::Stopping) + { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("session observes the stop request"); + let replacement: SessionBody = Box::new(|_| Box::pin(async { Ok(0) })); + assert_eq!( + host.start("generation", unbound(), None, replacement), + Err(SessionHostError::Duplicate) + ); + let stopped = removal.await.unwrap().expect("session was hosted"); + assert_eq!(stopped.phase, SessionPhase::Exited(0)); + assert!(host.status("generation").is_none()); + + let credentials = GatewayCredentials::from_gateway_token("panicking-token").unwrap(); + let panicking: SessionBody = Box::new(|controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + tokio::task::yield_now().await; + panic!("hosted session failure"); + }) + }); + host.start("panicking", unbound(), Some(credentials.clone()), panicking) + .unwrap(); + let status = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(status) = host.status("panicking") + && status.phase.is_terminal() + { + return status; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("panicking session reports an end"); + assert_eq!( + status, + SessionStatus { + phase: SessionPhase::Failed("session panicked".to_string()), + ready: false, + } + ); + let revoked = GatewayEndpoint::new("http://127.0.0.1:9", credentials); + let error = openshell_core::grpc_client::fetch_settings_snapshot(&revoked, "panicking") + .await + .unwrap_err(); + assert!(error.to_string().contains("revoked"), "{error}"); + assert!( + host.remove("panicking", Duration::from_secs(1)) + .await + .is_some() + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn removal_outlives_its_caller_and_reports_to_every_remover() { + let host = Arc::new(SessionHost::new()); + let winding_down = |code| -> SessionBody { + Box::new(move |controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + controls.shutdown.await; + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(code) + }) + }) + }; + host.start("generation", unbound(), None, winding_down(3)) + .unwrap(); + wait_supervising(&host, "generation").await; + // A caller that gives up, such as an expired RPC, strands nothing. + assert!( + tokio::time::timeout( + Duration::from_millis(20), + host.remove("generation", Duration::from_secs(10)) + ) + .await + .is_err() + ); + let removers = [(); 2].map(|()| { + let host = host.clone(); + tokio::spawn(async move { host.remove("generation", Duration::from_secs(10)).await }) + }); + for remover in removers { + let status = remover.await.unwrap().expect("session was hosted"); + assert_eq!(status.phase, SessionPhase::Exited(3)); + } + assert!(host.status("generation").is_none()); + + // The key is free again, and the earlier removal cannot touch its new owner. + host.start("generation", unbound(), None, winding_down(4)) + .unwrap(); + wait_supervising(&host, "generation").await; + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + host.status("generation").map(|status| status.phase), + Some(SessionPhase::Supervising) + ); + let status = host.remove("generation", Duration::from_secs(10)).await; + assert_eq!( + status.map(|status| status.phase), + Some(SessionPhase::Exited(4)) + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn hosted_sessions_keep_identity_credentials_and_lifecycle_separate() { + let (gateway_url, captured) = capture_gateway().await; + let host = SessionHost::new(); + let mut alpha = start_session(&host, &gateway_url, "sandbox-alpha"); + let mut beta = start_session(&host, &gateway_url, "sandbox-beta"); + wait_supervising(&host, "sandbox-alpha").await; + wait_supervising(&host, "sandbox-beta").await; + assert!(matches!( + host.start( + "sandbox-alpha", + SessionBinding { + event_context: Arc::new(crate::session_event_context( + None, + None, + String::new() + )), + ocsf_json: OcsfJsonSettings::default(), + log_sink: None, + denial_log: SessionDenialLog::default(), + }, + None, + Box::new(|_| Box::pin(async { Ok(0) })), + ), + Err(SessionHostError::Duplicate) + )); + + assert_eq!((&mut alpha.identity).await.unwrap(), "sandbox-alpha"); + assert_eq!((&mut beta.identity).await.unwrap(), "sandbox-beta"); + // Other tests in this process may set the process-wide context; a + // thread outside every session must still carry no session identity. + let outside = openshell_ocsf::ctx::ctx().sandbox_id.clone(); + assert!(!["sandbox-alpha", "sandbox-beta"].contains(&outside.as_str())); + assert_eq!( + bearers_for(&captured, "sandbox-alpha"), + ["Bearer token-sandbox-alpha"] + ); + assert_eq!( + bearers_for(&captured, "sandbox-beta"), + ["Bearer token-sandbox-beta"] + ); + for (session, sandbox) in [(&mut alpha, "sandbox-alpha"), (&mut beta, "sandbox-beta")] { + let line = session.logs.try_recv().expect("session log line"); + assert_eq!(line.sandbox_id, sandbox); + assert_eq!(line.message, format!("emitted by {sandbox}")); + assert!(session.logs.try_recv().is_err()); + } + + let stopped = host + .remove("sandbox-alpha", Duration::from_secs(10)) + .await + .expect("alpha was hosted"); + assert_eq!(stopped.phase, SessionPhase::Exited(0)); + assert!(!stopped.ready); + assert!(host.status("sandbox-alpha").is_none()); + assert_eq!(alpha.alive.recv().await, None, "alpha's tasks stopped"); + let revoked = GatewayEndpoint::new(gateway_url.as_str(), alpha.credentials.clone()); + let error = openshell_core::grpc_client::fetch_settings_snapshot(&revoked, "sandbox-alpha") + .await + .unwrap_err(); + assert!(error.to_string().contains("revoked"), "{error}"); + + beta.call_again.send(()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + while bearers_for(&captured, "sandbox-beta").len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("beta keeps calling the gateway"); + assert_eq!( + bearers_for(&captured, "sandbox-beta"), + ["Bearer token-sandbox-beta", "Bearer token-sandbox-beta"] + ); + assert_eq!( + host.status("sandbox-beta").map(|status| status.phase), + Some(SessionPhase::Supervising) + ); + assert!( + beta.alive + .try_recv() + .is_err_and(|error| error == mpsc::error::TryRecvError::Empty) + ); + + let stopped = host.remove("sandbox-beta", Duration::from_secs(10)).await; + assert_eq!( + stopped.map(|status| status.phase), + Some(SessionPhase::Exited(0)) + ); + } +} From 718885ff683ff5134970d27993a0e2290b02722e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 19:25:22 -0700 Subject: [PATCH 11/25] fix: tighten shared supervisor naming, removal deadlines, and route probes The shared supervisor's resource name appended its suffix before truncating, so a release fullname of 51 or 52 characters produced the gateway Service's own name. A fullname without room for the suffix now keeps a prefix and a hash of the whole fullname. Removing a session waits up to 20 s for its workload to stop and 5 s for the session runtime to shut down, but the driver gave DELETE the same 15 s budget as other session-control calls and reported normal removals as failures. The contract now defines the removal grace and a 45 s removal deadline, and deleting a sandbox no longer waits on a session whose workload is already going away. The shared host runs no supervisor OTLP trace or OCSF exporter. The driver warns at startup when the gateway configures that export, and the docs state the limit. experimental-host keeps its Unix-only imports in its Unix entry point. The e2e route probe fails when the supervisor neither refuses nor challenges a connection, or keeps a challenged connection open after a malformed TLS answer, and it checks that every probe reached the supervisor Pod. Signed-off-by: Drew Newberry --- crates/openshell-driver-kubernetes/README.md | 2 + .../openshell-driver-kubernetes/src/driver.rs | 15 +- .../src/shared_supervisor.rs | 26 +-- .../src/session_control.rs | 8 + crates/openshell-supervisor/README.md | 12 +- crates/openshell-supervisor/src/shared.rs | 9 +- deploy/helm/openshell/templates/_helpers.tpl | 13 +- .../helm/openshell/tests/supervisor_test.yaml | 16 ++ .../tests/kubernetes_shared_supervisor.rs | 160 ++++++++++++++---- 9 files changed, 205 insertions(+), 56 deletions(-) diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index aba9517d8d..b695e2c177 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -215,6 +215,8 @@ fails closed. The topology does not support upstream proxies or provider SPIFFE identity, and the shared supervisor is a single point of failure for the sandboxes it hosts. +Shared sessions export no supervisor OTLP traces or OCSF events; the driver +warns at startup when the gateway configures that export. When the setting is removed, reconciliation suspends shared generations, and a restart creates the dedicated generation's boundary Service the shared one never had. diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 28938bf1ce..40ac85d7b9 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1000,6 +1000,14 @@ impl KubernetesComputeDriver { }; let shared_supervisor = config.shared_supervisor.clone().map(SharedSupervisor::new); + if shared_supervisor.is_some() + && (config.supervisor_otlp_endpoint.is_some() + || config.supervisor_ocsf_export.is_some()) + { + warn!( + "the gateway supervisor topology does not export supervisor OTLP traces or OCSF events; shared sessions keep console logs and gateway log push" + ); + } let driver = Self { client, watch_client, @@ -4314,8 +4322,11 @@ impl KubernetesComputeDriver { match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { Ok(Ok(_response)) => { info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); - if let Some(placement) = &placement { - self.remove_shared_session(placement).await; + // The workload is going away, so its session cannot stop it + // gracefully; end the session without delaying the delete. + if let Some(placement @ SupervisorPlacement::Shared { .. }) = placement { + let driver = self.clone(); + tokio::spawn(async move { driver.remove_shared_session(&placement).await }); } { let pod_api = Api::::namespaced(self.client.clone(), &obj_namespace); diff --git a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs index 21df3cf20d..8061cef9f8 100644 --- a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs +++ b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs @@ -13,7 +13,7 @@ use bytes::Bytes; use http_body_util::{BodyExt as _, Full}; use kube::core::DynamicObject; use openshell_sandbox_backend::session_control::{ - SESSIONS_PATH, SessionList, SessionRegistration, SessionStatusReport, + SESSION_REMOVAL_TIMEOUT, SESSIONS_PATH, SessionList, SessionRegistration, SessionStatusReport, }; use crate::config::SharedSupervisorConfig; @@ -27,6 +27,8 @@ pub const ANNOTATION_SUPERVISOR_SESSION: &str = "openshell.ai/sandbox-runtime-su pub const GATEWAY_TOPOLOGY: &str = "gateway"; /// Bound on one session-control request, including connection and TLS. +/// Removals wait for the session's workload to stop and use the contract's +/// longer bound instead. const SESSION_CONTROL_TIMEOUT: Duration = Duration::from_secs(15); /// Where a generation's supervisor runs, as recorded when it started. @@ -152,16 +154,18 @@ impl SharedSupervisor { path: &str, body: Option>, ) -> Result<(http::StatusCode, Bytes), KubernetesDriverError> { - tokio::time::timeout( - SESSION_CONTROL_TIMEOUT, - self.request_once(method, path, body), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message( - "timed out calling the shared supervisor session-control API".to_string(), - ) - })? + let deadline = if method == http::Method::DELETE { + SESSION_REMOVAL_TIMEOUT + } else { + SESSION_CONTROL_TIMEOUT + }; + tokio::time::timeout(deadline, self.request_once(method, path, body)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out calling the shared supervisor session-control API".to_string(), + ) + })? } async fn request_once( diff --git a/crates/openshell-sandbox-backend/src/session_control.rs b/crates/openshell-sandbox-backend/src/session_control.rs index 3756244548..cced085a8c 100644 --- a/crates/openshell-sandbox-backend/src/session_control.rs +++ b/crates/openshell-sandbox-backend/src/session_control.rs @@ -11,6 +11,7 @@ //! Sandbox Protocol credentials it carries; it confers no other identity. use std::sync::Arc; +use std::time::Duration; use openshell_isolation_interface::contract::BackendError; use serde::{Deserialize, Serialize}; @@ -21,6 +22,13 @@ use crate::boundary_protocol::{SandboxRuntimeDescriptor, SandboxTransport}; pub const SESSIONS_PATH: &str = "/v1/sessions"; /// Largest registration body a supervisor accepts. pub const MAX_REGISTRATION_BYTES: usize = 256 * 1024; +/// Time a shared supervisor gives a removed session's workload to stop +/// before cancelling the session. +pub const SESSION_REMOVAL_GRACE: Duration = Duration::from_secs(20); +/// Bound on answering a removal request: the grace period, the session +/// runtime's shutdown, file cleanup, and the connection itself. A removal +/// continues on the supervisor if its caller stops waiting. +pub const SESSION_REMOVAL_TIMEOUT: Duration = Duration::from_secs(45); /// PEM private key that never appears in `Debug` output. #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 8a10177189..968abf96b4 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -124,7 +124,11 @@ one process for the Kubernetes gateway supervisor topology. It is Unix-only. session control. SIGTERM and SIGINT stop every session, which ends their workloads' supervision. -Sessions get no upstream proxy, extra CA bundle, OCSF OTLP export, or provider -workload identity: the Kubernetes driver refuses the topology when an upstream -proxy or provider SPIFFE identity is configured, so a workload never silently -loses them. +Sessions get no upstream proxy, extra CA bundle, or provider workload identity: +the Kubernetes driver refuses the topology when an upstream proxy or provider +SPIFFE identity is configured, so a workload never silently loses them. The +host also initializes no OTLP trace or OCSF exporters, because one process +cannot export each session's records with that session's collector headers and +identity. Its console logs and each session's gateway log push remain, and the +Kubernetes driver warns at startup when the gateway configures supervisor +export. diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index d4e4530640..5a7cb4bb3d 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -4,13 +4,9 @@ //! Supervisor shared by the sandboxes of one gateway (experimental). use clap::Parser; -use miette::{IntoDiagnostic, Result, WrapErr}; +use miette::Result; use std::net::SocketAddr; use std::path::PathBuf; -use std::time::Duration; - -/// Time a removed or shut-down session may take to stop its workload. -const SESSION_REMOVAL_GRACE: Duration = Duration::from_secs(20); #[derive(Debug, Parser)] #[command(name = "openshell-supervisor experimental-host")] @@ -51,6 +47,7 @@ pub fn run(raw_args: &[String]) -> Result<()> { #[cfg(unix)] fn run_platform(args: Args) -> Result<()> { + use miette::{IntoDiagnostic as _, WrapErr as _}; use std::sync::Arc; use std::sync::atomic::AtomicBool; use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _}; @@ -108,7 +105,7 @@ fn run_platform(args: Args) -> Result<()> { gateway_endpoint: args.openshell_endpoint.clone(), session_root: args.session_root.clone(), runtime_listen: args.runtime_listen, - removal_grace: SESSION_REMOVAL_GRACE, + removal_grace: openshell_sandbox_backend::session_control::SESSION_REMOVAL_GRACE, }, )?; let control = tokio::net::TcpListener::bind(args.session_control_listen) diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 8292b2a71e..f122b35c5a 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -363,10 +363,19 @@ raw gateway configuration so openshell.effectiveKubernetesConfig can use it. {{- end }} {{/* -Name of the shared supervisor Deployment, Service, and session-control Secrets. +Name of the shared supervisor Deployment, Service, and session-control Secrets: +the fullname with a fixed suffix, within the 63-character Service name limit. +A fullname without room for the suffix keeps a prefix and a hash of the whole +fullname, so the name never collapses into another release resource's name. */}} {{- define "openshell.supervisorName" -}} -{{- printf "%s-supervisor" (include "openshell.fullname" .) | trunc 52 | trimSuffix "-" -}} +{{- $suffix := "-supervisor" -}} +{{- $base := include "openshell.fullname" . -}} +{{- $room := sub 63 (len $suffix) | int -}} +{{- if gt (len $base) $room -}} +{{- $base = printf "%s-%s" ($base | trunc (sub $room 9 | int) | trimSuffix "-") ($base | sha256sum | trunc 8) -}} +{{- end -}} +{{- printf "%s%s" $base $suffix -}} {{- end }} {{/* diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index ca2c0caf01..1236a7b915 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -174,6 +174,22 @@ tests: asserts: - failedTemplate: errorMessage: supervisor_topology=gateway in operator workspace mode requires operator_namespace_label of the form key=value + - it: keeps a distinct name when the release fullname leaves no room for the suffix + template: templates/supervisor.yaml + set: + fullnameOverride: openshell-gateway-release-with-a-very-very-long-name-0057 + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - matchRegex: + path: metadata.name + pattern: ^openshell-gateway-release-with-a-very-very-[0-9a-f]{8}-supervisor$ + documentIndex: 1 + - notEqual: + path: metadata.name + value: openshell-gateway-release-with-a-very-very-long-name-0057 + documentIndex: 1 - it: omits the gateway CA when gateway TLS is disabled template: templates/supervisor.yaml set: diff --git a/e2e/rust/tests/kubernetes_shared_supervisor.rs b/e2e/rust/tests/kubernetes_shared_supervisor.rs index 59514a7365..a825350a1b 100644 --- a/e2e/rust/tests/kubernetes_shared_supervisor.rs +++ b/e2e/rust/tests/kubernetes_shared_supervisor.rs @@ -430,10 +430,36 @@ async fn install_provider(name: &str, port: u16, secret: &str) { assert!(created, "provider create {name} failed:\n{output}"); } +/// Bound on the shared supervisor answering a routed connection: its own +/// handshake deadline is 5 s. +const PROBE_DEADLINE: Duration = Duration::from_secs(10); + +/// What the shared supervisor did with a connection that named a route. +#[derive(Debug, PartialEq, Eq)] +enum RouteAnswer { + /// Closed or reset the connection before sending anything. + Refused, + /// Started the TLS handshake for the route's pinned identity; the first + /// bytes it sent. + Challenged(Vec), +} + +/// A connection closed or reset by the peer. +fn is_closed(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + ) +} + /// Port-forward to the shared supervisor's routed runtime listener. struct RuntimeListener { child: tokio::process::Child, port: u16, + stderr: Arc>>, } impl RuntimeListener { @@ -451,10 +477,18 @@ impl RuntimeListener { &format!(":{RUNTIME_PORT}"), ]) .stdout(Stdio::piped()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) .kill_on_drop(true) .spawn() .expect("spawn kubectl port-forward"); + let stderr = Arc::new(Mutex::new(Vec::new())); + let mut errors = tokio::io::BufReader::new(child.stderr.take().expect("stderr")).lines(); + let recorded = stderr.clone(); + tokio::spawn(async move { + while let Ok(Some(line)) = errors.next_line().await { + recorded.lock().unwrap().push(line); + } + }); let mut lines = tokio::io::BufReader::new(child.stdout.take().expect("stdout")).lines(); let port = tokio::time::timeout(Duration::from_secs(30), async { while let Some(line) = lines.next_line().await.expect("read port-forward") { @@ -474,12 +508,42 @@ impl RuntimeListener { // kubectl reports every forwarded connection on stdout and exits if // that pipe closes. tokio::spawn(async move { while let Ok(Some(_)) = lines.next_line().await {} }); - Self { child, port } + Self { + child, + port, + stderr, + } } - /// Name `route` in a routing preface and return the first bytes the - /// supervisor answers with before it closes the connection or goes quiet. - async fn probe(&self, route: &str, reply: &[u8]) -> Vec { + /// Every connection reached the supervisor Pod: kubectl reports a failed + /// pod-side dial on stderr, and the local side then sees the same EOF as a + /// refusal. + async fn assert_forwarded_every_connection(&mut self) { + tokio::time::sleep(Duration::from_millis(500)).await; + let failed = self.stderr.lock().unwrap().iter().any(|line| { + line.contains("an error occurred forwarding") || line.contains("error forwarding port") + }); + assert!( + !failed, + "a probe never reached the supervisor; {}", + self.diagnostics() + ); + } + + /// The forwarder's state, for failure messages. + fn diagnostics(&mut self) -> String { + format!( + "kubectl port-forward status {:?}, stderr {:?}", + self.child.try_wait(), + self.stderr.lock().unwrap() + ) + } + + /// Name `route` in a routing preface and report how the supervisor + /// answered. A challenged connection is then sent `reply` in place of the + /// pinned TLS identity and must be closed. Silence fails the test: a + /// listener that hangs has not rejected anything. + async fn probe(&mut self, route: &str, reply: &[u8]) -> RouteAnswer { let mut stream = TcpStream::connect(("127.0.0.1", self.port)) .await .expect("connect to forwarded runtime listener"); @@ -488,18 +552,33 @@ impl RuntimeListener { preface.extend_from_slice(route.as_bytes()); stream.write_all(&preface).await.expect("send preface"); let mut answer = vec![0_u8; 5]; - let read = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut answer)) - .await - .map_or(0, |read| read.unwrap_or(0)); + let read = match tokio::time::timeout(PROBE_DEADLINE, stream.read(&mut answer)).await { + Ok(Ok(read)) => read, + Ok(Err(error)) if is_closed(&error) => 0, + Ok(Err(error)) => panic!("read the answer to route {route}: {error}"), + Err(elapsed) => panic!( + "route {route} was neither refused nor challenged ({elapsed}, {PROBE_DEADLINE:?}); {}", + self.diagnostics() + ), + }; + if read == 0 { + return RouteAnswer::Refused; + } answer.truncate(read); - if read > 0 { - // Answer the supervisor's TLS handshake without its pinned identity. - let _ = stream.write_all(reply).await; - let mut rest = Vec::new(); - let _ = - tokio::time::timeout(Duration::from_secs(10), stream.read_to_end(&mut rest)).await; + if let Err(error) = stream.write_all(reply).await { + assert!(is_closed(&error), "answer route {route}: {error}"); } - answer + let mut rest = Vec::new(); + match tokio::time::timeout(PROBE_DEADLINE, stream.read_to_end(&mut rest)).await { + Ok(Ok(_)) => {} + Ok(Err(error)) if is_closed(&error) => {} + Ok(Err(error)) => panic!("wait for route {route} to close: {error}"), + Err(elapsed) => panic!( + "route {route} stayed open without its pinned TLS identity ({elapsed}, {PROBE_DEADLINE:?}); {}", + self.diagnostics() + ), + } + RouteAnswer::Challenged(answer) } } @@ -509,6 +588,8 @@ impl Drop for RuntimeListener { } } +/// A fatal TLS `handshake_failure` alert: a peer without the pinned identity. +const TLS_HANDSHAKE_FAILURE: &[u8] = b"\x15\x03\x03\x00\x02\x02\x28"; const TLS_HANDSHAKE_RECORD: u8 = 0x16; #[tokio::test] @@ -578,26 +659,35 @@ async fn one_supervisor_isolates_and_outlives_concurrent_sandbox_sessions() { // A peer naming a live route must present that session's pinned TLS // identity, and failing to does not displace the session. Stale and // unknown routes are refused before any handshake. - let listener = RuntimeListener::forward().await; + let mut listener = RuntimeListener::forward().await; for _ in 0..3 { - let answer = listener - .probe(&beta_route, b"\x15\x03\x03\x00\x02\x02\x28") - .await; - assert_eq!( - answer.first(), - Some(&TLS_HANDSHAKE_RECORD), - "a live route should demand the session's pinned TLS identity" - ); + match listener.probe(&beta_route, TLS_HANDSHAKE_FAILURE).await { + RouteAnswer::Challenged(answer) => assert_eq!( + answer.first(), + Some(&TLS_HANDSHAKE_RECORD), + "a live route should start the session's pinned TLS handshake" + ), + RouteAnswer::Refused => { + panic!("a live route should demand the session's pinned TLS identity") + } + } } - assert!( - listener.probe(&alpha_route, b"").await.is_empty(), + assert_eq!( + listener.probe(&alpha_route, &[]).await, + RouteAnswer::Refused, "stale route reached a session" ); let unknown = format!("rt-{:032x}", rand::random::()); - assert!( - listener.probe(&unknown, b"").await.is_empty(), + assert_eq!( + listener.probe(&unknown, &[]).await, + RouteAnswer::Refused, "unknown route reached a session" ); + assert!(matches!( + listener.probe(&beta_route, TLS_HANDSHAKE_FAILURE).await, + RouteAnswer::Challenged(_) + )); + listener.assert_forwarded_every_connection().await; drop(listener); assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); @@ -623,11 +713,19 @@ async fn one_supervisor_isolates_and_outlives_concurrent_sandbox_sessions() { "restarted alpha never regained supervision" ); assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); - let listener = RuntimeListener::forward().await; - assert!( - listener.probe(&alpha_route, b"").await.is_empty(), + let mut listener = RuntimeListener::forward().await; + assert_eq!( + listener.probe(&alpha_route, &[]).await, + RouteAnswer::Refused, "the replaced route reached a session" ); + assert!(matches!( + listener + .probe(&restarted_route, TLS_HANDSHAKE_FAILURE) + .await, + RouteAnswer::Challenged(_) + )); + listener.assert_forwarded_every_connection().await; drop(listener); assert_eq!( From a0eb29e93dcbb84a09766c3d2c6d8bc170c72fd3 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 09:03:28 -0700 Subject: [PATCH 12/25] refactor(supervisor): route shared supervisor crypto through openshell-crypto Main now routes first-party TLS providers and digests through the openshell-crypto facade. Build the session-control TLS configurations and the routed-runtime test server from its configuration provider, install the experimental host's default provider through it, and hash shared session registrations with its SHA-256 instead of a direct sha2 dependency. Signed-off-by: Drew Newberry --- Cargo.lock | 1 - crates/openshell-sandbox-backend/src/runtime_routed.rs | 6 +++--- crates/openshell-sandbox-backend/src/session_control.rs | 4 ++-- crates/openshell-supervisor/Cargo.toml | 1 - crates/openshell-supervisor/src/shared.rs | 2 +- crates/openshell-supervisor/src/shared_host.rs | 3 +-- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bb245b31b..3f0749059c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5123,7 +5123,6 @@ dependencies = [ "rustls", "serde", "serde_json", - "sha2 0.10.9", "socket2", "temp-env", "tempfile", diff --git a/crates/openshell-sandbox-backend/src/runtime_routed.rs b/crates/openshell-sandbox-backend/src/runtime_routed.rs index 1444f72da4..9073b89a9c 100644 --- a/crates/openshell-sandbox-backend/src/runtime_routed.rs +++ b/crates/openshell-sandbox-backend/src/runtime_routed.rs @@ -386,9 +386,9 @@ mod tests { stream .write_all(&encode_route_preface(route).unwrap()) .await?; - let mut config = rustls::ServerConfig::builder_with_provider(Arc::new( - rustls::crypto::aws_lc_rs::default_provider(), - )) + let mut config = rustls::ServerConfig::builder_with_provider( + openshell_crypto::tls::configuration_provider(), + ) .with_protocol_versions(&[&rustls::version::TLS13]) .unwrap() .with_no_client_auth() diff --git a/crates/openshell-sandbox-backend/src/session_control.rs b/crates/openshell-sandbox-backend/src/session_control.rs index cced085a8c..65e97f758f 100644 --- a/crates/openshell-sandbox-backend/src/session_control.rs +++ b/crates/openshell-sandbox-backend/src/session_control.rs @@ -231,7 +231,7 @@ pub fn server_tls_config( private_key_pem: &[u8], client_ca_pem: &[u8], ) -> Result, BackendError> { - let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let provider = openshell_crypto::tls::configuration_provider(); let verifier = rustls::server::WebPkiClientVerifier::builder_with_provider( Arc::new(root_store(client_ca_pem)?), provider.clone(), @@ -261,7 +261,7 @@ pub fn client_tls_config( private_key_pem: &[u8], server_ca_pem: &[u8], ) -> Result, BackendError> { - let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let provider = openshell_crypto::tls::configuration_provider(); let config = rustls::ClientConfig::builder_with_provider(provider) .with_protocol_versions(&[&rustls::version::TLS13]) .map_err(|error| BackendError::Descriptor(error.to_string()))? diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index f343147f87..f5ad4d3909 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -44,7 +44,6 @@ rustls = { workspace = true } rustix = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sha2 = { workspace = true } socket2 = { workspace = true } tempfile = "3" thiserror = { workspace = true } diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 5a7cb4bb3d..2291663948 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -57,7 +57,7 @@ fn run_platform(args: Args) -> Result<()> { .build() .into_diagnostic()?; runtime.block_on(async { - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + openshell_crypto::tls::ensure_default_provider(); let console_filter = tracing_subscriber::EnvFilter::try_from_default_env() .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(&args.log_level)); // Process-wide OCSF JSON output stays off; each session's own diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index 536e557be9..84a1f4b9c0 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -27,7 +27,6 @@ use openshell_sandbox_backend::session_control::{ MAX_REGISTRATION_BYTES, SESSIONS_PATH, SessionList, SessionPhaseReport, SessionRegistration, SessionStatusReport, }; -use sha2::{Digest as _, Sha256}; use tokio::sync::watch; use crate::session_host::{HostedSessionSpec, SessionHost, SessionPhase, SessionStatus}; @@ -361,7 +360,7 @@ impl SharedHost { fn registration_digest(registration: &SessionRegistration) -> Result<[u8; 32], ControlError> { let encoded = serde_json::to_vec(registration) .map_err(|error| ControlError::Invalid(format!("encode registration: {error}")))?; - Ok(Sha256::digest(&encoded).into()) + openshell_crypto::sha256(&encoded).map_err(|error| ControlError::Unavailable(error.to_string())) } fn status_report(route: &str, sandbox_id: String, status: &SessionStatus) -> SessionStatusReport { From a666a77d1569c8cf60536c52c9457189f242550e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 15:21:01 -0700 Subject: [PATCH 13/25] feat(supervisor): report shared supervisor health over TCP experimental-host takes --health-port instead of a Unix health socket, matching experimental-bootstrap. It binds the wildcard health listener after the session-control and runtime listeners and accepts and closes connections while the process runs. The chart probes the named container port health (5502) with native kubelet tcpSocket checks; session control stays on 5501, the Service exposes only runtime and session control, and /run/openshell remains for per-session files. Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 3 +- crates/openshell-supervisor/README.md | 4 ++- crates/openshell-supervisor/src/shared.rs | 31 +++++++++-------- .../helm/openshell/templates/supervisor.yaml | 15 +++++---- .../helm/openshell/tests/supervisor_test.yaml | 33 +++++++++++++++++-- 5 files changed, 60 insertions(+), 26 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 9082bf2d76..6936d9f470 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -72,7 +72,8 @@ The gateway chart creates a single-replica shared supervisor Deployment, Service, and NetworkPolicy in the release namespace, regardless of shared/managed/operator workspace mode, and the certgen hook issues its session-control Secrets. New sandbox generations run as sessions in that -process. Build the gateway, supervisor, and sandbox images from the same branch: +process. Its process-health `tcpSocket` probes use container port `health` +(5502); session control stays on 5501. Build the gateway, supervisor, and sandbox images from the same branch: all three must support the routed runtime connection. `workspace` is reserved and rejected. Omit the field to disable it. `mise run e2e:kubernetes:shared-supervisor` runs the focused e2e diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 968abf96b4..6a00e300c2 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -119,7 +119,9 @@ one process for the Kubernetes gateway supervisor topology. It is Unix-only. - **Runtime connections.** One routed listener (`--runtime-listen`) accepts every session's runtime-initiated Sandbox Protocol connections; see `openshell-sandbox-backend` for the routing preface. -- **Health.** `--health-socket` serves the existing `health --socket` check and +- **Health.** `--health-port` (5502 in the chart) binds a wildcard TCP listener + after the session-control and runtime listeners. It accepts and closes + connections for kubelet `tcpSocket` probes, keeps no filesystem state, and reports process health only. Session readiness is reported per session over session control. SIGTERM and SIGINT stop every session, which ends their workloads' supervision. diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 2291663948..a6600700f4 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -12,9 +12,10 @@ use std::path::PathBuf; #[command(name = "openshell-supervisor experimental-host")] #[command(about = "Supervise the sandboxes a gateway registers with this process")] struct Args { - /// Private socket reporting process health, not session readiness. + /// TCP port accepting connections while the process runs; reports process + /// health, not session readiness. #[arg(long)] - health_socket: PathBuf, + health_port: u16, /// Gateway gRPC endpoint; each session authenticates with its own credentials. #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] @@ -113,13 +114,10 @@ fn run_platform(args: Args) -> Result<()> { .into_diagnostic() .wrap_err("bind session-control listener")?; - // The private emptyDir survives a container restart after an unclean exit. - match std::fs::remove_file(&args.health_socket) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error).into_diagnostic(), - } - let health = tokio::net::UnixListener::bind(&args.health_socket).into_diagnostic()?; + let health = openshell_supervisor::bind_readiness_tcp(args.health_port) + .and_then(tokio::net::TcpListener::from_std) + .into_diagnostic() + .wrap_err("bind health listener")?; tracing::info!( runtime = %host.runtime_addr(), control = %args.session_control_listen, @@ -135,24 +133,25 @@ fn run_platform(args: Args) -> Result<()> { let _ = stopped.wait_for(|stop| *stop).await; }, )); - let result = loop { + loop { tokio::select! { + // A connection is the whole check; nothing is exchanged. result = health.accept() => { if let Err(error) = result { - break Err(error).into_diagnostic(); + tracing::warn!(%error, "health accept failed; retrying"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } } - _ = terminate.recv() => break Ok(()), - _ = interrupt.recv() => break Ok(()), + _ = terminate.recv() => break, + _ = interrupt.recv() => break, } - }; + } drop(health); - let _ = std::fs::remove_file(&args.health_socket); stop.send_replace(true); let _ = serve.await; // Sandboxes lose supervision with this process and fail closed. host.remove_all().await; - result + Ok(()) }) } diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index d556c6756f..ae9f6d53d0 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -57,8 +57,8 @@ spec: imagePullPolicy: {{ get (dict "always" "Always" "if_not_present" "IfNotPresent" "never" "Never") (get $kubernetes "supervisor_image_pull_policy" | default "if_not_present") | quote }} args: - experimental-host - - --health-socket - - /run/openshell/health.sock + - --health-port + - "5502" - --session-control-listen - 0.0.0.0:{{ get $shared "control_port" }} - --session-control-tls-dir @@ -87,6 +87,9 @@ spec: - name: session-control containerPort: {{ get $shared "control_port" }} protocol: TCP + - name: health + containerPort: 5502 + protocol: TCP securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -101,11 +104,11 @@ spec: # Process health only; each session reports its own readiness to # the gateway through the session-control API. readinessProbe: - exec: - command: [/openshell-supervisor, health, --socket, /run/openshell/health.sock] + tcpSocket: + port: health livenessProbe: - exec: - command: [/openshell-supervisor, health, --socket, /run/openshell/health.sock] + tcpSocket: + port: health volumeMounts: - name: run mountPath: /run/openshell diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index 1236a7b915..35b1c5b2ff 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -47,8 +47,8 @@ tests: path: spec.template.spec.containers[0].args value: - experimental-host - - --health-socket - - /run/openshell/health.sock + - --health-port + - "5502" - --session-control-listen - 0.0.0.0:5501 - --session-control-tls-dir @@ -58,6 +58,35 @@ tests: - --session-root - /run/openshell/sessions documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].ports + content: + name: health + containerPort: 5502 + protocol: TCP + documentIndex: 0 + - equal: + path: spec.template.spec.containers[0].readinessProbe + value: + tcpSocket: + port: health + documentIndex: 0 + - equal: + path: spec.template.spec.containers[0].livenessProbe + value: + tcpSocket: + port: health + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: run + mountPath: /run/openshell + documentIndex: 0 + - lengthEqual: + path: spec.ports + count: 2 + documentIndex: 1 - equal: path: spec.template.spec.automountServiceAccountToken value: false From db914ebbca10155e495d87bb2eb80c299ae6a3d0 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 15:47:50 -0700 Subject: [PATCH 14/25] refactor(supervisor): probe shared supervisor health on session control The shared supervisor's kubelet tcpSocket probes now connect to the mTLS session-control listener (5501) instead of a separate health listener. Drop the experimental-host --health-port flag, its accept loop, and the chart's 5502 container port. Probes open and close a TCP connection; every session-control request still requires mTLS. Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 5 ++-- crates/openshell-supervisor/README.md | 11 ++++--- crates/openshell-supervisor/src/shared.rs | 30 ++++--------------- .../helm/openshell/templates/supervisor.yaml | 14 ++++----- .../helm/openshell/tests/supervisor_test.yaml | 13 +++----- 5 files changed, 23 insertions(+), 50 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 6936d9f470..c8dcc40ec0 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -72,8 +72,9 @@ The gateway chart creates a single-replica shared supervisor Deployment, Service, and NetworkPolicy in the release namespace, regardless of shared/managed/operator workspace mode, and the certgen hook issues its session-control Secrets. New sandbox generations run as sessions in that -process. Its process-health `tcpSocket` probes use container port `health` -(5502); session control stays on 5501. Build the gateway, supervisor, and sandbox images from the same branch: +process. Its process-health `tcpSocket` probes connect to the mTLS +session-control port (5501); there is no separate health port. Build the +gateway, supervisor, and sandbox images from the same branch: all three must support the routed runtime connection. `workspace` is reserved and rejected. Omit the field to disable it. `mise run e2e:kubernetes:shared-supervisor` runs the focused e2e diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 6a00e300c2..d5b704e918 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -119,12 +119,11 @@ one process for the Kubernetes gateway supervisor topology. It is Unix-only. - **Runtime connections.** One routed listener (`--runtime-listen`) accepts every session's runtime-initiated Sandbox Protocol connections; see `openshell-sandbox-backend` for the routing preface. -- **Health.** `--health-port` (5502 in the chart) binds a wildcard TCP listener - after the session-control and runtime listeners. It accepts and closes - connections for kubelet `tcpSocket` probes, keeps no filesystem state, and - reports process health only. Session readiness is reported per session over - session control. SIGTERM and SIGINT stop every session, which ends their - workloads' supervision. +- **Health.** The chart's kubelet `tcpSocket` probes connect to the + session-control port (5501) and close without completing TLS, so they report + only that the process is listening; every request still requires mTLS. + Session readiness is reported per session over session control. SIGTERM and + SIGINT stop every session, which ends their workloads' supervision. Sessions get no upstream proxy, extra CA bundle, or provider workload identity: the Kubernetes driver refuses the topology when an upstream proxy or provider diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index a6600700f4..c36fa7b5bd 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -12,16 +12,12 @@ use std::path::PathBuf; #[command(name = "openshell-supervisor experimental-host")] #[command(about = "Supervise the sandboxes a gateway registers with this process")] struct Args { - /// TCP port accepting connections while the process runs; reports process - /// health, not session readiness. - #[arg(long)] - health_port: u16, - /// Gateway gRPC endpoint; each session authenticates with its own credentials. #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] openshell_endpoint: String, /// Mutually authenticated session-control listener used by the gateway. + /// Kubernetes TCP probes connect to it without completing TLS. #[arg(long)] session_control_listen: SocketAddr, @@ -82,7 +78,8 @@ fn run_platform(args: Args) -> Result<()> { ) .init(); - // Install shutdown handlers before publishing health. + // Install shutdown handlers before the control listener starts + // answering health probes. let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) .into_diagnostic()?; @@ -113,11 +110,6 @@ fn run_platform(args: Args) -> Result<()> { .await .into_diagnostic() .wrap_err("bind session-control listener")?; - - let health = openshell_supervisor::bind_readiness_tcp(args.health_port) - .and_then(tokio::net::TcpListener::from_std) - .into_diagnostic() - .wrap_err("bind health listener")?; tracing::info!( runtime = %host.runtime_addr(), control = %args.session_control_listen, @@ -133,20 +125,10 @@ fn run_platform(args: Args) -> Result<()> { let _ = stopped.wait_for(|stop| *stop).await; }, )); - loop { - tokio::select! { - // A connection is the whole check; nothing is exchanged. - result = health.accept() => { - if let Err(error) = result { - tracing::warn!(%error, "health accept failed; retrying"); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - } - _ = terminate.recv() => break, - _ = interrupt.recv() => break, - } + tokio::select! { + _ = terminate.recv() => {} + _ = interrupt.recv() => {} } - drop(health); stop.send_replace(true); let _ = serve.await; // Sandboxes lose supervision with this process and fail closed. diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index ae9f6d53d0..226f603f56 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -57,8 +57,6 @@ spec: imagePullPolicy: {{ get (dict "always" "Always" "if_not_present" "IfNotPresent" "never" "Never") (get $kubernetes "supervisor_image_pull_policy" | default "if_not_present") | quote }} args: - experimental-host - - --health-port - - "5502" - --session-control-listen - 0.0.0.0:{{ get $shared "control_port" }} - --session-control-tls-dir @@ -87,9 +85,6 @@ spec: - name: session-control containerPort: {{ get $shared "control_port" }} protocol: TCP - - name: health - containerPort: 5502 - protocol: TCP securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -101,14 +96,15 @@ spec: memory: 64Mi limits: memory: 1Gi - # Process health only; each session reports its own readiness to - # the gateway through the session-control API. + # Process health only: the probes open and close a TCP connection + # to the mTLS session-control listener. Each session reports its own + # readiness to the gateway through the session-control API. readinessProbe: tcpSocket: - port: health + port: session-control livenessProbe: tcpSocket: - port: health + port: session-control volumeMounts: - name: run mountPath: /run/openshell diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index 35b1c5b2ff..ff67ba523f 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -47,8 +47,6 @@ tests: path: spec.template.spec.containers[0].args value: - experimental-host - - --health-port - - "5502" - --session-control-listen - 0.0.0.0:5501 - --session-control-tls-dir @@ -58,24 +56,21 @@ tests: - --session-root - /run/openshell/sessions documentIndex: 0 - - contains: + - lengthEqual: path: spec.template.spec.containers[0].ports - content: - name: health - containerPort: 5502 - protocol: TCP + count: 2 documentIndex: 0 - equal: path: spec.template.spec.containers[0].readinessProbe value: tcpSocket: - port: health + port: session-control documentIndex: 0 - equal: path: spec.template.spec.containers[0].livenessProbe value: tcpSocket: - port: health + port: session-control documentIndex: 0 - contains: path: spec.template.spec.containers[0].volumeMounts From 404b460ae0bb2c1daaac382b97c2c162a1b16451 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 16:55:49 -0700 Subject: [PATCH 15/25] refactor(supervisor): use shared mode flags for the session host Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 4 +- crates/openshell-supervisor/README.md | 8 ++- crates/openshell-supervisor/src/main.rs | 50 ++++++++++++-- crates/openshell-supervisor/src/shared.rs | 68 +++++++------------ .../helm/openshell/templates/supervisor.yaml | 6 +- .../helm/openshell/tests/supervisor_test.yaml | 6 +- 6 files changed, 88 insertions(+), 54 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index c8dcc40ec0..64c64ea3ae 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -73,7 +73,9 @@ Service, and NetworkPolicy in the release namespace, regardless of shared/managed/operator workspace mode, and the certgen hook issues its session-control Secrets. New sandbox generations run as sessions in that process. Its process-health `tcpSocket` probes connect to the mTLS -session-control port (5501); there is no separate health port. Build the +session-control port (5501); there is no separate health port. The chart starts +`--mode=shared`, sets `--gateway-endpoint` to the gateway gRPC URL, and binds +`--address` to this control/probe listener. Build the gateway, supervisor, and sandbox images from the same branch: all three must support the routed runtime connection. `workspace` is reserved and rejected. Omit the field to disable it. `mise run diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index d5b704e918..ed0d11a582 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -99,11 +99,15 @@ backend or enable a Windows workload runtime. ## Shared supervisor host -`openshell-supervisor experimental-host` hosts the sessions of many sandboxes in +`openshell-supervisor --mode=shared --gateway-endpoint --address ` hosts the sessions of many sandboxes in one process for the Kubernetes gateway supervisor topology. It is Unix-only. +`--gateway-endpoint` selects the gateway each sandbox session dials. The current +admission RPCs still arrive from the gateway at `--address`; moving admission +onto a supervisor-initiated stream is a separate implementation step. + - **Admission.** A mutually authenticated TLS 1.3 listener - (`--session-control-listen`, credentials in `--session-control-tls-dir`) + (`--address`, credentials in `--session-control-tls-dir`) serves `PUT`, `GET`, and `DELETE /v1/sessions/{route}` and `GET /v1/sessions` as JSON. Only a client holding the admission certificate, which Helm gives only the gateway, can register sessions. A registration carries one diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 694f1eeeaf..f833515c82 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -24,6 +24,15 @@ const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; const HEALTH_SUBCOMMAND: &str = "health"; const SERVICE_NAME: &str = "openshell-supervisor"; +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] +enum SupervisorMode { + /// Supervise a single sandbox generation. + #[default] + Dedicated, + /// Host sandbox supervision sessions in a shared process. + Shared, +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)] enum SupervisorRole { /// Attach an Isolation Backend and supervise one sandbox generation. @@ -47,6 +56,30 @@ struct HealthArgs { #[command(about = "OpenShell policy and workload supervisor")] #[allow(clippy::struct_excessive_bools)] struct Args { + /// Supervisor deployment mode. + #[arg(long, value_enum, default_value_t)] + mode: SupervisorMode, + + /// Gateway endpoint for the shared supervisor's outbound connection. + #[arg(long, required_if_eq("mode", "shared"))] + gateway_endpoint: Option, + + /// Shared supervisor listen address, also used for TCP health probes. + #[arg(long, required_if_eq("mode", "shared"))] + address: Option, + + /// Directory with shared-host session-control TLS credentials. + #[arg(long, required_if_eq("mode", "shared"))] + session_control_tls_dir: Option, + + /// Listener for routed runtime-initiated Sandbox Protocol connections. + #[arg(long, required_if_eq("mode", "shared"))] + runtime_listen: Option, + + /// Private directory for shared-host session files. + #[arg(long, required_if_eq("mode", "shared"))] + session_root: Option, + /// Supervisor responsibility to run. #[arg(long, value_enum, default_value_t)] role: SupervisorRole, @@ -366,11 +399,20 @@ fn main() -> Result<()> { return openshell_supervisor::check_control_readiness(&args.socket); } - if raw_args.get(1).map(String::as_str) == Some("experimental-host") { - return shared::run(&raw_args[1..]); - } - let args = Args::parse(); + if args.mode == SupervisorMode::Shared { + return shared::run(args); + } + if args.gateway_endpoint.is_some() + || args.address.is_some() + || args.session_control_tls_dir.is_some() + || args.runtime_listen.is_some() + || args.session_root.is_some() + { + return Err(miette::miette!( + "shared supervisor options require --mode=shared" + )); + } validate_role_arguments(&args)?; arm_parent_liveness(args.parent_liveness_stdin)?; validate_main_exit_marker(args.main_exit_marker.as_deref())?; diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index c36fa7b5bd..3ff983b6e9 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -3,52 +3,34 @@ //! Supervisor shared by the sandboxes of one gateway (experimental). -use clap::Parser; use miette::Result; -use std::net::SocketAddr; -use std::path::PathBuf; -#[derive(Debug, Parser)] -#[command(name = "openshell-supervisor experimental-host")] -#[command(about = "Supervise the sandboxes a gateway registers with this process")] -struct Args { - /// Gateway gRPC endpoint; each session authenticates with its own credentials. - #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] - openshell_endpoint: String, - - /// Mutually authenticated session-control listener used by the gateway. - /// Kubernetes TCP probes connect to it without completing TLS. - #[arg(long)] - session_control_listen: SocketAddr, - - /// Directory with the session-control `tls.crt`, `tls.key`, and client `ca.crt`. - #[arg(long)] - session_control_tls_dir: PathBuf, - - /// Listener for routed runtime-initiated Sandbox Protocol connections. - #[arg(long)] - runtime_listen: SocketAddr, - - /// Private directory for per-session files. - #[arg(long)] - session_root: PathBuf, - - #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] - log_level: String, -} - -pub fn run(raw_args: &[String]) -> Result<()> { - let args = Args::parse_from(raw_args); +pub fn run(args: crate::Args) -> Result<()> { run_platform(args) } #[cfg(unix)] -fn run_platform(args: Args) -> Result<()> { +fn run_platform(args: crate::Args) -> Result<()> { use miette::{IntoDiagnostic as _, WrapErr as _}; use std::sync::Arc; use std::sync::atomic::AtomicBool; use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _}; + let address = args + .address + .ok_or_else(|| miette::miette!("--address is required for --mode=shared"))?; + let gateway_endpoint = args + .gateway_endpoint + .ok_or_else(|| miette::miette!("--gateway-endpoint is required for --mode=shared"))?; + let session_control_tls_dir = args.session_control_tls_dir.ok_or_else(|| { + miette::miette!("--session-control-tls-dir is required for --mode=shared") + })?; + let runtime_listen = args + .runtime_listen + .ok_or_else(|| miette::miette!("--runtime-listen is required for --mode=shared"))?; + let session_root = args + .session_root + .ok_or_else(|| miette::miette!("--session-root is required for --mode=shared"))?; let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() @@ -87,7 +69,7 @@ fn run_platform(args: Args) -> Result<()> { tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()) .into_diagnostic()?; let tls_file = |name: &str| { - let path = args.session_control_tls_dir.join(name); + let path = session_control_tls_dir.join(name); std::fs::read(&path) .into_diagnostic() .wrap_err_with(|| format!("read session-control TLS file {}", path.display())) @@ -100,19 +82,19 @@ fn run_platform(args: Args) -> Result<()> { .map_err(|error| miette::miette!("{error}"))?; let host = openshell_supervisor::shared_host::SharedHost::start( openshell_supervisor::shared_host::SharedHostConfig { - gateway_endpoint: args.openshell_endpoint.clone(), - session_root: args.session_root.clone(), - runtime_listen: args.runtime_listen, + gateway_endpoint, + session_root, + runtime_listen, removal_grace: openshell_sandbox_backend::session_control::SESSION_REMOVAL_GRACE, }, )?; - let control = tokio::net::TcpListener::bind(args.session_control_listen) + let control = tokio::net::TcpListener::bind(address) .await .into_diagnostic() .wrap_err("bind session-control listener")?; tracing::info!( runtime = %host.runtime_addr(), - control = %args.session_control_listen, + control = %address, "Shared supervisor ready for sessions" ); let (stop, stopped) = tokio::sync::watch::channel(false); @@ -138,6 +120,6 @@ fn run_platform(args: Args) -> Result<()> { } #[cfg(not(unix))] -fn run_platform(_args: Args) -> Result<()> { - Err(miette::miette!("experimental-host requires Unix")) +fn run_platform(_args: crate::Args) -> Result<()> { + Err(miette::miette!("--mode=shared requires Unix")) } diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index 226f603f56..eb31b93f55 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -56,8 +56,10 @@ spec: image: {{ get $kubernetes "supervisor_image" | quote }} imagePullPolicy: {{ get (dict "always" "Always" "if_not_present" "IfNotPresent" "never" "Never") (get $kubernetes "supervisor_image_pull_policy" | default "if_not_present") | quote }} args: - - experimental-host - - --session-control-listen + - --mode=shared + - --gateway-endpoint + - {{ $endpoint | quote }} + - --address - 0.0.0.0:{{ get $shared "control_port" }} - --session-control-tls-dir - /etc/openshell/session-control diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index ff67ba523f..e85713e2e1 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -46,8 +46,10 @@ tests: - equal: path: spec.template.spec.containers[0].args value: - - experimental-host - - --session-control-listen + - --mode=shared + - --gateway-endpoint + - https://openshell.gateway.svc.cluster.local:8080 + - --address - 0.0.0.0:5501 - --session-control-tls-dir - /etc/openshell/session-control From da1a5fdde5550cc58773ce0f55531d8127322d15 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 17:06:31 -0700 Subject: [PATCH 16/25] refactor(supervisor): describe the shared host module by its mode Drop the experimental label from the shared host module documentation so it matches the --mode=shared entry point. No behavior changes. Signed-off-by: Drew Newberry --- crates/openshell-supervisor/src/shared.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 3ff983b6e9..12ad4f0a9f 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Supervisor shared by the sandboxes of one gateway (experimental). +//! Supervisor host shared by the sandboxes of one gateway. use miette::Result; From cdedef6c21eef4db578122cc8c4fd4a727c5704e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 19:21:16 -0700 Subject: [PATCH 17/25] fix(test): preserve shared supervisor workspace overlays Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 11 +++++++++-- tasks/test.toml | 5 +++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 64c64ea3ae..9055dab691 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -80,8 +80,15 @@ gateway, supervisor, and sandbox images from the same branch: all three must support the routed runtime connection. `workspace` is reserved and rejected. Omit the field to disable it. `mise run e2e:kubernetes:shared-supervisor` runs the focused e2e -(`ci/values-supervisor.yaml`); layer a workspace-mode overlay -through `OPENSHELL_E2E_KUBE_EXTRA_VALUES` to cover managed namespaces. +(`ci/values-supervisor.yaml`). The task prepends that overlay to any +colon-separated `OPENSHELL_E2E_KUBE_EXTRA_VALUES` list, so layer a +workspace-mode overlay to cover managed namespaces: + +```bash +OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-workspace-managed.yaml \ +OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET=e2e-regcred \ + mise run e2e:kubernetes:shared-supervisor +``` ### 2. Deploy OpenShell diff --git a/tasks/test.toml b/tasks/test.toml index f9611b5430..8eea7c39be 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -268,8 +268,9 @@ run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:shared-supervisor"] description = "Run Kubernetes e2e with one supervisor shared by the gateway's sandboxes (gateway supervisor topology)" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-supervisor.yaml", OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-shared-supervisor" } -run = "e2e/rust/e2e-kubernetes.sh" +env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-shared-supervisor" } +# Prepend the supervisor overlay to any caller overlays, such as a workspace mode. +run = 'OPENSHELL_E2E_KUBE_EXTRA_VALUES="deploy/helm/openshell/ci/values-supervisor.yaml${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh' ["e2e:kubernetes:workspace-operator"] description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" From ccfdfbbb66726b0f9c8f3f053f93040a85319043 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 19:59:01 -0700 Subject: [PATCH 18/25] refactor(supervisor): name hosted lifetimes sandbox instances A sandbox instance is one running generation of a stable Sandbox resource. Gateway transport connections and the gateway supervisor sessions they carry can reconnect within one instance, so the hosted object, its configuration, controls, host, and per-instance network, denial, and token-grant state now use the sandbox instance name. Rename session_host to sandbox_instance_host, run_session to run_sandbox_instance, and SessionConfig, SessionControls, SessionHost, and related hosted types to their SandboxInstance equivalents. Gateway supervisor sessions, SSH sessions, and authentication session types keep their names. No wire, configuration, or lifecycle behavior changes. Signed-off-by: Drew Newberry --- crates/openshell-core/src/grpc_client.rs | 53 +- crates/openshell-ocsf/src/builders/mod.rs | 7 +- crates/openshell-ocsf/src/ctx.rs | 15 +- .../src/tracing_layers/jsonl_layer.rs | 15 +- crates/openshell-otel/src/propagation.rs | 4 +- crates/openshell-supervisor-network/README.md | 20 +- .../src/l7/token_grant_injection.rs | 6 +- .../src/policy_local.rs | 24 +- .../src/proxy/relay.rs | 2 +- .../openshell-supervisor-network/src/run.rs | 18 +- .../src/token_grant.rs | 16 +- .../src/log_push.rs | 9 +- crates/openshell-supervisor/README.md | 62 +-- crates/openshell-supervisor/src/lib.rs | 76 +-- ...ssion_host.rs => sandbox_instance_host.rs} | 462 ++++++++++-------- 15 files changed, 422 insertions(+), 367 deletions(-) rename crates/openshell-supervisor/src/{session_host.rs => sandbox_instance_host.rs} (65%) diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 80b20b0264..75672d0263 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -5,7 +5,7 @@ //! `OpenShell` server. //! //! Every request carries a sandbox bearer credential in the `Authorization` -//! header. Each supervisor session owns its [`GatewayCredentials`], usually +//! header. Each sandbox instance owns its [`GatewayCredentials`], usually //! built from the trusted launch bundle with //! [`GatewayCredentials::from_auth_bundle`]. Diagnostic tools can instead //! resolve a token with [`GatewayCredentials::from_environment`] from one of: @@ -16,8 +16,8 @@ //! caller exchanges it for a gateway JWT via `IssueSandboxToken`. //! //! Callers reach the gateway through a [`GatewayEndpoint`], which pairs the -//! URL with one session's credentials. No credential is installed -//! process-wide, so sessions hosted by one supervisor process cannot use or +//! URL with one sandbox instance's credentials. No credential is installed +//! process-wide, so instances hosted by one supervisor process cannot use or //! rotate each other's identity. use std::collections::HashMap; @@ -65,7 +65,7 @@ pub const MAX_SUPERVISOR_STREAM_MESSAGE_BYTES: usize = 9 * 1024 * 1024; /// Shared, refreshable Bearer header. All [`AuthInterceptor`] clones read /// the same slot, so the renewal task can replace the token in place without -/// rebuilding the channel. `None` means the owning session revoked it. +/// rebuilding the channel. `None` means the owning sandbox instance revoked it. type TokenSlot = Arc>>; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -75,7 +75,7 @@ enum TokenSource { K8sServiceAccount, } -/// Gateway credentials owned by one supervisor session. +/// Gateway credentials owned by one sandbox instance. /// /// Clients built from a [`GatewayEndpoint`] read only these slots, and the /// renewal task started by the first connection updates only them. Dropping @@ -200,7 +200,7 @@ impl GatewayCredentials { } /// Start the renewal task once for these credentials, on the caller's - /// runtime, so a hosted session's renewal stops with that session. + /// runtime, so a hosted sandbox instance's renewal stops with that instance. fn start_renewal(&self, channel: &AuthedChannel, plain_channel: &Channel, endpoint: &str) { let mut renewal = self .inner @@ -239,14 +239,16 @@ impl GatewayCredentials { .bearer .write() .unwrap_or_else(std::sync::PoisonError::into_inner); - // A revoked session must not be revived by an in-flight response. + // Revoked instance credentials must not be revived by an in-flight + // response. if slot.is_some() { *slot = Some(bearer); } } } -/// The gateway URL a session uses, paired with that session's credentials. +/// The gateway URL a sandbox instance uses, paired with that instance's +/// credentials. #[derive(Clone)] pub struct GatewayEndpoint { url: Arc, @@ -271,7 +273,7 @@ impl GatewayEndpoint { &self.credentials } - /// Reach another gateway address with the same session credentials. + /// Reach another gateway address with the same sandbox instance credentials. #[must_use] pub fn with_url(&self, url: impl Into>) -> Self { Self::new(url, self.credentials.clone()) @@ -357,7 +359,9 @@ impl tonic::service::Interceptor for AuthInterceptor { .read() .unwrap_or_else(std::sync::PoisonError::into_inner) .clone() - .ok_or_else(|| Status::unauthenticated("gateway session credentials were revoked"))?; + .ok_or_else(|| { + Status::unauthenticated("sandbox instance gateway credentials were revoked") + })?; req.metadata_mut().insert("authorization", bearer); #[cfg(feature = "trace-context")] let req = @@ -458,11 +462,14 @@ impl Drop for ClientSpanStatus { } } -/// Build a Bearer-authenticated channel to the gateway with one session's -/// credentials. The first connection starts that session's renewal task. +/// Build a Bearer-authenticated channel to the gateway with one sandbox +/// instance's credentials. The first connection starts that instance's +/// renewal task. async fn connect_channel(gateway: &GatewayEndpoint) -> Result { if gateway.credentials.is_revoked() { - return Err(miette::miette!("gateway session credentials were revoked")); + return Err(miette::miette!( + "sandbox instance gateway credentials were revoked" + )); } let channel = build_plain_channel(gateway.url()).await?; let intercepted = InterceptedService::new( @@ -577,8 +584,9 @@ pub async fn report_provider_readiness( .into_diagnostic() } -/// Slots and transport retained by one session's renewal task. It holds no -/// reference to [`GatewayCredentials`], so dropping them still stops it. +/// Slots and transport retained by one sandbox instance's renewal task. It +/// holds no reference to [`GatewayCredentials`], so dropping them still stops +/// it. struct RenewalTarget { channel: AuthedChannel, bearer: TokenSlot, @@ -590,10 +598,10 @@ struct RenewalTarget { } /// Background task that renews the sandbox JWT at ~80% of its remaining -/// lifetime. The new token replaces the value in the session's slot, so all -/// in-flight and future clients of that session pick it up on their next -/// request. The loop never panics: every failure is logged and re-attempted -/// after a bounded backoff. +/// lifetime. The new token replaces the value in the sandbox instance's slot, +/// so all in-flight and future clients of that instance pick it up on their +/// next request. The loop never panics: every failure is logged and +/// re-attempted after a bounded backoff. async fn refresh_token_loop(target: RenewalTarget) { let RenewalTarget { channel, @@ -832,7 +840,7 @@ fn compute_refresh_delay(slot: &TokenSlot) -> Option { let current = slot .read() .unwrap_or_else(std::sync::PoisonError::into_inner); - // A revoked session has nothing to renew. + // Revoked instance credentials have nothing to renew. current.as_ref()?.to_str().ok().map(str::to_string) } .unwrap_or_default(); @@ -1058,7 +1066,7 @@ mod workspace_tests { } } -/// Connect to the `OpenShell` server with one session's credentials. +/// Connect to the `OpenShell` server with one sandbox instance's credentials. async fn connect(gateway: &GatewayEndpoint) -> Result> { let channel = connect_channel(gateway).await?; Ok(OpenShellClient::new(channel)) @@ -1711,7 +1719,8 @@ mod token_grant_exchange_tests { #[derive(Clone)] pub struct CachedOpenShellClient { client: OpenShellClient, - /// Session credentials rotated alongside extension credentials. + /// Sandbox instance gateway credentials rotated alongside extension + /// credentials. credentials: GatewayCredentials, workspace: Arc>, /// Extension credentials for this supervisor. Cloning the client shares diff --git a/crates/openshell-ocsf/src/builders/mod.rs b/crates/openshell-ocsf/src/builders/mod.rs index 5c2219cc6d..7026979860 100644 --- a/crates/openshell-ocsf/src/builders/mod.rs +++ b/crates/openshell-ocsf/src/builders/mod.rs @@ -200,10 +200,11 @@ pub struct EventContext { pub origin: EventOrigin, } -/// Event context borrowed from the caller or shared by a hosted session. +/// Event context borrowed from the caller or shared by a hosted sandbox +/// instance. /// -/// Builders accept either form, so events attributed to a hosted supervisor -/// session need not leak a `'static` context for each session. +/// Builders accept either form, so events attributed to a hosted sandbox +/// instance need not leak a `'static` context for each instance. #[derive(Debug, Clone)] pub enum ContextRef<'a> { Borrowed(&'a EventContext), diff --git a/crates/openshell-ocsf/src/ctx.rs b/crates/openshell-ocsf/src/ctx.rs index 9dd542a9aa..726d4ad0a9 100644 --- a/crates/openshell-ocsf/src/ctx.rs +++ b/crates/openshell-ocsf/src/ctx.rs @@ -1,11 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Process-wide and session-bound [`EventContext`]. +//! Process-wide and sandbox-instance-bound [`EventContext`]. //! //! A dedicated supervisor initialises the process-wide context once via -//! [`set_ctx`]. A process hosting several supervisor sessions instead binds -//! each session's context to the threads of that session's runtime with +//! [`set_ctx`]. A process hosting several sandbox instances instead binds +//! each instance's context to the threads of that instance's runtime with //! [`bind_thread_ctx`]; a bound context takes precedence. [`ctx`] falls back //! to a default context when neither is set (e.g. unit tests that exercise //! builders without booting the sandbox). @@ -28,7 +28,7 @@ static OCSF_CTX_FALLBACK: LazyLock = LazyLock::new(|| EventContext }); std::thread_local! { - /// Context of the hosted session whose runtime owns this thread. + /// Context of the hosted sandbox instance whose runtime owns this thread. static THREAD_CTX: RefCell>> = const { RefCell::new(None) }; } @@ -40,16 +40,17 @@ pub fn set_ctx(ctx: EventContext) -> bool { OCSF_CTX.set(ctx).is_ok() } -/// Attribute events emitted on the calling thread to one hosted session. +/// Attribute events emitted on the calling thread to one hosted sandbox +/// instance. /// -/// Session runtimes bind every thread they start; `None` unbinds it. +/// Instance runtimes bind every thread they start; `None` unbinds it. pub fn bind_thread_ctx(ctx: Option>) { THREAD_CTX.with(|current| *current.borrow_mut() = ctx); } /// Return the context for events emitted on the calling thread. /// -/// A session-bound context wins over the process-wide one. Falls back to a +/// An instance-bound context wins over the process-wide one. Falls back to a /// default context if neither is set. #[must_use] pub fn ctx() -> ContextRef<'static> { diff --git a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs index 3f0936e841..40484ce43b 100644 --- a/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs +++ b/crates/openshell-ocsf/src/tracing_layers/jsonl_layer.rs @@ -16,10 +16,10 @@ use tracing_subscriber::layer::Context; use crate::format::downgrade::downgrade_event; use crate::tracing_layers::event_bridge::{OCSF_TARGET, clone_current_event}; -/// OCSF JSON output settings owned by one hosted supervisor session. +/// OCSF JSON output settings owned by one hosted sandbox instance. /// -/// Sessions apply their own `ocsf_json_enabled` and schema-version settings -/// to these values. Binding them to the session's runtime threads makes every +/// Instances apply their own `ocsf_json_enabled` and schema-version settings +/// to these values. Binding them to the instance's runtime threads makes every /// [`OcsfJsonlLayer`] use them for events those threads emit. #[derive(Clone, Debug, Default)] pub struct OcsfJsonSettings { @@ -32,7 +32,8 @@ std::thread_local! { const { std::cell::RefCell::new(None) }; } -/// Apply one hosted session's JSON settings to events emitted on this thread. +/// Apply one hosted sandbox instance's JSON settings to events emitted on this +/// thread. pub fn bind_thread_json_settings(settings: Option) { THREAD_JSON_SETTINGS.with(|current| *current.borrow_mut() = settings); } @@ -111,9 +112,9 @@ where return; } - // A hosted session's settings replace the process-wide ones. - let session = THREAD_JSON_SETTINGS.with(|current| current.borrow().clone()); - let (enabled, target_version) = session.as_ref().map_or( + // A hosted sandbox instance's settings replace the process-wide ones. + let instance = THREAD_JSON_SETTINGS.with(|current| current.borrow().clone()); + let (enabled, target_version) = instance.as_ref().map_or( (self.enabled.as_ref(), self.target_version.as_ref()), |settings| (Some(&settings.enabled), Some(&settings.target_version)), ); diff --git a/crates/openshell-otel/src/propagation.rs b/crates/openshell-otel/src/propagation.rs index 82e5fa784d..02f2bdb120 100644 --- a/crates/openshell-otel/src/propagation.rs +++ b/crates/openshell-otel/src/propagation.rs @@ -124,8 +124,8 @@ pub fn set_parent_from_environment(span: &tracing::Span) { /// Parent `span` under an explicit W3C `traceparent` and `tracestate`. /// /// A process supervising several sandboxes receives each launch's context -/// with its session instead of through process environment. Leaves `span` -/// unchanged when the values carry no valid context. +/// with its sandbox instance instead of through process environment. Leaves +/// `span` unchanged when the values carry no valid context. pub fn set_parent_from_trace_context( span: &tracing::Span, traceparent: Option<&str>, diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index b191655d1a..7ce403a15e 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -15,14 +15,14 @@ default features, so this instrumentation does not require the compute runtime or Unix-only sandbox library. It is a development dependency only. -## Session inputs +## Sandbox instance inputs -`run_networking` takes a `NetworkSession` with the sandbox identity, the -session's `GatewayEndpoint`, the SPIFFE Workload API socket for provider token -grants, the `policy.local` denial source, and proxy CA files. None of these -are read from the process environment, so one process can run several -sessions. Each session owns its provider token cache and HTTP client; a token -acquired for one sandbox is never served to another. Without a Workload API -socket, token grants fail closed. A process hosting several sessions serves -`/v1/denials` from a per-session `SessionDenialLog` instead of the shared -shorthand log files. +`run_networking` takes a `SandboxInstanceNetwork` with the sandbox identity, +the sandbox instance's `GatewayEndpoint`, the SPIFFE Workload API socket for +provider token grants, the `policy.local` denial source, and proxy CA files. +None of these are read from the process environment, so one process can run +several sandbox instances. Each instance owns its provider token cache and +HTTP client; a token acquired for one sandbox is never served to another. +Without a Workload API socket, token grants fail closed. A process hosting +several instances serves `/v1/denials` from a per-instance +`SandboxInstanceDenialLog` instead of the shared shorthand log files. diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 456cf1ab51..65d7b70d08 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -53,14 +53,14 @@ pub trait TokenGrantResolver: Send + Sync { ) -> Pin> + Send + 'a>>; } -/// Resolves provider tokens through one session's authenticated grant flow. +/// Resolves provider tokens through one sandbox instance's authenticated grant flow. pub struct SpiffeTokenGrantResolver { - grants: crate::token_grant::SessionTokenGrants, + grants: crate::token_grant::SandboxInstanceTokenGrants, } impl SpiffeTokenGrantResolver { #[must_use] - pub fn new(grants: crate::token_grant::SessionTokenGrants) -> Self { + pub fn new(grants: crate::token_grant::SandboxInstanceTokenGrants) -> Self { Self { grants } } } diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index a581800c15..033f492aa7 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -84,8 +84,8 @@ const SHORTHAND_LOG_PREFIX: &str = "openshell."; /// log entry (very long URL path, etc.) cannot blow up the response. const MAX_DENIAL_LINE_BYTES: usize = 4096; -/// Most recent denial lines retained for one hosted session. -const MAX_SESSION_DENIAL_LINES: usize = 256; +/// Most recent denial lines retained for one hosted sandbox instance. +const MAX_INSTANCE_DENIAL_LINES: usize = 256; /// Where `/v1/denials` reads this sandbox's recent denials. #[derive(Clone, Debug)] @@ -93,9 +93,9 @@ pub enum DenialLogSource { /// Shorthand log files written by a supervisor process that serves one /// sandbox. ShorthandFiles(PathBuf), - /// Denials recorded for this session alone. A process hosting several + /// Denials recorded for this sandbox instance alone. A process hosting several /// sandboxes cannot expose its shared log files to any one of them. - Session(SessionDenialLog), + SandboxInstance(SandboxInstanceDenialLog), } impl Default for DenialLogSource { @@ -104,13 +104,13 @@ impl Default for DenialLogSource { } } -/// Bounded recent OCSF denial lines recorded for one hosted session. +/// Bounded recent OCSF denial lines recorded for one hosted sandbox instance. #[derive(Clone, Default)] -pub struct SessionDenialLog { +pub struct SandboxInstanceDenialLog { lines: Arc>>, } -impl std::fmt::Debug for SessionDenialLog { +impl std::fmt::Debug for SandboxInstanceDenialLog { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let retained = self .lines @@ -118,13 +118,13 @@ impl std::fmt::Debug for SessionDenialLog { .unwrap_or_else(std::sync::PoisonError::into_inner) .len(); formatter - .debug_struct("SessionDenialLog") + .debug_struct("SandboxInstanceDenialLog") .field("retained", &retained) .finish() } } -impl SessionDenialLog { +impl SandboxInstanceDenialLog { /// Retain `line` when it is a rendered OCSF denial; ignore anything else. pub fn record(&self, line: &str) { let line = line.trim_end(); @@ -135,7 +135,7 @@ impl SessionDenialLog { .lines .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - if lines.len() == MAX_SESSION_DENIAL_LINES { + if lines.len() == MAX_INSTANCE_DENIAL_LINES { lines.pop_front(); } lines.push_back(line.to_string()); @@ -227,7 +227,7 @@ impl PolicyLocalContext { self } - /// The session's provider token grant resolver, absent when grants are + /// The sandbox instance's provider token grant resolver, absent when grants are /// unavailable and must fail closed. pub(crate) fn token_grant_resolver(&self) -> Option> { self.token_grants.clone() @@ -425,7 +425,7 @@ async fn recent_denials_response( let limit = parse_last_query(query).unwrap_or(DEFAULT_DENIALS_LIMIT); let log_dir = match &ctx.denials { DenialLogSource::ShorthandFiles(log_dir) => log_dir.clone(), - DenialLogSource::Session(log) => { + DenialLogSource::SandboxInstance(log) => { return ( 200, serde_json::json!({ diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index 05f685e122..3e722e3b50 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -58,7 +58,7 @@ pub(super) fn http_context( .is_none() .then_some(secret_resolver) .flatten(); - // Grants use the owning session's resolver; without one they fail closed. + // Grants use the owning sandbox instance's resolver; without one they fail closed. let token_grant_resolver = provider_credentials .as_ref() .and(policy_local) diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 34f6e116d2..340b3984c5 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -39,7 +39,7 @@ use crate::l7::token_grant_injection::SpiffeTokenGrantResolver; use crate::opa::OpaEngine; use crate::policy_local::{DenialLogSource, PolicyLocalContext}; use crate::proxy::ProxyHandle; -use crate::token_grant::SessionTokenGrants; +use crate::token_grant::SandboxInstanceTokenGrants; use openshell_core::endpoint_status::EndpointObservationSender; use openshell_core::grpc_client::GatewayEndpoint; @@ -194,14 +194,14 @@ fn append_additional_ca_bundle( /// Identity and authority of the one sandbox a [`run_networking`] call serves. /// -/// A process hosting several sandboxes builds one per session; no value is -/// read from the process environment, so sessions cannot share gateway +/// A process hosting several sandboxes builds one per sandbox instance; no value +/// is read from the process environment, so instances cannot share gateway /// credentials, provider token grants, denial history, or proxy CA files. #[derive(Debug, Default)] -pub struct NetworkSession { +pub struct SandboxInstanceNetwork { pub sandbox_id: Option, pub sandbox_name: Option, - /// Gateway access with this session's credentials. + /// Gateway access with this instance's credentials. pub gateway: Option, /// SPIFFE Workload API socket for provider token grants. Without one, /// grants fail closed. @@ -284,7 +284,7 @@ pub async fn run_networking( entrypoint_pid: Arc, process_enabled: bool, provider_credentials: &ProviderCredentialState, - session: NetworkSession, + instance: SandboxInstanceNetwork, denial_tx: Option>, activity_tx: Option, endpoint_observation_tx: Option, @@ -300,15 +300,15 @@ pub async fn run_networking( // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so // it can publish updated policy snapshots after a successful reload. - let NetworkSession { + let SandboxInstanceNetwork { sandbox_id, sandbox_name, gateway, provider_workload_api_socket, denial_log, proxy_tls, - } = session; - let token_grants = SpiffeTokenGrantResolver::new(SessionTokenGrants::new( + } = instance; + let token_grants = SpiffeTokenGrantResolver::new(SandboxInstanceTokenGrants::new( gateway.clone(), sandbox_id.clone(), provider_workload_api_socket, diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index 05331e128d..83d36d36b3 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -27,9 +27,9 @@ //! - `scopes` — `OAuth2` scopes to request (optional) //! - `cache_ttl` — Optional exact cache override; zero disables caching //! -//! ## Session scope +//! ## Sandbox instance scope //! -//! Each supervised sandbox owns a [`SessionTokenGrants`]: its token cache, +//! Each sandbox instance owns a [`SandboxInstanceTokenGrants`]: its token cache, //! HTTP client, gateway credentials, sandbox ID, and SPIFFE Workload API //! socket. A token is never served to another sandbox. Without a configured //! Workload API socket, grants fail closed with a configuration failure. @@ -103,9 +103,9 @@ impl TokenCache { /// Provider token grants owned by one supervised sandbox. /// /// The cache, HTTP connection pool, gateway credentials, sandbox ID, and -/// workload identity socket all belong to this session, so a token acquired +/// workload identity socket all belong to this sandbox instance, so a token acquired /// for one sandbox is never served to another. -pub struct SessionTokenGrants { +pub struct SandboxInstanceTokenGrants { cache: TokenCache, // Initialization failures stay bounded and typed; acquisition must not panic. http_client: Result, @@ -114,7 +114,7 @@ pub struct SessionTokenGrants { workload_api_socket: Option, } -impl SessionTokenGrants { +impl SandboxInstanceTokenGrants { /// Grants authenticated as `sandbox_id` through `gateway`. A missing /// Workload API socket makes every grant fail closed. #[must_use] @@ -134,12 +134,12 @@ impl SessionTokenGrants { /// Obtain an `OAuth2` access token for a provider using JWT client assertion grant. /// - /// This fetches the sandbox's SPIFFE JWT-SVID from the session's Workload + /// This fetches the sandbox's SPIFFE JWT-SVID from the instance's Workload /// API socket, then exchanges it for an access token with a POST request to /// the provider's token endpoint with the JWT client assertion grant flow /// (RFC 7523). /// - /// Tokens are cached per provider name with TTL in this session's cache. + /// Tokens are cached per provider name with TTL in this instance's cache. /// Subsequent calls return the cached token if it has not expired. /// /// # Errors @@ -247,7 +247,7 @@ impl SessionTokenGrants { } } -/// One provider token request; see [`SessionTokenGrants::obtain_provider_token`]. +/// One provider token request; see [`SandboxInstanceTokenGrants::obtain_provider_token`]. pub struct ObtainProviderTokenRequest<'a> { pub provider_name: &'a str, pub token_endpoint: &'a str, diff --git a/crates/openshell-supervisor-process/src/log_push.rs b/crates/openshell-supervisor-process/src/log_push.rs index 930ef87d5d..c2ee97760e 100644 --- a/crates/openshell-supervisor-process/src/log_push.rs +++ b/crates/openshell-supervisor-process/src/log_push.rs @@ -35,13 +35,14 @@ impl LogSink { } std::thread_local! { - /// Sink of the hosted session whose runtime owns this thread. + /// Sink of the hosted sandbox instance whose runtime owns this thread. static THREAD_SINK: RefCell> = const { RefCell::new(None) }; } -/// Route events emitted on the calling thread to a hosted session's sink. +/// Route events emitted on the calling thread to a hosted sandbox instance's +/// sink. /// -/// Hosted sessions bind every thread of their runtime. Unbound threads push +/// Hosted instances bind every thread of their runtime. Unbound threads push /// nothing, so an event cannot reach another sandbox's log stream. pub fn bind_thread_sink(sink: Option) { THREAD_SINK.with(|current| *current.borrow_mut() = sink); @@ -51,7 +52,7 @@ pub fn bind_thread_sink(sink: Option) { enum Route { /// The process supervises one sandbox; every event belongs to it. Fixed(LogSink), - /// Events belong to the session bound to the emitting thread, if any. + /// Events belong to the sandbox instance bound to the emitting thread, if any. ThreadBound, } diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 12c08cebb0..d603af388f 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -22,41 +22,47 @@ only after boundary attachment and confirmation; the supervisor owns policy evaluation, live credentials, and listener lifetime. These settings do not add a proxy hook to the isolation interface or the shared Sandbox Protocol descriptor. -## Session ownership - -`run_session` supervises one sandbox generation from a `SessionConfig` and -`SessionControls`. Everything that belongs to that generation is an input or -is created by the session: gateway credentials and their renewal, the Sandbox -Protocol credential, policy, provider credentials and token grants, middleware -credentials, OCSF JSON settings, proxy trust files, the denial history served -by `policy.local`, the backend connection, access plane, readiness, and stop -signal. Gateway clients receive a `GatewayEndpoint`, which pairs the URL with -that session's `GatewayCredentials`; nothing is installed process-wide. +## Sandbox instance ownership + +A sandbox instance is one running generation of a stable Sandbox resource. +Gateway connections, and the gateway supervisor sessions carried over them, +can reconnect within an instance without creating another one; the generation +remains the fencing identifier. + +`run_sandbox_instance` supervises one sandbox instance from a +`SandboxInstanceConfig` and `SandboxInstanceControls`. Everything that belongs +to that instance is an input or is created by the instance: gateway +credentials and their renewal, the Sandbox Protocol credential, policy, +provider credentials and token grants, middleware credentials, OCSF JSON +settings, proxy trust files, the denial history served by `policy.local`, the +backend connection, access plane, readiness, and stop signal. Gateway clients +receive a `GatewayEndpoint`, which pairs the URL with that instance's +`GatewayCredentials`; nothing is installed process-wide. `run_sandbox_with_backend` is the dedicated wrapper. It keeps the existing process contract: it sets the process-wide OCSF context, reads launch inputs -from the environment, binds the readiness socket and port to session -readiness, and stops the session on SIGTERM or SIGINT. The binary shares one +from the environment, binds the readiness socket and port to instance +readiness, and stops the instance on SIGTERM or SIGINT. The binary shares one `GatewayCredentials` between log push and supervision, so renewal by either keeps both current. -`session_host::SessionHost` runs several sessions in one process. Each session -gets its own Tokio runtime; every thread of that runtime, and the thread -driving it, is bound to the session's OCSF context, OCSF JSON settings, log -sink, and denial log. Spawned tasks, blocking work, callbacks, reconnects, and -renewal therefore inherit the session's identity, and threads outside a -session carry none and push no logs. The OCSF context also supplies the -sandbox identity in middleware requests. - -Removing a session stops its workload, shuts down only its runtime, and -revokes its gateway credentials; the key stays registered until the session's -thread has finished. A session that panics reports a failure and is revoked -the same way. A session that is still starting is cancelled at once; the -sandbox runtime then loses supervision and fails closed, as it does when a -dedicated supervisor exits. A hosted session without an explicit proxy trust -directory receives a private one for its lifetime. Hosted processes must +`sandbox_instance_host::SandboxInstanceHost` runs several sandbox instances in +one process. Each instance gets its own Tokio runtime; every thread of that +runtime, and the thread driving it, is bound to the instance's OCSF context, +OCSF JSON settings, log sink, and denial log. Spawned tasks, blocking work, +callbacks, reconnects, and renewal therefore inherit the instance's identity, +and threads outside an instance carry none and push no logs. The OCSF context +also supplies the sandbox identity in middleware requests. + +Removing an instance stops its workload, shuts down only its runtime, and +revokes its gateway credentials; the key stays registered until the +instance's thread has finished. An instance that panics reports a failure and +is revoked the same way. An instance that is still starting is cancelled at +once; the sandbox runtime then loses supervision and fails closed, as it does +when a dedicated supervisor exits. A hosted instance without an explicit proxy +trust directory receives a private one for its lifetime. Hosted processes must install `LogPushLayer::thread_bound` and an `OcsfShorthandLayer` writing to -`SessionDenialWriter`. +`SandboxInstanceDenialWriter`. ## Console logging diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index bd402b5d38..69ee814a34 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -24,7 +24,7 @@ mod denial_aggregator; mod endpoint_status; mod mechanistic_mapper; mod provider_readiness; -pub mod session_host; +pub mod sandbox_instance_host; use config_runtime::{agent_proposals_enabled_from_settings, run_policy_poll_loop}; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -612,7 +612,7 @@ pub async fn run_network_proxy( Arc::new(AtomicU32::new(0)), false, &provider_credentials, - openshell_supervisor_network::run::NetworkSession { + openshell_supervisor_network::run::SandboxInstanceNetwork { sandbox_name: Some("network-proxy".to_string()), proxy_tls, ..Default::default() @@ -754,11 +754,11 @@ pub struct SandboxRunConfig { pub main_exit_marker: Option, } -/// Supervise the one sandbox generation this process was launched for. +/// Supervise the one sandbox instance this process was launched for. /// /// The process-wide OCSF context, SIGTERM/SIGINT handling, readiness /// endpoints, and the environment-provided launch inputs belong to the -/// dedicated process; [`run_session`] owns everything else. +/// dedicated process; [`run_sandbox_instance`] owns everything else. /// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. @@ -796,7 +796,7 @@ pub async fn run_sandbox_with_backend( // during policy loading (filesystem config, validation) have a context. // Proxy IP/port use defaults here; the boundary mediation source carries // workload-side connection metadata. - if !openshell_ocsf::ctx::set_ctx(session_event_context( + if !openshell_ocsf::ctx::set_ctx(sandbox_instance_event_context( sandbox_id.as_deref(), sandbox.as_deref(), std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), @@ -804,16 +804,16 @@ pub async fn run_sandbox_with_backend( debug!("OCSF context already initialized, keeping existing"); } - let (readiness, session_readiness) = tokio::sync::watch::channel(false); + let (readiness, instance_readiness) = tokio::sync::watch::channel(false); let control_readiness = health_socket_path .map(ReadinessEndpoint::Unix) .into_iter() .chain(health_port.map(ReadinessEndpoint::Tcp)) - .map(|endpoint| ControlReadiness::start(endpoint, Some(session_readiness.clone()))) + .map(|endpoint| ControlReadiness::start(endpoint, Some(instance_readiness.clone()))) .collect::>>()?; - let result = Box::pin(run_session( + let result = Box::pin(run_sandbox_instance( backend_setup, - SessionConfig { + SandboxInstanceConfig { command, workdir, timeout_secs, @@ -846,7 +846,7 @@ pub async fn run_sandbox_with_backend( proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), }, - SessionControls { + SandboxInstanceControls { shutdown: Box::pin(wait_for_control_shutdown_signal()), readiness, }, @@ -858,7 +858,7 @@ pub async fn run_sandbox_with_backend( /// OCSF identity of one supervised sandbox. #[must_use] -pub fn session_event_context( +pub fn sandbox_instance_event_context( sandbox_id: Option<&str>, sandbox: Option<&str>, container_image: String, @@ -902,12 +902,14 @@ impl TraceContext { } } -/// Trusted inputs owned by one supervised sandbox generation. +/// Trusted inputs owned by one supervised sandbox instance. /// -/// Nothing in a session reads process-wide sandbox state: identity, +/// A sandbox instance is one running generation of a stable Sandbox resource; +/// gateway connections may reconnect within it without creating another. +/// Nothing in an instance reads process-wide sandbox state: identity, /// credentials, policy inputs, logging settings, and file locations all come -/// from here, so one process can host several sessions. -pub struct SessionConfig { +/// from here, so one process can host several instances. +pub struct SandboxInstanceConfig { pub command: Vec, pub workdir: Option, pub timeout_secs: u64, @@ -921,7 +923,7 @@ pub struct SessionConfig { pub ssh_socket_path: Option, /// Grant the SSH socket to the supervisor's group rather than its owner. pub shared_ssh_socket: bool, - /// OCSF JSON output settings this session's gateway configuration controls. + /// OCSF JSON output settings this instance's gateway configuration controls. pub ocsf_json: openshell_ocsf::OcsfJsonSettings, pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, pub additional_ca_bundle: Option, @@ -940,24 +942,24 @@ pub struct SessionConfig { pub denial_log: openshell_supervisor_network::policy_local::DenialLogSource, } -/// How the owner of a session stops it and observes its readiness. -pub struct SessionControls { - /// Completes when the owner asks the session to stop its workload. It is +/// How the owner of a sandbox instance stops it and observes its readiness. +pub struct SandboxInstanceControls { + /// Completes when the owner asks the instance to stop its workload. It is /// first polled once the workload has started. pub shutdown: Pin + Send>>, - /// True while the gateway has accepted this session's supervision and its + /// True while the gateway has accepted this instance's supervision and its /// access plane is retained; false otherwise. pub readiness: tokio::sync::watch::Sender, } /// Mirrors gateway session acceptance into the owner's readiness until the /// access plane is released. -struct SessionReadiness { +struct SandboxInstanceReadiness { readiness: tokio::sync::watch::Sender, forward: Option>, } -impl SessionReadiness { +impl SandboxInstanceReadiness { fn start( readiness: tokio::sync::watch::Sender, accepted: Option>, @@ -994,24 +996,24 @@ impl SessionReadiness { } } -impl Drop for SessionReadiness { +impl Drop for SandboxInstanceReadiness { fn drop(&mut self) { self.release(); } } -/// Supervise one sandbox generation until its workload exits or the owner +/// Supervise one sandbox instance until its workload exits or the owner /// stops it. /// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. #[allow(clippy::similar_names)] -pub async fn run_session( +pub async fn run_sandbox_instance( backend_setup: &dyn backend_setup::BackendSetup, - config: SessionConfig, - controls: SessionControls, + config: SandboxInstanceConfig, + controls: SandboxInstanceControls, ) -> Result { - let SessionConfig { + let SandboxInstanceConfig { command, workdir, timeout_secs, @@ -1037,7 +1039,7 @@ pub async fn run_session( proxy_tls, denial_log, } = config; - let SessionControls { + let SandboxInstanceControls { shutdown: shutdown_requested, readiness, } = controls; @@ -1056,7 +1058,7 @@ pub async fn run_session( |(program, args)| (program.clone(), args.to_vec()), ); - // Extension credentials are owned by this supervisor session and shared + // Extension credentials are owned by this sandbox instance and shared // by every gateway connection it opens, so the middleware registry's // bearer slots and the policy poll loop that rotates them stay the same // objects. @@ -1076,7 +1078,7 @@ pub async fn run_session( let sandbox_bearer = gateway_credentials.sandbox_bearer().ok_or_else(|| { miette::miette!("supervisor gateway credentials carry no Sandbox Protocol credential") })?; - // Every gateway client of this session uses its own credentials. + // Every gateway client of this instance uses its own credentials. let gateway = openshell_endpoint.as_deref().map(|endpoint| { openshell_core::grpc_client::GatewayEndpoint::new(endpoint, gateway_credentials.clone()) }); @@ -1366,7 +1368,7 @@ pub async fn run_session( // unnecessarily delay DNS and network readiness. false, &provider_credentials, - openshell_supervisor_network::run::NetworkSession { + openshell_supervisor_network::run::SandboxInstanceNetwork { sandbox_id: sandbox_id.clone(), sandbox_name: sandbox_name_for_agg.clone(), gateway: gateway.clone(), @@ -1618,8 +1620,8 @@ pub async fn run_session( running.exec(), ) }); - let mut session_readiness = - SessionReadiness::start(readiness, boundary_access.session_readiness()); + let mut instance_readiness = + SandboxInstanceReadiness::start(readiness, boundary_access.session_readiness()); let instance_id = boundary_access.instance_id().to_string(); let wait_agent = agent.clone(); let mut shutdown_requested = shutdown_requested; @@ -1672,7 +1674,7 @@ pub async fn run_session( } }; if !retain_access { - session_readiness.release(); + instance_readiness.release(); } boundary_access .publish_main_exit(exit_code, await_main_process_attachment) @@ -1715,13 +1717,13 @@ pub async fn run_session( } if completion_cancelled { retain_access = false; - session_readiness.release(); + instance_readiness.release(); } if retain_access { info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; } - drop(session_readiness); + drop(instance_readiness); drop(running); drop(boundary_access); exit_code diff --git a/crates/openshell-supervisor/src/session_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs similarity index 65% rename from crates/openshell-supervisor/src/session_host.rs rename to crates/openshell-supervisor/src/sandbox_instance_host.rs index fcb4e989ff..4511b40176 100644 --- a/crates/openshell-supervisor/src/session_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -1,16 +1,21 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Several supervisor sessions in one process. +//! Several sandbox instances in one supervisor process. //! -//! Each session runs on its own Tokio runtime. Every thread that runtime -//! starts, and the thread driving it, is bound to the session's OCSF context, +//! A sandbox instance is one running generation of a stable Sandbox +//! resource. Gateway transport connections, and the gateway supervisor +//! sessions carried over them, may reconnect within one instance without +//! creating another. +//! +//! Each instance runs on its own Tokio runtime. Every thread that runtime +//! starts, and the thread driving it, is bound to the instance's OCSF context, //! OCSF JSON settings, log sink, and denial log, so tasks, callbacks, -//! reconnects, and credential renewal that the session spawns inherit its +//! reconnects, and credential renewal that the instance spawns inherit its //! identity. Unbound threads carry no sandbox identity and push no logs. -//! Gateway credentials are never bound to threads; each session passes its -//! own explicitly. Stopping a session shuts down only its runtime and revokes -//! its gateway credentials. +//! Gateway credentials are never bound to threads; each instance passes its +//! own explicitly. Stopping an instance shuts down only its runtime and +//! revokes its gateway credentials. use std::collections::HashMap; use std::future::Future; @@ -22,27 +27,27 @@ use std::time::Duration; use miette::Result; use openshell_core::grpc_client::GatewayCredentials; use openshell_ocsf::{EventContext, OcsfJsonSettings}; -use openshell_supervisor_network::policy_local::SessionDenialLog; +use openshell_supervisor_network::policy_local::SandboxInstanceDenialLog; use openshell_supervisor_process::log_push::LogSink; use tokio::sync::watch; -use crate::SessionControls; +use crate::SandboxInstanceControls; -/// Worker threads in each session runtime. -const SESSION_WORKER_THREADS: usize = 2; -/// Bound on joining a session thread after its runtime is told to stop. +/// Worker threads in each sandbox instance runtime. +const INSTANCE_WORKER_THREADS: usize = 2; +/// Bound on joining an instance thread after its runtime is told to stop. const RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); -/// Per-session values bound to every thread of the session's runtime. +/// Per-instance values bound to every thread of the instance's runtime. #[derive(Clone)] -pub struct SessionBinding { +pub struct SandboxInstanceBinding { pub event_context: Arc, pub ocsf_json: OcsfJsonSettings, pub log_sink: Option, - pub denial_log: SessionDenialLog, + pub denial_log: SandboxInstanceDenialLog, } -impl SessionBinding { +impl SandboxInstanceBinding { fn bind(&self) { openshell_ocsf::ctx::bind_thread_ctx(Some(self.event_context.clone())); openshell_ocsf::bind_thread_json_settings(Some(self.ocsf_json.clone())); @@ -59,23 +64,23 @@ impl SessionBinding { } std::thread_local! { - static THREAD_DENIAL_LOG: std::cell::RefCell> = + static THREAD_DENIAL_LOG: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; } -fn bind_thread_denial_log(log: Option) { +fn bind_thread_denial_log(log: Option) { THREAD_DENIAL_LOG.with(|current| *current.borrow_mut() = log); } -/// Shorthand-log destination for hosted sessions' recent denials. +/// Shorthand-log destination for hosted sandbox instances' recent denials. /// -/// Each line goes to the denial log of the session bound to the emitting -/// thread. Lines from unbound threads are dropped, so no session can read +/// Each line goes to the denial log of the instance bound to the emitting +/// thread. Lines from unbound threads are dropped, so no instance can read /// another's denials through `policy.local`. #[derive(Debug, Default, Clone, Copy)] -pub struct SessionDenialWriter; +pub struct SandboxInstanceDenialWriter; -impl std::io::Write for SessionDenialWriter { +impl std::io::Write for SandboxInstanceDenialWriter { fn write(&mut self, bytes: &[u8]) -> std::io::Result { THREAD_DENIAL_LOG.with(|log| { if let Some(log) = log.borrow().as_ref() { @@ -92,14 +97,14 @@ impl std::io::Write for SessionDenialWriter { } } -/// Observable lifecycle of one hosted session. +/// Observable lifecycle of one hosted sandbox instance. #[derive(Clone, Debug, PartialEq, Eq)] -pub enum SessionPhase { +pub enum SandboxInstancePhase { /// Startup has not yet produced an accepted gateway session. Starting, /// The workload started and the gateway accepted supervision at least once. Supervising, - /// The owner asked the session to stop. + /// The owner asked the instance to stop. Stopping, /// Supervision ended with the workload's exit status. Exited(i32), @@ -107,80 +112,85 @@ pub enum SessionPhase { Failed(String), } -impl SessionPhase { +impl SandboxInstancePhase { #[must_use] pub fn is_terminal(&self) -> bool { matches!(self, Self::Exited(_) | Self::Failed(_)) } } -/// Current state of one hosted session. +/// Current state of one hosted sandbox instance. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct SessionStatus { - pub phase: SessionPhase, - /// Gateway supervision is currently accepted for this session. +pub struct SandboxInstanceStatus { + pub phase: SandboxInstancePhase, + /// Gateway supervision is currently accepted for this instance. pub ready: bool, } -/// Body of a hosted session, constructed on its own runtime. -pub type SessionBody = - Box Pin>>> + Send + 'static>; +/// Body of a hosted sandbox instance, constructed on its own runtime. +pub type SandboxInstanceBody = Box< + dyn FnOnce(SandboxInstanceControls) -> Pin>>> + + Send + + 'static, +>; -/// The hosted-session registration was rejected. +/// The hosted sandbox instance registration was rejected. #[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum SessionHostError { - #[error("a session with this key is already hosted")] +pub enum SandboxInstanceHostError { + #[error("a sandbox instance with this key is already hosted")] Duplicate, - #[error("start session thread: {0}")] + #[error("start sandbox instance thread: {0}")] Thread(String), } -struct HostedSession { - /// Distinguishes this session from a later one that reuses its key. +struct HostedSandboxInstance { + /// Distinguishes this instance from a later one that reuses its key. serial: u64, stop: watch::Sender, abort: watch::Sender, - phase: watch::Receiver, + phase: watch::Receiver, ready: watch::Receiver, thread: Option>, /// Final status of the removal in progress, shared by every remover. - removal: Option>>, + removal: Option>>, } -impl HostedSession { - fn status(&self) -> SessionStatus { +impl HostedSandboxInstance { + fn status(&self) -> SandboxInstanceStatus { let phase = self.phase.borrow().clone(); // The runner always reports a terminal phase, even while unwinding; // a closed channel without one still means supervision has ended. if !phase.is_terminal() && self.phase.has_changed().is_err() { - return SessionStatus { - phase: SessionPhase::Failed("session thread ended unexpectedly".to_string()), + return SandboxInstanceStatus { + phase: SandboxInstancePhase::Failed( + "sandbox instance thread ended unexpectedly".to_string(), + ), ready: false, }; } - SessionStatus { + SandboxInstanceStatus { phase, ready: *self.ready.borrow(), } } } -type Sessions = Arc>>; +type Instances = Arc>>; -fn lock(sessions: &Sessions) -> MutexGuard<'_, HashMap> { - sessions +fn lock(instances: &Instances) -> MutexGuard<'_, HashMap> { + instances .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } -/// Runs and stops supervisor sessions, each on its own bound runtime. +/// Runs and stops sandbox instances, each on its own bound runtime. #[derive(Default)] -pub struct SessionHost { - sessions: Sessions, +pub struct SandboxInstanceHost { + instances: Instances, next_serial: AtomicU64, } -impl SessionHost { +impl SandboxInstanceHost { #[must_use] pub fn new() -> Self { Self::default() @@ -188,36 +198,36 @@ impl SessionHost { /// Start `body` on a new runtime bound to `binding`. /// - /// `credentials`, when present, are revoked once the session's runtime has - /// stopped, so nothing can reuse them after the session ends. + /// `credentials`, when present, are revoked once the instance's runtime + /// has stopped, so nothing can reuse them after the instance ends. /// /// # Errors /// - /// Returns [`SessionHostError::Duplicate`] when `key` is already hosted, - /// including a session that ended but has not been removed. + /// Returns [`SandboxInstanceHostError::Duplicate`] when `key` is already hosted, + /// including an instance that ended but has not been removed. pub fn start( &self, key: &str, - binding: SessionBinding, + binding: SandboxInstanceBinding, credentials: Option, - body: SessionBody, - ) -> std::result::Result<(), SessionHostError> { - let mut sessions = lock(&self.sessions); - if sessions.contains_key(key) { - return Err(SessionHostError::Duplicate); + body: SandboxInstanceBody, + ) -> std::result::Result<(), SandboxInstanceHostError> { + let mut instances = lock(&self.instances); + if instances.contains_key(key) { + return Err(SandboxInstanceHostError::Duplicate); } let (stop, stop_requested) = watch::channel(false); let (abort, abort_requested) = watch::channel(false); - let (phase_tx, phase) = watch::channel(SessionPhase::Starting); + let (phase_tx, phase) = watch::channel(SandboxInstancePhase::Starting); let (ready_tx, ready) = watch::channel(false); let thread = std::thread::Builder::new() - .name("openshell-session".to_string()) + .name("openshell-instance".to_string()) .spawn(move || { - run_hosted_session( + run_hosted_sandbox_instance( &binding, credentials, body, - SessionSignals { + SandboxInstanceSignals { stop_requested, abort_requested, phase: phase_tx, @@ -225,10 +235,10 @@ impl SessionHost { }, ); }) - .map_err(|error| SessionHostError::Thread(error.to_string()))?; - sessions.insert( + .map_err(|error| SandboxInstanceHostError::Thread(error.to_string()))?; + instances.insert( key.to_string(), - HostedSession { + HostedSandboxInstance { serial: self.next_serial.fetch_add(1, Ordering::Relaxed), stop, abort, @@ -241,30 +251,32 @@ impl SessionHost { Ok(()) } - /// Status of a hosted session, including one that ended but has not been - /// removed. + /// Status of a hosted sandbox instance, including one that ended but has + /// not been removed. #[must_use] - pub fn status(&self, key: &str) -> Option { - lock(&self.sessions).get(key).map(HostedSession::status) + pub fn status(&self, key: &str) -> Option { + lock(&self.instances) + .get(key) + .map(HostedSandboxInstance::status) } - /// Keys of every hosted session. + /// Keys of every hosted sandbox instance. #[must_use] pub fn keys(&self) -> Vec { - lock(&self.sessions).keys().cloned().collect() + lock(&self.instances).keys().cloned().collect() } - /// Stop one session and forget it. Other sessions keep running. + /// Stop one sandbox instance and forget it. Other instances keep running. /// - /// A supervising session terminates its workload gracefully within + /// A supervising instance terminates its workload gracefully within /// `grace`; one still starting is cancelled at once, because startup has /// no safe intermediate state to wind down. Removal runs on its own task, /// so it completes even when the caller stops waiting, and concurrent /// callers share it and its result. The key stays registered until the - /// session's thread has finished, so the same key cannot start a second + /// instance's thread has finished, so the same key cannot start a second /// owner while this one is stopping. Returns the final status, or `None` /// when `key` is not hosted. - pub async fn remove(&self, key: &str, grace: Duration) -> Option { + pub async fn remove(&self, key: &str, grace: Duration) -> Option { let (mut removed, phase) = self.begin_removal(key, grace)?; // The removal task ends without a report only with its runtime. removed @@ -278,126 +290,132 @@ impl SessionHost { key: &str, grace: Duration, ) -> Option<( - watch::Receiver>, - watch::Receiver, + watch::Receiver>, + watch::Receiver, )> { - let mut sessions = lock(&self.sessions); - let session = sessions.get_mut(key)?; - if let Some(removed) = &session.removal { - return Some((removed.clone(), session.phase.clone())); + let mut instances = lock(&self.instances); + let instance = instances.get_mut(key)?; + if let Some(removed) = &instance.removal { + return Some((removed.clone(), instance.phase.clone())); } - session.stop.send_replace(true); - if *session.phase.borrow() == SessionPhase::Starting { - session.abort.send_replace(true); + instance.stop.send_replace(true); + if *instance.phase.borrow() == SandboxInstancePhase::Starting { + instance.abort.send_replace(true); } let (report, removed) = watch::channel(None); - session.removal = Some(removed.clone()); + instance.removal = Some(removed.clone()); tokio::spawn( Removal { - sessions: self.sessions.clone(), + instances: self.instances.clone(), key: key.to_string(), - serial: session.serial, - phase: session.phase.clone(), - thread: session.thread.take(), + serial: instance.serial, + phase: instance.phase.clone(), + thread: instance.thread.take(), grace, report, } .run(), ); - Some((removed, session.phase.clone())) + Some((removed, instance.phase.clone())) } } -fn terminal_status(phase: &watch::Receiver) -> SessionStatus { - SessionStatus { +fn terminal_status(phase: &watch::Receiver) -> SandboxInstanceStatus { + SandboxInstanceStatus { phase: phase.borrow().clone(), ready: false, } } -/// The single owner of one session's removal. +/// The single owner of one sandbox instance's removal. struct Removal { - sessions: Sessions, + instances: Instances, key: String, serial: u64, - phase: watch::Receiver, + phase: watch::Receiver, thread: Option>, grace: Duration, - report: watch::Sender>, + report: watch::Sender>, } impl Removal { async fn run(mut self) { - let finished = - tokio::time::timeout(self.grace, self.phase.wait_for(SessionPhase::is_terminal)) - .await - .is_ok(); + let finished = tokio::time::timeout( + self.grace, + self.phase.wait_for(SandboxInstancePhase::is_terminal), + ) + .await + .is_ok(); if !finished - && let Some(session) = lock(&self.sessions) + && let Some(instance) = lock(&self.instances) .get(&self.key) - .filter(|session| session.serial == self.serial) + .filter(|instance| instance.serial == self.serial) { - session.abort.send_replace(true); + instance.abort.send_replace(true); } if let Some(thread) = self.thread.take() { let _ = tokio::task::spawn_blocking(move || thread.join()).await; } let removed = { - let mut sessions = lock(&self.sessions); - match sessions.get(&self.key) { - Some(session) if session.serial == self.serial => sessions.remove(&self.key), + let mut instances = lock(&self.instances); + match instances.get(&self.key) { + Some(instance) if instance.serial == self.serial => instances.remove(&self.key), _ => None, } }; - let status = - removed.map_or_else(|| terminal_status(&self.phase), |session| session.status()); + let status = removed.map_or_else( + || terminal_status(&self.phase), + |instance| instance.status(), + ); self.report.send_replace(Some(status)); } } -struct SessionSignals { +struct SandboxInstanceSignals { stop_requested: watch::Receiver, abort_requested: watch::Receiver, - phase: watch::Sender, + phase: watch::Sender, ready: watch::Sender, } -/// Reports a hosted session's end and revokes its credentials, including when -/// the session thread unwinds. -struct SessionExit { +/// Reports a hosted sandbox instance's end and revokes its credentials, +/// including when the instance thread unwinds. +struct SandboxInstanceExit { credentials: Option, - phase: watch::Sender, + phase: watch::Sender, ready: watch::Sender, - outcome: Option, + outcome: Option, } -impl Drop for SessionExit { +impl Drop for SandboxInstanceExit { fn drop(&mut self) { if let Some(credentials) = self.credentials.take() { credentials.revoke(); } self.ready.send_replace(false); let outcome = self.outcome.take().unwrap_or_else(|| { - SessionPhase::Failed("session thread ended without reporting a result".to_string()) + SandboxInstancePhase::Failed( + "sandbox instance thread ended without reporting a result".to_string(), + ) }); self.phase.send_replace(outcome); - SessionBinding::unbind(); + SandboxInstanceBinding::unbind(); } } -fn run_hosted_session( - binding: &SessionBinding, +fn run_hosted_sandbox_instance( + binding: &SandboxInstanceBinding, credentials: Option, - body: SessionBody, - signals: SessionSignals, + body: SandboxInstanceBody, + signals: SandboxInstanceSignals, ) { - let SessionSignals { + let SandboxInstanceSignals { stop_requested, mut abort_requested, phase, ready, } = signals; - let mut exit = SessionExit { + let mut exit = SandboxInstanceExit { credentials, phase: phase.clone(), ready: ready.clone(), @@ -406,11 +424,11 @@ fn run_hosted_session( binding.bind(); let thread_binding = binding.clone(); let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(SESSION_WORKER_THREADS) - .thread_name("openshell-session-worker") + .worker_threads(INSTANCE_WORKER_THREADS) + .thread_name("openshell-instance-worker") .enable_all() .on_thread_start(move || thread_binding.bind()) - .on_thread_stop(SessionBinding::unbind) + .on_thread_stop(SandboxInstanceBinding::unbind) .build(); let outcome = match runtime { Ok(runtime) => { @@ -425,9 +443,9 @@ fn run_hosted_session( mirror_ready.send_replace(is_ready); if is_ready { mirror_phase.send_if_modified(|current| { - let promote = *current == SessionPhase::Starting; + let promote = *current == SandboxInstancePhase::Starting; if promote { - *current = SessionPhase::Supervising; + *current = SandboxInstancePhase::Supervising; } promote }); @@ -442,16 +460,16 @@ fn run_hosted_session( let stop_phase = phase.clone(); let shutdown = Box::pin(async move { let _ = stop.wait_for(|requested| *requested).await; - stop_phase.send_replace(SessionPhase::Stopping); + stop_phase.send_replace(SandboxInstancePhase::Stopping); }); - let session = body(SessionControls { + let instance = body(SandboxInstanceControls { shutdown, readiness, }); let outcome = tokio::select! { - result = session => result, + result = instance => result, _ = abort_requested.wait_for(|requested| *requested) => { - Err(miette::miette!("session was cancelled before it finished")) + Err(miette::miette!("sandbox instance was cancelled before it finished")) } }; mirror.abort(); @@ -459,43 +477,43 @@ fn run_hosted_session( }) })); runtime.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT); - outcome.unwrap_or_else(|_| Err(miette::miette!("session panicked"))) + outcome.unwrap_or_else(|_| Err(miette::miette!("sandbox instance panicked"))) } - Err(error) => Err(miette::miette!("build session runtime: {error}")), + Err(error) => Err(miette::miette!("build sandbox instance runtime: {error}")), }; exit.outcome = Some(match outcome { - Ok(code) => SessionPhase::Exited(code), - Err(error) => SessionPhase::Failed(error.to_string()), + Ok(code) => SandboxInstancePhase::Exited(code), + Err(error) => SandboxInstancePhase::Failed(error.to_string()), }); } -/// Trusted inputs for one supervisor session hosted with others. -pub struct HostedSessionSpec { +/// Trusted inputs for one sandbox instance hosted with others. +pub struct SandboxInstanceSpec { /// Unique key for this generation within the host. pub key: String, pub backend_setup: Box, - pub config: crate::SessionConfig, - /// Workload image recorded in this session's OCSF events. + pub config: crate::SandboxInstanceConfig, + /// Workload image recorded in this instance's OCSF events. pub container_image: String, } -impl SessionHost { - /// Start a supervisor session with its own OCSF identity, OCSF JSON +impl SandboxInstanceHost { + /// Start a sandbox instance with its own OCSF identity, OCSF JSON /// settings, log push stream, denial log, and gateway credentials. /// /// # Errors /// /// Fails when the launch credentials are invalid or the key is hosted. - pub fn start_supervisor_session(&self, spec: HostedSessionSpec) -> Result<()> { - let HostedSessionSpec { + pub fn start_sandbox_instance(&self, spec: SandboxInstanceSpec) -> Result<()> { + let SandboxInstanceSpec { key, backend_setup, mut config, container_image, } = spec; let credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; - // Sessions must never share proxy trust files; give one a private - // directory that lives exactly as long as the session. + // Instances must never share proxy trust files; give one a private + // directory that lives exactly as long as the instance. let private_tls_dir = if config.proxy_tls.directory.is_none() { let prepared = crate::prepare_network_proxy_tls_dir(None)?; config.proxy_tls.directory = Some(prepared.path.clone()); @@ -510,8 +528,8 @@ impl SessionHost { openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES, ); let log_push = config.sandbox_id.clone().zip(gateway); - let binding = SessionBinding { - event_context: Arc::new(crate::session_event_context( + let binding = SandboxInstanceBinding { + event_context: Arc::new(crate::sandbox_instance_event_context( config.sandbox_id.as_deref(), config.sandbox.as_deref(), container_image, @@ -520,14 +538,15 @@ impl SessionHost { log_sink: log_push .as_ref() .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), - denial_log: SessionDenialLog::default(), + denial_log: SandboxInstanceDenialLog::default(), }; config.gateway_credentials = Some(credentials.clone()); config.ocsf_json = binding.ocsf_json.clone(); - config.denial_log = openshell_supervisor_network::policy_local::DenialLogSource::Session( - binding.denial_log.clone(), - ); - let body: SessionBody = Box::new(move |controls| { + config.denial_log = + openshell_supervisor_network::policy_local::DenialLogSource::SandboxInstance( + binding.denial_log.clone(), + ); + let body: SandboxInstanceBody = Box::new(move |controls| { Box::pin(async move { let _private_tls_dir = private_tls_dir; let _log_push = log_push.map(|(sandbox_id, gateway)| { @@ -535,7 +554,12 @@ impl SessionHost { gateway, sandbox_id, log_rx, ) }); - Box::pin(crate::run_session(backend_setup.as_ref(), config, controls)).await + Box::pin(crate::run_sandbox_instance( + backend_setup.as_ref(), + config, + controls, + )) + .await }) }); self.start(&key, binding, Some(credentials), body) @@ -613,7 +637,7 @@ mod tests { .collect() } - struct Session { + struct Instance { identity: tokio::sync::oneshot::Receiver, logs: mpsc::Receiver, call_again: mpsc::Sender<()>, @@ -621,31 +645,35 @@ mod tests { credentials: GatewayCredentials, } - fn start_session(host: &SessionHost, gateway_url: &str, sandbox: &'static str) -> Session { + fn start_instance( + host: &SandboxInstanceHost, + gateway_url: &str, + sandbox: &'static str, + ) -> Instance { let credentials = GatewayCredentials::from_gateway_token(&format!("token-{sandbox}")).unwrap(); let gateway = GatewayEndpoint::new(gateway_url, credentials.clone()); let (log_tx, logs) = mpsc::channel(16); - let binding = SessionBinding { - event_context: Arc::new(crate::session_event_context( + let binding = SandboxInstanceBinding { + event_context: Arc::new(crate::sandbox_instance_event_context( Some(sandbox), Some(sandbox), String::new(), )), ocsf_json: OcsfJsonSettings::default(), log_sink: Some(LogSink::new(sandbox, log_tx)), - denial_log: SessionDenialLog::default(), + denial_log: SandboxInstanceDenialLog::default(), }; let (identity_tx, identity) = tokio::sync::oneshot::channel(); let (call_again, mut calls) = mpsc::channel(1); let (alive_tx, alive) = mpsc::channel::<()>(1); - let body: SessionBody = Box::new(move |controls| { + let body: SandboxInstanceBody = Box::new(move |controls| { Box::pin(async move { - let SessionControls { + let SandboxInstanceControls { shutdown, readiness, } = controls; - // Nested tasks and the blocking pool inherit the session identity. + // Nested tasks and the blocking pool inherit the instance identity. let observed = tokio::spawn(async { tokio::task::spawn_blocking(|| openshell_ocsf::ctx::ctx().sandbox_id.clone()) .await @@ -662,12 +690,12 @@ mod tests { ); tokio::spawn(async move { tracing::dispatcher::with_default(&dispatch, || { - tracing::info!(target: "session_test", "emitted by {sandbox}"); + tracing::info!(target: "instance_test", "emitted by {sandbox}"); }); }) .await .unwrap(); - // A long-lived task the session never stops itself. + // A long-lived task the instance never stops itself. tokio::spawn(async move { let _alive = alive_tx; std::future::pending::<()>().await; @@ -689,7 +717,7 @@ mod tests { }); host.start(sandbox, binding, Some(credentials.clone()), body) .unwrap(); - Session { + Instance { identity, logs, call_again, @@ -698,29 +726,35 @@ mod tests { } } - async fn wait_supervising(host: &SessionHost, key: &str) { + async fn wait_supervising(host: &SandboxInstanceHost, key: &str) { tokio::time::timeout(Duration::from_secs(10), async { - while host.status(key).map(|status| status.phase) != Some(SessionPhase::Supervising) { + while host.status(key).map(|status| status.phase) + != Some(SandboxInstancePhase::Supervising) + { tokio::time::sleep(Duration::from_millis(10)).await; } }) .await - .expect("session reaches supervision"); + .expect("instance reaches supervision"); } - fn unbound() -> SessionBinding { - SessionBinding { - event_context: Arc::new(crate::session_event_context(None, None, String::new())), + fn unbound() -> SandboxInstanceBinding { + SandboxInstanceBinding { + event_context: Arc::new(crate::sandbox_instance_event_context( + None, + None, + String::new(), + )), ocsf_json: OcsfJsonSettings::default(), log_sink: None, - denial_log: SessionDenialLog::default(), + denial_log: SandboxInstanceDenialLog::default(), } } #[tokio::test(flavor = "multi_thread")] - async fn stopping_and_panicking_sessions_keep_their_key_and_fail_closed() { - let host = Arc::new(SessionHost::new()); - let body: SessionBody = Box::new(|controls| { + async fn stopping_and_panicking_instances_keep_their_key_and_fail_closed() { + let host = Arc::new(SandboxInstanceHost::new()); + let body: SandboxInstanceBody = Box::new(|controls| { Box::pin(async move { controls.readiness.send_replace(true); controls.shutdown.await; @@ -737,28 +771,28 @@ mod tests { }); tokio::time::timeout(Duration::from_secs(10), async { while host.status("generation").map(|status| status.phase) - != Some(SessionPhase::Stopping) + != Some(SandboxInstancePhase::Stopping) { tokio::time::sleep(Duration::from_millis(5)).await; } }) .await - .expect("session observes the stop request"); - let replacement: SessionBody = Box::new(|_| Box::pin(async { Ok(0) })); + .expect("instance observes the stop request"); + let replacement: SandboxInstanceBody = Box::new(|_| Box::pin(async { Ok(0) })); assert_eq!( host.start("generation", unbound(), None, replacement), - Err(SessionHostError::Duplicate) + Err(SandboxInstanceHostError::Duplicate) ); - let stopped = removal.await.unwrap().expect("session was hosted"); - assert_eq!(stopped.phase, SessionPhase::Exited(0)); + let stopped = removal.await.unwrap().expect("instance was hosted"); + assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); assert!(host.status("generation").is_none()); let credentials = GatewayCredentials::from_gateway_token("panicking-token").unwrap(); - let panicking: SessionBody = Box::new(|controls| { + let panicking: SandboxInstanceBody = Box::new(|controls| { Box::pin(async move { controls.readiness.send_replace(true); tokio::task::yield_now().await; - panic!("hosted session failure"); + panic!("hosted sandbox instance failure"); }) }); host.start("panicking", unbound(), Some(credentials.clone()), panicking) @@ -774,11 +808,11 @@ mod tests { } }) .await - .expect("panicking session reports an end"); + .expect("panicking instance reports an end"); assert_eq!( status, - SessionStatus { - phase: SessionPhase::Failed("session panicked".to_string()), + SandboxInstanceStatus { + phase: SandboxInstancePhase::Failed("sandbox instance panicked".to_string()), ready: false, } ); @@ -796,8 +830,8 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn removal_outlives_its_caller_and_reports_to_every_remover() { - let host = Arc::new(SessionHost::new()); - let winding_down = |code| -> SessionBody { + let host = Arc::new(SandboxInstanceHost::new()); + let winding_down = |code| -> SandboxInstanceBody { Box::new(move |controls| { Box::pin(async move { controls.readiness.send_replace(true); @@ -824,8 +858,8 @@ mod tests { tokio::spawn(async move { host.remove("generation", Duration::from_secs(10)).await }) }); for remover in removers { - let status = remover.await.unwrap().expect("session was hosted"); - assert_eq!(status.phase, SessionPhase::Exited(3)); + let status = remover.await.unwrap().expect("instance was hosted"); + assert_eq!(status.phase, SandboxInstancePhase::Exited(3)); } assert!(host.status("generation").is_none()); @@ -836,46 +870,46 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; assert_eq!( host.status("generation").map(|status| status.phase), - Some(SessionPhase::Supervising) + Some(SandboxInstancePhase::Supervising) ); let status = host.remove("generation", Duration::from_secs(10)).await; assert_eq!( status.map(|status| status.phase), - Some(SessionPhase::Exited(4)) + Some(SandboxInstancePhase::Exited(4)) ); } #[tokio::test(flavor = "multi_thread")] - async fn hosted_sessions_keep_identity_credentials_and_lifecycle_separate() { + async fn hosted_instances_keep_identity_credentials_and_lifecycle_separate() { let (gateway_url, captured) = capture_gateway().await; - let host = SessionHost::new(); - let mut alpha = start_session(&host, &gateway_url, "sandbox-alpha"); - let mut beta = start_session(&host, &gateway_url, "sandbox-beta"); + let host = SandboxInstanceHost::new(); + let mut alpha = start_instance(&host, &gateway_url, "sandbox-alpha"); + let mut beta = start_instance(&host, &gateway_url, "sandbox-beta"); wait_supervising(&host, "sandbox-alpha").await; wait_supervising(&host, "sandbox-beta").await; assert!(matches!( host.start( "sandbox-alpha", - SessionBinding { - event_context: Arc::new(crate::session_event_context( + SandboxInstanceBinding { + event_context: Arc::new(crate::sandbox_instance_event_context( None, None, String::new() )), ocsf_json: OcsfJsonSettings::default(), log_sink: None, - denial_log: SessionDenialLog::default(), + denial_log: SandboxInstanceDenialLog::default(), }, None, Box::new(|_| Box::pin(async { Ok(0) })), ), - Err(SessionHostError::Duplicate) + Err(SandboxInstanceHostError::Duplicate) )); assert_eq!((&mut alpha.identity).await.unwrap(), "sandbox-alpha"); assert_eq!((&mut beta.identity).await.unwrap(), "sandbox-beta"); // Other tests in this process may set the process-wide context; a - // thread outside every session must still carry no session identity. + // thread outside every instance must still carry no instance identity. let outside = openshell_ocsf::ctx::ctx().sandbox_id.clone(); assert!(!["sandbox-alpha", "sandbox-beta"].contains(&outside.as_str())); assert_eq!( @@ -886,18 +920,18 @@ mod tests { bearers_for(&captured, "sandbox-beta"), ["Bearer token-sandbox-beta"] ); - for (session, sandbox) in [(&mut alpha, "sandbox-alpha"), (&mut beta, "sandbox-beta")] { - let line = session.logs.try_recv().expect("session log line"); + for (instance, sandbox) in [(&mut alpha, "sandbox-alpha"), (&mut beta, "sandbox-beta")] { + let line = instance.logs.try_recv().expect("instance log line"); assert_eq!(line.sandbox_id, sandbox); assert_eq!(line.message, format!("emitted by {sandbox}")); - assert!(session.logs.try_recv().is_err()); + assert!(instance.logs.try_recv().is_err()); } let stopped = host .remove("sandbox-alpha", Duration::from_secs(10)) .await .expect("alpha was hosted"); - assert_eq!(stopped.phase, SessionPhase::Exited(0)); + assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); assert!(!stopped.ready); assert!(host.status("sandbox-alpha").is_none()); assert_eq!(alpha.alive.recv().await, None, "alpha's tasks stopped"); @@ -921,7 +955,7 @@ mod tests { ); assert_eq!( host.status("sandbox-beta").map(|status| status.phase), - Some(SessionPhase::Supervising) + Some(SandboxInstancePhase::Supervising) ); assert!( beta.alive @@ -932,7 +966,7 @@ mod tests { let stopped = host.remove("sandbox-beta", Duration::from_secs(10)).await; assert_eq!( stopped.map(|status| status.phase), - Some(SessionPhase::Exited(0)) + Some(SandboxInstancePhase::Exited(0)) ); } } From e1be7e9c35fe4dac942eef38520736f013aa7895 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 20 Oct 2026 20:21:32 -0700 Subject: [PATCH 19/25] refactor(supervisor): launch sandbox instances from one config and prepared bindings Replace SandboxRunConfig and its field-by-field copy into SandboxInstanceConfig with a single launch configuration. Live per-instance state (gateway credentials, OCSF context and JSON settings, log sink, and denial log) is now an explicit SandboxInstanceBindings value that the owner prepares once, instead of placeholders the runner or host overwrote. The dedicated binary loads launch inputs from its arguments and environment and shares one GatewayCredentials with log push; run_sandbox_with_backend keeps the process-wide OCSF context, health endpoints, and signal handling. The instance host binds the same SandboxInstanceBindings to its runtime threads and revokes their credentials when the instance ends. Signed-off-by: Drew Newberry --- crates/openshell-supervisor/README.md | 45 +-- .../src/backend_setup/tests.rs | 58 ++-- .../src/legacy_startup_regression_tests.rs | 64 ++-- crates/openshell-supervisor/src/lib.rs | 321 +++++++----------- crates/openshell-supervisor/src/main.rs | 66 ++-- .../src/sandbox_instance_host.rs | 174 +++++----- 6 files changed, 337 insertions(+), 391 deletions(-) diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index d603af388f..ac2f3ed888 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -4,7 +4,7 @@ The supervisor loads and reconciles policy, maintains provider credentials, appl ## Backend startup -The public `run_sandbox` entry point selects the OpenShell Sandbox Protocol backend and collects its startup inputs into a public `SandboxRunConfig`. Shared startup receives that config and the trusted backend setup separately. The `backend_setup` module owns that backend's launch-data decoder, workload policy discovery, and client construction. Descriptor contents cannot select an implementation. +The supervisor binary selects a backend setup and collects its launch inputs into a public `SandboxInstanceConfig`. Shared startup receives that config and the trusted backend setup separately. The `backend_setup` module owns that backend's launch-data decoder, workload policy discovery, and client construction. Descriptor contents cannot select an implementation. Shared startup checks the admitted backend name before passing the opaque payload to its decoder. It then compares the decoded sandbox, session, and runtime generation with the trusted launch inputs before installing credentials or discovering workload policy. A mismatch stops startup. @@ -14,7 +14,7 @@ The supervisor admits policy and prepares credentials before constructing and at The client receives the supervisor's live provider state, bearer-token slot, and CA-path slot. Provider refresh, token rotation, and later CA publication must remain visible through those shared handles. Startup does not create independent copies of their current values. -`backend_setup`, `SandboxRunConfig`, and `run_sandbox_with_backend` are public so the supervisor binary can select a registered backend. External backends should implement the `IsolationBackend` and `SandboxProtocol` services rather than link this crate. +`backend_setup`, `SandboxInstanceConfig`, `SandboxInstanceBindings`, and `run_sandbox_with_backend` are public so the supervisor binary can select a registered backend. External backends should implement the `IsolationBackend` and `SandboxProtocol` services rather than link this crate. The startup result separates the backend transport payload from optional authenticated CONNECT listener settings. Shared networking starts the listener @@ -29,27 +29,32 @@ Gateway connections, and the gateway supervisor sessions carried over them, can reconnect within an instance without creating another one; the generation remains the fencing identifier. -`run_sandbox_instance` supervises one sandbox instance from a -`SandboxInstanceConfig` and `SandboxInstanceControls`. Everything that belongs -to that instance is an input or is created by the instance: gateway -credentials and their renewal, the Sandbox Protocol credential, policy, -provider credentials and token grants, middleware credentials, OCSF JSON -settings, proxy trust files, the denial history served by `policy.local`, the -backend connection, access plane, readiness, and stop signal. Gateway clients -receive a `GatewayEndpoint`, which pairs the URL with that instance's -`GatewayCredentials`; nothing is installed process-wide. - -`run_sandbox_with_backend` is the dedicated wrapper. It keeps the existing -process contract: it sets the process-wide OCSF context, reads launch inputs -from the environment, binds the readiness socket and port to instance -readiness, and stops the instance on SIGTERM or SIGINT. The binary shares one -`GatewayCredentials` between log push and supervision, so renewal by either -keeps both current. +`run_sandbox_instance` supervises one sandbox instance from three inputs. +`SandboxInstanceConfig` is the single launch configuration: identity, command, +policy inputs, auth bundle, backend descriptor, and file locations. +`SandboxInstanceBindings` is the live state its owner prepares once: the +`GatewayCredentials` issued from the auth bundle, the OCSF context and JSON +settings, the log sink, and the denial history served by `policy.local`. +`SandboxInstanceControls` carries the owner's stop signal and readiness. +Everything else, including credential renewal, policy, provider credentials and +token grants, middleware credentials, the backend connection, and the access +plane, is created by the instance. Gateway clients receive a `GatewayEndpoint`, +which pairs the URL with that instance's `GatewayCredentials`; nothing is +installed process-wide. + +The dedicated process wrapper keeps the existing process contract. The binary +loads the launch configuration from its arguments and environment and prepares +`SandboxInstanceBindings::dedicated`, sharing one `GatewayCredentials` with log +push and the OCSF JSON settings with its logging layers, so renewal by either +gateway client keeps both current. `run_sandbox_with_backend` sets the +process-wide OCSF context, binds the readiness socket and port to instance +readiness, and stops the instance on SIGTERM or SIGINT. `sandbox_instance_host::SandboxInstanceHost` runs several sandbox instances in one process. Each instance gets its own Tokio runtime; every thread of that -runtime, and the thread driving it, is bound to the instance's OCSF context, -OCSF JSON settings, log sink, and denial log. Spawned tasks, blocking work, +runtime, and the thread driving it, is bound to the instance's +`SandboxInstanceBindings`: OCSF context, OCSF JSON settings, log sink, and +denial log. Spawned tasks, blocking work, callbacks, reconnects, and renewal therefore inherit the instance's identity, and threads outside an instance carry none and push no logs. The OCSF context also supplies the sandbox identity in middleware requests. diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index b9b5c1cdc9..b4704bf623 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -680,32 +680,42 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .require_networking .store(true, Ordering::SeqCst); let startup = |await_main_process_attachment| { + let config = crate::SandboxInstanceConfig { + command: vec!["test-agent".into(), "test-argument".into()], + workdir: Some("/test-workspace".into()), + timeout_secs: 15, + interactive: false, + await_main_process_attachment, + sandbox_id: Some(setup.sandbox_id.clone()), + sandbox: None, + container_image: String::new(), + openshell_endpoint: None, + policy_rules: Some(rules.to_string_lossy().into_owned()), + policy_data: Some(data.to_string_lossy().into_owned()), + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: TestSetup::descriptor(), + auth_bundle: setup.auth.clone(), + admitted_isolation_backend: Some(TEST_BACKEND.into()), + main_exit_marker: Some(marker.clone()), + trace_context: None, + provider_workload_api_socket: None, + // The binary loads the proxy trust directory from its environment. + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + }; + let bindings = crate::SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&setup.auth).unwrap(), + openshell_ocsf::OcsfJsonSettings::default(), + ); Box::pin(crate::run_sandbox_with_backend( &setup, - crate::SandboxRunConfig { - command: vec!["test-agent".into(), "test-argument".into()], - workdir: Some("/test-workspace".into()), - timeout_secs: 15, - interactive: false, - await_main_process_attachment, - sandbox_id: Some(setup.sandbox_id.clone()), - sandbox: None, - openshell_endpoint: None, - policy_rules: Some(rules.to_string_lossy().into_owned()), - policy_data: Some(data.to_string_lossy().into_owned()), - ssh_socket_path: None, - health_socket_path: Some(readiness.clone()), - health_port: Some(health_port), - ocsf_enabled: Arc::new(AtomicBool::new(false)), - ocsf_schema_version: Arc::new(Mutex::new(String::new())), - upstream_proxy_args: UpstreamProxyArgs::default(), - additional_ca_bundle: None, - backend_descriptor: TestSetup::descriptor(), - auth_bundle: setup.auth.clone(), - gateway_credentials: None, - admitted_isolation_backend: Some(TEST_BACKEND.into()), - main_exit_marker: Some(marker.clone()), - }, + config, + bindings, + Some(readiness.clone()), + Some(health_port), )) }; diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index e27bd23455..699880f1ad 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -11,7 +11,7 @@ use openshell_core::proto::{ UpdateConfigResponse, gateway_message, supervisor_message, }; use openshell_isolation_interface::contract::{BackendError, ResolvedWorkloadIdentity}; -use std::sync::{Mutex, atomic::Ordering}; +use std::sync::atomic::Ordering; use tonic::codegen::tokio_stream::{StreamExt as _, wrappers::TcpListenerStream}; use tonic::codegen::{Body, BoxFuture, BoxStream, Bytes, Service, StdError, http}; @@ -277,37 +277,41 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl session_id, probe: probe.clone(), }; - let startup = run_sandbox_with_backend( - &setup, - SandboxRunConfig { - command: vec!["true".into()], - workdir: None, - timeout_secs: 0, - interactive: false, - await_main_process_attachment: false, - sandbox_id: Some("sb-legacy-restart".into()), - sandbox: Some("legacy-restart".into()), - openshell_endpoint: Some(endpoint), - policy_rules: None, - policy_data: None, - ssh_socket_path: None, - health_socket_path: None, - health_port: None, - ocsf_enabled: Arc::new(AtomicBool::new(false)), - ocsf_schema_version: Arc::new(Mutex::new(String::new())), - upstream_proxy_args: - openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), - additional_ca_bundle: None, - backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { - backend_name: "test-held".into(), - payload: Vec::new(), - }, - auth_bundle, - gateway_credentials: None, - admitted_isolation_backend: Some("test-held".into()), - main_exit_marker: None, + let config = SandboxInstanceConfig { + command: vec!["true".into()], + workdir: None, + timeout_secs: 0, + interactive: false, + await_main_process_attachment: false, + sandbox_id: Some("sb-legacy-restart".into()), + sandbox: Some("legacy-restart".into()), + container_image: String::new(), + openshell_endpoint: Some(endpoint), + policy_rules: None, + policy_data: None, + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: + openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { + backend_name: "test-held".into(), + payload: Vec::new(), }, + auth_bundle, + admitted_isolation_backend: Some("test-held".into()), + main_exit_marker: None, + trace_context: None, + provider_workload_api_socket: None, + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), + }; + let bindings = SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) + .unwrap(), + openshell_ocsf::OcsfJsonSettings::default(), ); + let startup = run_sandbox_with_backend(&setup, config, bindings, None, None); tokio::pin!(startup); let result = tokio::select! { () = probe.ready_event.notified() => None, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 69ee814a34..39699751e2 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -67,7 +67,7 @@ use openshell_ocsf::{ /// /// The singleton lives in `openshell-ocsf` so both supervisor leaves can /// reach it without depending on `openshell-sandbox`. Initialised once during -/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. +/// `run_sandbox_with_backend()` startup via `openshell_ocsf::ctx::set_ctx`. pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; /// The sandbox rejected the image working directory while starting the agent. @@ -388,7 +388,10 @@ use provider_readiness::{EnvironmentIdentity, Tracker as ProviderReadinessTracke use tokio::sync::mpsc::UnboundedSender; use tokio::time::timeout; -fn shared_ssh_socket_from_env() -> bool { +/// Whether a dedicated supervisor's environment grants the SSH socket to its +/// group rather than its owner. +#[must_use] +pub fn shared_ssh_socket_from_env() -> bool { std::env::var(openshell_core::sandbox_env::SSH_SOCKET_SHARED) .is_ok_and(|value| shared_ssh_socket_value(&value)) } @@ -661,146 +664,35 @@ pub async fn run_network_proxy( } } -/// Run a command in the sandbox. -/// -/// # Errors -/// -/// Returns an error if the command fails to start or encounters a fatal error. -#[allow( - clippy::too_many_arguments, - clippy::implicit_hasher, - clippy::similar_names, - clippy::fn_params_excessive_bools -)] -pub async fn run_sandbox( - command: Vec, - workdir: Option, - timeout_secs: u64, - interactive: bool, - await_main_process_attachment: bool, - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - ssh_socket_path: Option, - health_socket_path: Option, - health_port: Option, - ocsf_enabled: Arc, - ocsf_schema_version: Arc>, - upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, - additional_ca_bundle: Option, - backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, - auth_bundle: openshell_core::jwt::SupervisorAuthBundle, - admitted_isolation_backend: Option, - main_exit_marker: Option, -) -> Result { - // Shared startup retains policy and networking state; box it to keep callers' futures small. - Box::pin(run_sandbox_with_backend( - &backend_setup::OpenShellBackendSetup, - SandboxRunConfig { - command, - workdir, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id, - sandbox, - openshell_endpoint, - policy_rules, - policy_data, - ssh_socket_path, - health_socket_path, - health_port, - ocsf_enabled, - ocsf_schema_version, - upstream_proxy_args, - additional_ca_bundle, - backend_descriptor, - auth_bundle, - gateway_credentials: None, - admitted_isolation_backend, - main_exit_marker, - }, - )) - .await -} - -/// Trusted inputs for one supervisor launch. -pub struct SandboxRunConfig { - pub command: Vec, - pub workdir: Option, - pub timeout_secs: u64, - pub interactive: bool, - pub await_main_process_attachment: bool, - pub sandbox_id: Option, - pub sandbox: Option, - pub openshell_endpoint: Option, - pub policy_rules: Option, - pub policy_data: Option, - pub ssh_socket_path: Option, - pub health_socket_path: Option, - pub health_port: Option, - pub ocsf_enabled: Arc, - pub ocsf_schema_version: Arc>, - pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, - pub additional_ca_bundle: Option, - pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, - pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, - /// Credentials issued from `auth_bundle` and shared with other gateway - /// clients of this process, such as log push. Created when absent. - pub gateway_credentials: Option, - pub admitted_isolation_backend: Option, - pub main_exit_marker: Option, -} - /// Supervise the one sandbox instance this process was launched for. /// -/// The process-wide OCSF context, SIGTERM/SIGINT handling, readiness -/// endpoints, and the environment-provided launch inputs belong to the -/// dedicated process; [`run_sandbox_instance`] owns everything else. +/// This is the dedicated process wrapper. It sets the process-wide OCSF +/// context from `bindings`, publishes instance readiness on the health socket +/// and port, and stops the instance on SIGTERM or SIGINT; +/// [`run_sandbox_instance`] owns everything else. The caller loads `config` +/// from the process's arguments and environment and prepares `bindings` once, +/// sharing their gateway credentials and OCSF JSON settings with the +/// process's log push and logging layers. /// /// Trusted composition chooses the setup before shared admission, policy, and /// lifecycle handling. Payload contents never select a backend implementation. -#[allow(clippy::similar_names)] +/// +/// # Errors +/// +/// Returns an error if a readiness endpoint cannot be prepared or the +/// instance fails to start or supervise its workload. pub async fn run_sandbox_with_backend( backend_setup: &dyn backend_setup::BackendSetup, - config: SandboxRunConfig, + config: SandboxInstanceConfig, + bindings: SandboxInstanceBindings, + health_socket_path: Option, + health_port: Option, ) -> Result { - let SandboxRunConfig { - command, - workdir, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id, - sandbox, - openshell_endpoint, - policy_rules, - policy_data, - ssh_socket_path, - health_socket_path, - health_port, - ocsf_enabled, - ocsf_schema_version, - upstream_proxy_args, - additional_ca_bundle, - backend_descriptor, - auth_bundle, - gateway_credentials, - admitted_isolation_backend, - main_exit_marker, - } = config; - // Initialize the process-wide OCSF context early so that events emitted // during policy loading (filesystem config, validation) have a context. // Proxy IP/port use defaults here; the boundary mediation source carries // workload-side connection metadata. - if !openshell_ocsf::ctx::set_ctx(sandbox_instance_event_context( - sandbox_id.as_deref(), - sandbox.as_deref(), - std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - )) { + if !openshell_ocsf::ctx::set_ctx((*bindings.event_context).clone()) { debug!("OCSF context already initialized, keeping existing"); } @@ -813,39 +705,8 @@ pub async fn run_sandbox_with_backend( .collect::>>()?; let result = Box::pin(run_sandbox_instance( backend_setup, - SandboxInstanceConfig { - command, - workdir, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id, - sandbox, - openshell_endpoint, - policy_rules, - policy_data, - ssh_socket_path, - shared_ssh_socket: shared_ssh_socket_from_env(), - ocsf_json: openshell_ocsf::OcsfJsonSettings { - enabled: ocsf_enabled, - target_version: ocsf_schema_version, - }, - upstream_proxy_args, - additional_ca_bundle, - backend_descriptor, - auth_bundle, - gateway_credentials, - admitted_isolation_backend, - main_exit_marker, - trace_context: TraceContext::from_environment(), - provider_workload_api_socket: std::env::var_os( - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - ) - .filter(|path| !path.is_empty()) - .map(std::path::PathBuf::from), - proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), - denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), - }, + config, + bindings, SandboxInstanceControls { shutdown: Box::pin(wait_for_control_shutdown_signal()), readiness, @@ -902,13 +763,14 @@ impl TraceContext { } } -/// Trusted inputs owned by one supervised sandbox instance. +/// Trusted launch inputs of one supervised sandbox instance. /// /// A sandbox instance is one running generation of a stable Sandbox resource; /// gateway connections may reconnect within it without creating another. -/// Nothing in an instance reads process-wide sandbox state: identity, -/// credentials, policy inputs, logging settings, and file locations all come -/// from here, so one process can host several instances. +/// Nothing in an instance reads process-wide sandbox state: identity, policy +/// inputs, and file locations come from here, and live per-instance state +/// from its [`SandboxInstanceBindings`], so one process can host several +/// instances. pub struct SandboxInstanceConfig { pub command: Vec, pub workdir: Option, @@ -917,20 +779,18 @@ pub struct SandboxInstanceConfig { pub await_main_process_attachment: bool, pub sandbox_id: Option, pub sandbox: Option, + /// Workload image recorded in this instance's OCSF events. + pub container_image: String, pub openshell_endpoint: Option, pub policy_rules: Option, pub policy_data: Option, pub ssh_socket_path: Option, /// Grant the SSH socket to the supervisor's group rather than its owner. pub shared_ssh_socket: bool, - /// OCSF JSON output settings this instance's gateway configuration controls. - pub ocsf_json: openshell_ocsf::OcsfJsonSettings, pub upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, pub additional_ca_bundle: Option, pub backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, - /// Credentials issued from `auth_bundle`; created from it when absent. - pub gateway_credentials: Option, pub admitted_isolation_backend: Option, pub main_exit_marker: Option, /// Parent of the startup span. @@ -938,10 +798,59 @@ pub struct SandboxInstanceConfig { /// SPIFFE Workload API socket for provider token grants; absent fails closed. pub provider_workload_api_socket: Option, pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, +} + +impl SandboxInstanceConfig { + /// OCSF identity of this sandbox instance. + #[must_use] + pub fn event_context(&self) -> EventContext { + sandbox_instance_event_context( + self.sandbox_id.as_deref(), + self.sandbox.as_deref(), + self.container_image.clone(), + ) + } +} + +/// Live state bound to one sandbox instance, prepared once by its owner. +/// +/// A dedicated supervisor shares these with its process-wide logging and log +/// push; a hosted instance binds them to every thread of its runtime. Every +/// gateway client of the instance, including log push and credential renewal, +/// uses the same `gateway_credentials`, so renewal by one keeps all current. +#[derive(Clone)] +pub struct SandboxInstanceBindings { + /// Gateway credentials issued from the instance's auth bundle. + pub gateway_credentials: openshell_core::grpc_client::GatewayCredentials, + pub event_context: Arc, + /// OCSF JSON output settings this instance's gateway configuration controls. + pub ocsf_json: openshell_ocsf::OcsfJsonSettings, + /// Log push destination of a hosted instance's threads. A dedicated + /// process pushes through its process-wide layer instead. + pub log_sink: Option, /// Source of `policy.local` recent denials. pub denial_log: openshell_supervisor_network::policy_local::DenialLogSource, } +impl SandboxInstanceBindings { + /// Bindings of a process dedicated to the instance `config` launches. Its + /// process-wide log push and shorthand log files serve the instance. + #[must_use] + pub fn dedicated( + config: &SandboxInstanceConfig, + gateway_credentials: openshell_core::grpc_client::GatewayCredentials, + ocsf_json: openshell_ocsf::OcsfJsonSettings, + ) -> Self { + Self { + gateway_credentials, + event_context: Arc::new(config.event_context()), + ocsf_json, + log_sink: None, + denial_log: openshell_supervisor_network::policy_local::DenialLogSource::default(), + } + } +} + /// How the owner of a sandbox instance stops it and observes its readiness. pub struct SandboxInstanceControls { /// Completes when the owner asks the instance to stop its workload. It is @@ -1011,6 +920,7 @@ impl Drop for SandboxInstanceReadiness { pub async fn run_sandbox_instance( backend_setup: &dyn backend_setup::BackendSetup, config: SandboxInstanceConfig, + bindings: SandboxInstanceBindings, controls: SandboxInstanceControls, ) -> Result { let SandboxInstanceConfig { @@ -1021,24 +931,30 @@ pub async fn run_sandbox_instance( await_main_process_attachment, sandbox_id, sandbox, + container_image: _, openshell_endpoint, policy_rules, policy_data, ssh_socket_path, shared_ssh_socket, - ocsf_json, upstream_proxy_args, additional_ca_bundle, backend_descriptor, auth_bundle, - gateway_credentials, admitted_isolation_backend, main_exit_marker, trace_context, provider_workload_api_socket, proxy_tls, - denial_log, } = config; + // The owner binds the event context and log sink to the instance's + // threads; the instance itself uses the rest. + let SandboxInstanceBindings { + gateway_credentials, + ocsf_json, + denial_log, + .. + } = bindings; let SandboxInstanceControls { shutdown: shutdown_requested, readiness, @@ -1071,10 +987,6 @@ pub async fn run_sandbox_instance( sandbox_id.as_deref(), &auth_bundle, )?; - let gateway_credentials = match gateway_credentials { - Some(credentials) => credentials, - None => openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&auth_bundle)?, - }; let sandbox_bearer = gateway_credentials.sandbox_bearer().ok_or_else(|| { miette::miette!("supervisor gateway credentials carry no Sandbox Protocol credential") })?; @@ -4435,29 +4347,44 @@ mod tests { ssh_host_private_key: None, ocsf_otlp_headers: None, }; - let error = run_sandbox( - vec!["true".to_string()], - None, - 0, - false, - false, - Some("sandbox-1".to_string()), - None, - None, - None, - None, - None, - None, - None, - Arc::new(AtomicBool::new(false)), - Arc::new(std::sync::Mutex::new(String::new())), - openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), - None, - openshell_isolation_interface::contract::BackendDescriptor { + let config = SandboxInstanceConfig { + command: vec!["true".to_string()], + workdir: None, + timeout_secs: 0, + interactive: false, + await_main_process_attachment: false, + sandbox_id: Some("sandbox-1".to_string()), + sandbox: None, + container_image: String::new(), + openshell_endpoint: None, + policy_rules: None, + policy_data: None, + ssh_socket_path: None, + shared_ssh_socket: false, + upstream_proxy_args: + openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor { backend_name: "in-process-test".to_string(), payload: b"in-process-only-launch-data".to_vec(), }, auth_bundle, + admitted_isolation_backend: None, + main_exit_marker: None, + trace_context: None, + provider_workload_api_socket: None, + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), + }; + let bindings = SandboxInstanceBindings::dedicated( + &config, + openshell_core::grpc_client::GatewayCredentials::from_auth_bundle(&config.auth_bundle) + .expect("credentials"), + openshell_ocsf::OcsfJsonSettings::default(), + ); + let error = run_sandbox_with_backend( + &backend_setup::OpenShellBackendSetup, + config, + bindings, None, None, ) diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index daafbc364b..b222c1c196 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -556,32 +556,50 @@ fn main() -> Result<()> { "isolation-backend role started without validated runtime inputs" )); }; + let config = openshell_supervisor::SandboxInstanceConfig { + command, + workdir, + timeout_secs: args.timeout, + interactive, + await_main_process_attachment, + sandbox_id: args.sandbox_id, + sandbox: args.sandbox, + container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + openshell_endpoint: args.openshell_endpoint, + policy_rules: args.policy_rules, + policy_data: args.policy_data, + ssh_socket_path: args.ssh_socket_path, + shared_ssh_socket: openshell_supervisor::shared_ssh_socket_from_env(), + upstream_proxy_args, + additional_ca_bundle: args.additional_ca_bundle, + backend_descriptor, + auth_bundle, + admitted_isolation_backend: Some(admitted_backend), + main_exit_marker: args.main_exit_marker, + trace_context: openshell_supervisor::TraceContext::from_environment(), + provider_workload_api_socket: std::env::var_os( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ) + .filter(|path| !path.is_empty()) + .map(PathBuf::from), + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + }; + // Log push already shares these credentials and the logging + // layers these OCSF JSON settings. + let bindings = openshell_supervisor::SandboxInstanceBindings::dedicated( + &config, + gateway_credentials, + openshell_ocsf::OcsfJsonSettings { + enabled: ocsf_enabled, + target_version: ocsf_schema_version, + }, + ); let result = Box::pin(openshell_supervisor::run_sandbox_with_backend( backend_setup.as_ref(), - openshell_supervisor::SandboxRunConfig { - command, - workdir, - timeout_secs: args.timeout, - interactive, - await_main_process_attachment, - sandbox_id: args.sandbox_id, - sandbox: args.sandbox, - openshell_endpoint: args.openshell_endpoint, - policy_rules: args.policy_rules, - policy_data: args.policy_data, - ssh_socket_path: args.ssh_socket_path, - health_socket_path: args.health_socket_path, - health_port: args.health_port, - ocsf_enabled, - ocsf_schema_version, - upstream_proxy_args, - additional_ca_bundle: args.additional_ca_bundle, - backend_descriptor, - auth_bundle, - gateway_credentials: Some(gateway_credentials), - admitted_isolation_backend: Some(admitted_backend), - main_exit_marker: args.main_exit_marker, - }, + config, + bindings, + args.health_socket_path, + args.health_port, )) .await; supervisor_exit(result) diff --git a/crates/openshell-supervisor/src/sandbox_instance_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs index 4511b40176..65a8b27534 100644 --- a/crates/openshell-supervisor/src/sandbox_instance_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -9,13 +9,13 @@ //! creating another. //! //! Each instance runs on its own Tokio runtime. Every thread that runtime -//! starts, and the thread driving it, is bound to the instance's OCSF context, -//! OCSF JSON settings, log sink, and denial log, so tasks, callbacks, -//! reconnects, and credential renewal that the instance spawns inherit its -//! identity. Unbound threads carry no sandbox identity and push no logs. -//! Gateway credentials are never bound to threads; each instance passes its -//! own explicitly. Stopping an instance shuts down only its runtime and -//! revokes its gateway credentials. +//! starts, and the thread driving it, is bound to the instance's +//! [`SandboxInstanceBindings`]: OCSF context, OCSF JSON settings, log sink, and +//! denial log, so tasks, callbacks, reconnects, and credential renewal that the +//! instance spawns inherit its identity. Unbound threads carry no sandbox +//! identity and push no logs. Gateway credentials are never bound to threads; +//! each instance passes its own explicitly. Stopping an instance shuts down +//! only its runtime and revokes its gateway credentials. use std::collections::HashMap; use std::future::Future; @@ -26,41 +26,34 @@ use std::time::Duration; use miette::Result; use openshell_core::grpc_client::GatewayCredentials; -use openshell_ocsf::{EventContext, OcsfJsonSettings}; -use openshell_supervisor_network::policy_local::SandboxInstanceDenialLog; +use openshell_ocsf::OcsfJsonSettings; +use openshell_supervisor_network::policy_local::{DenialLogSource, SandboxInstanceDenialLog}; use openshell_supervisor_process::log_push::LogSink; use tokio::sync::watch; -use crate::SandboxInstanceControls; +use crate::{SandboxInstanceBindings, SandboxInstanceControls}; /// Worker threads in each sandbox instance runtime. const INSTANCE_WORKER_THREADS: usize = 2; /// Bound on joining an instance thread after its runtime is told to stop. const RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); -/// Per-instance values bound to every thread of the instance's runtime. -#[derive(Clone)] -pub struct SandboxInstanceBinding { - pub event_context: Arc, - pub ocsf_json: OcsfJsonSettings, - pub log_sink: Option, - pub denial_log: SandboxInstanceDenialLog, +/// Bind the calling thread to one instance's bindings. +fn bind_thread(bindings: &SandboxInstanceBindings) { + openshell_ocsf::ctx::bind_thread_ctx(Some(bindings.event_context.clone())); + openshell_ocsf::bind_thread_json_settings(Some(bindings.ocsf_json.clone())); + openshell_supervisor_process::log_push::bind_thread_sink(bindings.log_sink.clone()); + bind_thread_denial_log(match &bindings.denial_log { + DenialLogSource::SandboxInstance(log) => Some(log.clone()), + DenialLogSource::ShorthandFiles(_) => None, + }); } -impl SandboxInstanceBinding { - fn bind(&self) { - openshell_ocsf::ctx::bind_thread_ctx(Some(self.event_context.clone())); - openshell_ocsf::bind_thread_json_settings(Some(self.ocsf_json.clone())); - openshell_supervisor_process::log_push::bind_thread_sink(self.log_sink.clone()); - bind_thread_denial_log(Some(self.denial_log.clone())); - } - - fn unbind() { - openshell_ocsf::ctx::bind_thread_ctx(None); - openshell_ocsf::bind_thread_json_settings(None); - openshell_supervisor_process::log_push::bind_thread_sink(None); - bind_thread_denial_log(None); - } +fn unbind_thread() { + openshell_ocsf::ctx::bind_thread_ctx(None); + openshell_ocsf::bind_thread_json_settings(None); + openshell_supervisor_process::log_push::bind_thread_sink(None); + bind_thread_denial_log(None); } std::thread_local! { @@ -196,10 +189,10 @@ impl SandboxInstanceHost { Self::default() } - /// Start `body` on a new runtime bound to `binding`. + /// Start `body` on a new runtime bound to `bindings`. /// - /// `credentials`, when present, are revoked once the instance's runtime - /// has stopped, so nothing can reuse them after the instance ends. + /// The bindings' gateway credentials are revoked once the instance's + /// runtime has stopped, so nothing can reuse them after the instance ends. /// /// # Errors /// @@ -208,8 +201,7 @@ impl SandboxInstanceHost { pub fn start( &self, key: &str, - binding: SandboxInstanceBinding, - credentials: Option, + bindings: SandboxInstanceBindings, body: SandboxInstanceBody, ) -> std::result::Result<(), SandboxInstanceHostError> { let mut instances = lock(&self.instances); @@ -224,8 +216,7 @@ impl SandboxInstanceHost { .name("openshell-instance".to_string()) .spawn(move || { run_hosted_sandbox_instance( - &binding, - credentials, + &bindings, body, SandboxInstanceSignals { stop_requested, @@ -381,7 +372,7 @@ struct SandboxInstanceSignals { /// Reports a hosted sandbox instance's end and revokes its credentials, /// including when the instance thread unwinds. struct SandboxInstanceExit { - credentials: Option, + credentials: GatewayCredentials, phase: watch::Sender, ready: watch::Sender, outcome: Option, @@ -389,9 +380,7 @@ struct SandboxInstanceExit { impl Drop for SandboxInstanceExit { fn drop(&mut self) { - if let Some(credentials) = self.credentials.take() { - credentials.revoke(); - } + self.credentials.revoke(); self.ready.send_replace(false); let outcome = self.outcome.take().unwrap_or_else(|| { SandboxInstancePhase::Failed( @@ -399,13 +388,12 @@ impl Drop for SandboxInstanceExit { ) }); self.phase.send_replace(outcome); - SandboxInstanceBinding::unbind(); + unbind_thread(); } } fn run_hosted_sandbox_instance( - binding: &SandboxInstanceBinding, - credentials: Option, + bindings: &SandboxInstanceBindings, body: SandboxInstanceBody, signals: SandboxInstanceSignals, ) { @@ -416,19 +404,19 @@ fn run_hosted_sandbox_instance( ready, } = signals; let mut exit = SandboxInstanceExit { - credentials, + credentials: bindings.gateway_credentials.clone(), phase: phase.clone(), ready: ready.clone(), outcome: None, }; - binding.bind(); - let thread_binding = binding.clone(); + bind_thread(bindings); + let thread_bindings = bindings.clone(); let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(INSTANCE_WORKER_THREADS) .thread_name("openshell-instance-worker") .enable_all() - .on_thread_start(move || thread_binding.bind()) - .on_thread_stop(SandboxInstanceBinding::unbind) + .on_thread_start(move || bind_thread(&thread_bindings)) + .on_thread_stop(unbind_thread) .build(); let outcome = match runtime { Ok(runtime) => { @@ -493,8 +481,6 @@ pub struct SandboxInstanceSpec { pub key: String, pub backend_setup: Box, pub config: crate::SandboxInstanceConfig, - /// Workload image recorded in this instance's OCSF events. - pub container_image: String, } impl SandboxInstanceHost { @@ -509,9 +495,7 @@ impl SandboxInstanceHost { key, backend_setup, mut config, - container_image, } = spec; - let credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; // Instances must never share proxy trust files; give one a private // directory that lives exactly as long as the instance. let private_tls_dir = if config.proxy_tls.directory.is_none() { @@ -521,31 +505,30 @@ impl SandboxInstanceHost { } else { None }; - let gateway = config.openshell_endpoint.as_deref().map(|endpoint| { - openshell_core::grpc_client::GatewayEndpoint::new(endpoint, credentials.clone()) - }); + // Log push, supervision, and renewal share these credentials. + let gateway_credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; + let log_push = config + .sandbox_id + .clone() + .zip(config.openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new( + endpoint, + gateway_credentials.clone(), + ) + })); let (log_tx, log_rx) = tokio::sync::mpsc::channel( openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES, ); - let log_push = config.sandbox_id.clone().zip(gateway); - let binding = SandboxInstanceBinding { - event_context: Arc::new(crate::sandbox_instance_event_context( - config.sandbox_id.as_deref(), - config.sandbox.as_deref(), - container_image, - )), + let bindings = SandboxInstanceBindings { + gateway_credentials, + event_context: Arc::new(config.event_context()), ocsf_json: OcsfJsonSettings::default(), log_sink: log_push .as_ref() .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), - denial_log: SandboxInstanceDenialLog::default(), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), }; - config.gateway_credentials = Some(credentials.clone()); - config.ocsf_json = binding.ocsf_json.clone(); - config.denial_log = - openshell_supervisor_network::policy_local::DenialLogSource::SandboxInstance( - binding.denial_log.clone(), - ); + let instance_bindings = bindings.clone(); let body: SandboxInstanceBody = Box::new(move |controls| { Box::pin(async move { let _private_tls_dir = private_tls_dir; @@ -557,12 +540,13 @@ impl SandboxInstanceHost { Box::pin(crate::run_sandbox_instance( backend_setup.as_ref(), config, + instance_bindings, controls, )) .await }) }); - self.start(&key, binding, Some(credentials), body) + self.start(&key, bindings, body) .map_err(|error| miette::miette!("{error}")) } } @@ -654,7 +638,8 @@ mod tests { GatewayCredentials::from_gateway_token(&format!("token-{sandbox}")).unwrap(); let gateway = GatewayEndpoint::new(gateway_url, credentials.clone()); let (log_tx, logs) = mpsc::channel(16); - let binding = SandboxInstanceBinding { + let bindings = SandboxInstanceBindings { + gateway_credentials: credentials.clone(), event_context: Arc::new(crate::sandbox_instance_event_context( Some(sandbox), Some(sandbox), @@ -662,7 +647,7 @@ mod tests { )), ocsf_json: OcsfJsonSettings::default(), log_sink: Some(LogSink::new(sandbox, log_tx)), - denial_log: SandboxInstanceDenialLog::default(), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), }; let (identity_tx, identity) = tokio::sync::oneshot::channel(); let (call_again, mut calls) = mpsc::channel(1); @@ -715,8 +700,7 @@ mod tests { } }) }); - host.start(sandbox, binding, Some(credentials.clone()), body) - .unwrap(); + host.start(sandbox, bindings, body).unwrap(); Instance { identity, logs, @@ -738,8 +722,9 @@ mod tests { .expect("instance reaches supervision"); } - fn unbound() -> SandboxInstanceBinding { - SandboxInstanceBinding { + fn unbound() -> SandboxInstanceBindings { + SandboxInstanceBindings { + gateway_credentials: GatewayCredentials::from_gateway_token("unbound-token").unwrap(), event_context: Arc::new(crate::sandbox_instance_event_context( None, None, @@ -747,7 +732,7 @@ mod tests { )), ocsf_json: OcsfJsonSettings::default(), log_sink: None, - denial_log: SandboxInstanceDenialLog::default(), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), } } @@ -763,7 +748,7 @@ mod tests { Ok(0) }) }); - host.start("generation", unbound(), None, body).unwrap(); + host.start("generation", unbound(), body).unwrap(); wait_supervising(&host, "generation").await; let removal = tokio::spawn({ let host = host.clone(); @@ -780,7 +765,7 @@ mod tests { .expect("instance observes the stop request"); let replacement: SandboxInstanceBody = Box::new(|_| Box::pin(async { Ok(0) })); assert_eq!( - host.start("generation", unbound(), None, replacement), + host.start("generation", unbound(), replacement), Err(SandboxInstanceHostError::Duplicate) ); let stopped = removal.await.unwrap().expect("instance was hosted"); @@ -795,8 +780,15 @@ mod tests { panic!("hosted sandbox instance failure"); }) }); - host.start("panicking", unbound(), Some(credentials.clone()), panicking) - .unwrap(); + host.start( + "panicking", + SandboxInstanceBindings { + gateway_credentials: credentials.clone(), + ..unbound() + }, + panicking, + ) + .unwrap(); let status = tokio::time::timeout(Duration::from_secs(10), async { loop { if let Some(status) = host.status("panicking") @@ -841,7 +833,7 @@ mod tests { }) }) }; - host.start("generation", unbound(), None, winding_down(3)) + host.start("generation", unbound(), winding_down(3)) .unwrap(); wait_supervising(&host, "generation").await; // A caller that gives up, such as an expired RPC, strands nothing. @@ -864,7 +856,7 @@ mod tests { assert!(host.status("generation").is_none()); // The key is free again, and the earlier removal cannot touch its new owner. - host.start("generation", unbound(), None, winding_down(4)) + host.start("generation", unbound(), winding_down(4)) .unwrap(); wait_supervising(&host, "generation").await; tokio::time::sleep(Duration::from_millis(50)).await; @@ -890,17 +882,7 @@ mod tests { assert!(matches!( host.start( "sandbox-alpha", - SandboxInstanceBinding { - event_context: Arc::new(crate::sandbox_instance_event_context( - None, - None, - String::new() - )), - ocsf_json: OcsfJsonSettings::default(), - log_sink: None, - denial_log: SandboxInstanceDenialLog::default(), - }, - None, + unbound(), Box::new(|_| Box::pin(async { Ok(0) })), ), Err(SandboxInstanceHostError::Duplicate) From 0aafe59ac0f01172991893ccc54a599ed171e77d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 21 Oct 2026 14:41:15 -0700 Subject: [PATCH 20/25] refactor(supervisor): use session-control wire types for instance status and trace context Drop the supervisor's duplicate SandboxInstancePhase, SandboxInstanceStatus, and TraceContext families. The sandbox instance host reports SandboxInstanceStatusReport directly, with its key as the route, so the shared host no longer converts or re-keys status, lists, or removal results. SandboxInstanceConfig carries RegistrationTraceContext and the dedicated process reads it through trace_context_from_environment. No wire definitions change. Signed-off-by: Drew Newberry --- crates/openshell-supervisor/src/lib.rs | 34 ++-- crates/openshell-supervisor/src/main.rs | 2 +- .../src/sandbox_instance_host.rs | 160 ++++++++---------- .../openshell-supervisor/src/shared_host.rs | 46 +---- 4 files changed, 96 insertions(+), 146 deletions(-) diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index be119584e3..a13b0d5af5 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -38,6 +38,7 @@ use std::time::Duration; use tracing::{Instrument as _, debug, info, warn}; use openshell_core::PolicyValidationFailureMode; +use openshell_sandbox_backend::session_control::RegistrationTraceContext; use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, @@ -741,27 +742,18 @@ pub fn sandbox_instance_event_context( } } -/// W3C trace context of the operation that launched a sandbox generation. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TraceContext { - pub traceparent: String, - pub tracestate: Option, -} - -impl TraceContext { - /// The context a driver passed to a dedicated supervisor process. - #[must_use] - pub fn from_environment() -> Option { - let traceparent = std::env::var(openshell_otel::TRACEPARENT_ENV) +/// The trace context a driver passed to a dedicated supervisor process. +#[must_use] +pub fn trace_context_from_environment() -> Option { + let traceparent = std::env::var(openshell_otel::TRACEPARENT_ENV) + .ok() + .filter(|value| !value.is_empty())?; + Some(RegistrationTraceContext { + traceparent, + tracestate: std::env::var(openshell_otel::TRACESTATE_ENV) .ok() - .filter(|value| !value.is_empty())?; - Some(Self { - traceparent, - tracestate: std::env::var(openshell_otel::TRACESTATE_ENV) - .ok() - .filter(|value| !value.is_empty()), - }) - } + .filter(|value| !value.is_empty()), + }) } /// Trusted launch inputs of one supervised sandbox instance. @@ -795,7 +787,7 @@ pub struct SandboxInstanceConfig { pub admitted_isolation_backend: Option, pub main_exit_marker: Option, /// Parent of the startup span. - pub trace_context: Option, + pub trace_context: Option, /// SPIFFE Workload API socket for provider token grants; absent fails closed. pub provider_workload_api_socket: Option, pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 37abe83a1e..7cee04d542 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -586,7 +586,7 @@ fn main() -> Result<()> { auth_bundle, admitted_isolation_backend: Some(admitted_backend), main_exit_marker: args.main_exit_marker, - trace_context: openshell_supervisor::TraceContext::from_environment(), + trace_context: openshell_supervisor::trace_context_from_environment(), provider_workload_api_socket: std::env::var_os( openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, ) diff --git a/crates/openshell-supervisor/src/sandbox_instance_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs index 3c01aa7002..508c32eba2 100644 --- a/crates/openshell-supervisor/src/sandbox_instance_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -28,6 +28,9 @@ use std::time::Duration; use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use openshell_core::grpc_client::GatewayCredentials; use openshell_ocsf::OcsfJsonSettings; +use openshell_sandbox_backend::session_control::{ + SandboxInstancePhaseReport, SandboxInstanceStatusReport, +}; use openshell_supervisor_network::policy_local::{DenialLogSource, SandboxInstanceDenialLog}; use openshell_supervisor_process::log_push::LogSink; use tokio::sync::watch; @@ -91,38 +94,6 @@ impl std::io::Write for SandboxInstanceDenialWriter { } } -/// Observable lifecycle of one hosted sandbox instance. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum SandboxInstancePhase { - /// Startup has not yet produced an accepted gateway session. - Starting, - /// The workload started and the gateway accepted supervision at least once. - Supervising, - /// The owner asked the instance to stop. - Stopping, - /// Supervision ended with the workload's exit status. - Exited(i32), - /// Startup or supervision failed; the workload is no longer supervised. - Failed(String), -} - -impl SandboxInstancePhase { - #[must_use] - pub fn is_terminal(&self) -> bool { - matches!(self, Self::Exited(_) | Self::Failed(_)) - } -} - -/// Current state of one hosted sandbox instance. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SandboxInstanceStatus { - /// Sandbox the instance was admitted for. - pub sandbox_id: String, - pub phase: SandboxInstancePhase, - /// Gateway supervision is currently accepted for this instance. - pub ready: bool, -} - /// Body of a hosted sandbox instance, constructed on its own runtime. pub type SandboxInstanceBody = Box< dyn FnOnce(SandboxInstanceControls) -> Pin>>> @@ -165,7 +136,7 @@ pub enum SandboxInstanceHostError { } /// Final status of one removal, shared by every remover. -type RemovalReport = watch::Receiver>; +type RemovalReport = watch::Receiver>; struct HostedSandboxInstance { /// Distinguishes this instance from a later one that reuses its key. @@ -174,7 +145,7 @@ struct HostedSandboxInstance { sandbox_id: String, stop: watch::Sender, abort: watch::Sender, - phase: watch::Receiver, + phase: watch::Receiver, ready: watch::Receiver, thread: Option>, /// The removal in progress, or the report of one that could not delete @@ -183,7 +154,8 @@ struct HostedSandboxInstance { } impl HostedSandboxInstance { - fn status(&self) -> SandboxInstanceStatus { + /// Status of the instance hosted under `key`, which is its route. + fn status(&self, key: &str) -> SandboxInstanceStatusReport { if let Some(reported) = self .removal .as_ref() @@ -195,15 +167,17 @@ impl HostedSandboxInstance { // The runner always reports a terminal phase, even while unwinding; // a closed channel without one still means supervision has ended. if !phase.is_terminal() && self.phase.has_changed().is_err() { - return SandboxInstanceStatus { + return SandboxInstanceStatusReport { + route: key.to_string(), sandbox_id: self.sandbox_id.clone(), - phase: SandboxInstancePhase::Failed( - "sandbox instance thread ended unexpectedly".to_string(), - ), + phase: SandboxInstancePhaseReport::Failed { + error: "sandbox instance thread ended unexpectedly".to_string(), + }, ready: false, }; } - SandboxInstanceStatus { + SandboxInstanceStatusReport { + route: key.to_string(), sandbox_id: self.sandbox_id.clone(), phase, ready: *self.ready.borrow(), @@ -320,20 +294,20 @@ impl SandboxInstanceHost { } /// Status of a hosted sandbox instance, including one that ended but has - /// not been removed. + /// not been removed. The report's route is the instance's key. #[must_use] - pub fn status(&self, key: &str) -> Option { + pub fn status(&self, key: &str) -> Option { lock(&self.instances) .get(key) - .map(HostedSandboxInstance::status) + .map(|instance| instance.status(key)) } - /// Key and status of every hosted sandbox instance. + /// Status of every hosted sandbox instance. #[must_use] - pub fn list(&self) -> Vec<(String, SandboxInstanceStatus)> { + pub fn list(&self) -> Vec { lock(&self.instances) .iter() - .map(|(key, instance)| (key.clone(), instance.status())) + .map(|(key, instance)| instance.status(key)) .collect() } @@ -349,7 +323,7 @@ impl SandboxInstanceHost { /// the directory cannot be deleted, the key stays reserved and the next /// removal tries again. Returns the final status, or `None` when `key` is /// not hosted. - pub async fn remove(&self, key: &str, grace: Duration) -> Option { + pub async fn remove(&self, key: &str, grace: Duration) -> Option { let (mut removed, sandbox_id) = self.begin_removal(key, grace)?; // The removal task ends without a report only with its runtime. let reported = removed @@ -357,11 +331,12 @@ impl SandboxInstanceHost { .await .ok() .and_then(|status| status.clone()); - Some(reported.unwrap_or_else(|| SandboxInstanceStatus { + Some(reported.unwrap_or_else(|| SandboxInstanceStatusReport { + route: key.to_string(), sandbox_id, - phase: SandboxInstancePhase::Failed( - "sandbox instance removal did not finish".to_string(), - ), + phase: SandboxInstancePhaseReport::Failed { + error: "sandbox instance removal did not finish".to_string(), + }, ready: false, })) } @@ -382,7 +357,7 @@ impl SandboxInstanceHost { return Some((removed.clone(), instance.sandbox_id.clone())); } instance.stop.send_replace(true); - if *instance.phase.borrow() == SandboxInstancePhase::Starting { + if *instance.phase.borrow() == SandboxInstancePhaseReport::Starting { instance.abort.send_replace(true); } let (report, removed) = watch::channel(None); @@ -409,18 +384,18 @@ struct Removal { instances: Instances, key: String, serial: u64, - phase: watch::Receiver, + phase: watch::Receiver, thread: Option>, dir: PathBuf, grace: Duration, - report: watch::Sender>, + report: watch::Sender>, } impl Removal { async fn run(mut self) { let finished = tokio::time::timeout( self.grace, - self.phase.wait_for(SandboxInstancePhase::is_terminal), + self.phase.wait_for(SandboxInstancePhaseReport::is_terminal), ) .await .is_ok(); @@ -448,7 +423,7 @@ impl Removal { else { return; }; - let mut status = instance.status(); + let mut status = instance.status(&self.key); status.ready = false; match cleanup { Ok(()) => { @@ -456,9 +431,11 @@ impl Removal { } Err(error) => { tracing::warn!(%error, "could not delete removed sandbox instance files; keeping its key reserved"); - status.phase = SandboxInstancePhase::Failed(format!( - "sandbox instance stopped, but its private files remain: {error}" - )); + status.phase = SandboxInstancePhaseReport::Failed { + error: format!( + "sandbox instance stopped, but its private files remain: {error}" + ), + }; } } self.report.send_replace(Some(status)); @@ -469,7 +446,7 @@ impl Removal { struct RunningInstance { stop: watch::Sender, abort: watch::Sender, - phase: watch::Receiver, + phase: watch::Receiver, ready: watch::Receiver, thread: std::thread::JoinHandle<()>, } @@ -480,7 +457,7 @@ fn spawn_instance( ) -> std::result::Result { let (stop, stop_requested) = watch::channel(false); let (abort, abort_requested) = watch::channel(false); - let (phase_tx, phase) = watch::channel(SandboxInstancePhase::Starting); + let (phase_tx, phase) = watch::channel(SandboxInstancePhaseReport::Starting); let (ready_tx, ready) = watch::channel(false); let credentials = bindings.gateway_credentials.clone(); let thread = std::thread::Builder::new() @@ -513,7 +490,7 @@ fn spawn_instance( struct SandboxInstanceSignals { stop_requested: watch::Receiver, abort_requested: watch::Receiver, - phase: watch::Sender, + phase: watch::Sender, ready: watch::Sender, } @@ -521,20 +498,21 @@ struct SandboxInstanceSignals { /// including when the instance thread unwinds. struct SandboxInstanceExit { credentials: GatewayCredentials, - phase: watch::Sender, + phase: watch::Sender, ready: watch::Sender, - outcome: Option, + outcome: Option, } impl Drop for SandboxInstanceExit { fn drop(&mut self) { self.credentials.revoke(); self.ready.send_replace(false); - let outcome = self.outcome.take().unwrap_or_else(|| { - SandboxInstancePhase::Failed( - "sandbox instance thread ended without reporting a result".to_string(), - ) - }); + let outcome = self + .outcome + .take() + .unwrap_or_else(|| SandboxInstancePhaseReport::Failed { + error: "sandbox instance thread ended without reporting a result".to_string(), + }); self.phase.send_replace(outcome); unbind_thread(); } @@ -579,9 +557,10 @@ fn run_hosted_sandbox_instance( mirror_ready.send_replace(is_ready); if is_ready { mirror_phase.send_if_modified(|current| { - let promote = *current == SandboxInstancePhase::Starting; + let promote = + *current == SandboxInstancePhaseReport::Starting; if promote { - *current = SandboxInstancePhase::Supervising; + *current = SandboxInstancePhaseReport::Supervising; } promote }); @@ -596,7 +575,7 @@ fn run_hosted_sandbox_instance( let stop_phase = phase.clone(); let shutdown = Box::pin(async move { let _ = stop.wait_for(|requested| *requested).await; - stop_phase.send_replace(SandboxInstancePhase::Stopping); + stop_phase.send_replace(SandboxInstancePhaseReport::Stopping); }); let instance = body(SandboxInstanceControls { shutdown, @@ -618,8 +597,10 @@ fn run_hosted_sandbox_instance( Err(error) => Err(miette::miette!("build sandbox instance runtime: {error}")), }; exit.outcome = Some(match outcome { - Ok(code) => SandboxInstancePhase::Exited(code), - Err(error) => SandboxInstancePhase::Failed(error.to_string()), + Ok(exit_code) => SandboxInstancePhaseReport::Exited { exit_code }, + Err(error) => SandboxInstancePhaseReport::Failed { + error: error.to_string(), + }, }); } @@ -891,7 +872,7 @@ mod tests { async fn wait_supervising(host: &SandboxInstanceHost, key: &str) { tokio::time::timeout(Duration::from_secs(10), async { while host.status(key).map(|status| status.phase) - != Some(SandboxInstancePhase::Supervising) + != Some(SandboxInstancePhaseReport::Supervising) { tokio::time::sleep(Duration::from_millis(10)).await; } @@ -995,7 +976,7 @@ mod tests { ); tokio::time::timeout(Duration::from_secs(10), async { while host.status("generation").map(|status| status.phase) - != Some(SandboxInstancePhase::Stopping) + != Some(SandboxInstancePhaseReport::Stopping) { tokio::time::sleep(Duration::from_millis(5)).await; } @@ -1011,7 +992,10 @@ mod tests { for remover in removers { let status = remover.await.unwrap().expect("instance was hosted"); assert_eq!(status.sandbox_id, "sandbox-1"); - assert_eq!(status.phase, SandboxInstancePhase::Exited(3)); + assert_eq!( + status.phase, + SandboxInstancePhaseReport::Exited { exit_code: 3 } + ); } assert!(host.status("generation").is_none()); assert!(!dir.exists()); @@ -1022,7 +1006,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; assert_eq!( host.status("generation").map(|status| status.phase), - Some(SandboxInstancePhase::Supervising) + Some(SandboxInstancePhaseReport::Supervising) ); assert_eq!(std::fs::read(dir.join("launch")).unwrap(), [4]); @@ -1039,7 +1023,7 @@ mod tests { .await .unwrap(); assert!( - matches!(&status.phase, SandboxInstancePhase::Failed(error) if error.contains("private files remain")), + matches!(&status.phase, SandboxInstancePhaseReport::Failed { error } if error.contains("private files remain")), "{status:?}" ); assert_eq!(host.status("generation"), Some(status)); @@ -1049,7 +1033,7 @@ mod tests { let status = host.remove("generation", Duration::from_secs(10)).await; assert_eq!( status.map(|status| status.phase), - Some(SandboxInstancePhase::Exited(4)) + Some(SandboxInstancePhaseReport::Exited { exit_code: 4 }) ); assert!(!dir.exists()); host.remove_all(Duration::from_secs(10)).await; @@ -1093,9 +1077,12 @@ mod tests { .expect("panicking instance reports an end"); assert_eq!( status, - SandboxInstanceStatus { + SandboxInstanceStatusReport { + route: "panicking".to_string(), sandbox_id: "sandbox-panicking".to_string(), - phase: SandboxInstancePhase::Failed("sandbox instance panicked".to_string()), + phase: SandboxInstancePhaseReport::Failed { + error: "sandbox instance panicked".to_string(), + }, ready: false, } ); @@ -1156,7 +1143,10 @@ mod tests { .remove("sandbox-alpha", Duration::from_secs(10)) .await .expect("alpha was hosted"); - assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); + assert_eq!( + stopped.phase, + SandboxInstancePhaseReport::Exited { exit_code: 0 } + ); assert!(!stopped.ready); assert!(host.status("sandbox-alpha").is_none()); assert_eq!(alpha.alive.recv().await, None, "alpha's tasks stopped"); @@ -1180,7 +1170,7 @@ mod tests { ); assert_eq!( host.status("sandbox-beta").map(|status| status.phase), - Some(SandboxInstancePhase::Supervising) + Some(SandboxInstancePhaseReport::Supervising) ); assert!( beta.alive @@ -1191,7 +1181,7 @@ mod tests { let stopped = host.remove("sandbox-beta", Duration::from_secs(10)).await; assert_eq!( stopped.map(|status| status.phase), - Some(SandboxInstancePhase::Exited(0)) + Some(SandboxInstancePhaseReport::Exited { exit_code: 0 }) ); } } diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index ba060b3481..cca0964014 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -24,13 +24,13 @@ use http_body_util::{BodyExt as _, Full, Limited}; use miette::Result; use openshell_sandbox_backend::runtime_routed::RuntimeRouter; use openshell_sandbox_backend::session_control::{ - MAX_REGISTRATION_BYTES, SESSIONS_PATH, SandboxInstanceList, SandboxInstancePhaseReport, - SandboxInstanceRegistration, SandboxInstanceStatusReport, + MAX_REGISTRATION_BYTES, SESSIONS_PATH, SandboxInstanceList, SandboxInstanceRegistration, + SandboxInstanceStatusReport, }; use crate::sandbox_instance_host::{ SandboxInstanceAdmission, SandboxInstanceAdmitted, SandboxInstanceBody, SandboxInstanceHost, - SandboxInstanceHostError, SandboxInstancePhase, SandboxInstanceStatus, + SandboxInstanceHostError, }; /// Concurrent session-control connections; the only client is the gateway, @@ -128,30 +128,20 @@ impl SharedHost { /// Status of one registered sandbox instance. #[must_use] pub fn status(&self, route: &str) -> Option { - self.host - .status(route) - .map(|status| status_report(route, status)) + self.host.status(route) } /// Status of every registered sandbox instance. #[must_use] pub fn list(&self) -> SandboxInstanceList { SandboxInstanceList { - sessions: self - .host - .list() - .into_iter() - .map(|(route, status)| status_report(&route, status)) - .collect(), + sessions: self.host.list(), } } /// Stop one sandbox instance, delete its private files, and forget it. pub async fn remove(&self, route: &str) -> Option { - self.host - .remove(route, self.removal_grace) - .await - .map(|status| status_report(route, status)) + self.host.remove(route, self.removal_grace).await } /// Stop every sandbox instance, for process shutdown. @@ -216,12 +206,7 @@ impl SharedHost { auth_bundle: registration.auth, admitted_isolation_backend: Some(openshell_sandbox_backend::BACKEND_NAME.to_string()), main_exit_marker: None, - trace_context: registration - .trace_context - .map(|context| crate::TraceContext { - traceparent: context.traceparent, - tracestate: context.tracestate, - }), + trace_context: registration.trace_context, // One process cannot hold a per-sandbox workload identity, so // providers that need one fail closed. provider_workload_api_socket: None, @@ -251,23 +236,6 @@ fn registration_digest( openshell_crypto::sha256(&encoded).map_err(|error| ControlError::Unavailable(error.to_string())) } -fn status_report(route: &str, status: SandboxInstanceStatus) -> SandboxInstanceStatusReport { - SandboxInstanceStatusReport { - route: route.to_string(), - sandbox_id: status.sandbox_id, - phase: match status.phase { - SandboxInstancePhase::Starting => SandboxInstancePhaseReport::Starting, - SandboxInstancePhase::Supervising => SandboxInstancePhaseReport::Supervising, - SandboxInstancePhase::Stopping => SandboxInstancePhaseReport::Stopping, - SandboxInstancePhase::Exited(exit_code) => { - SandboxInstancePhaseReport::Exited { exit_code } - } - SandboxInstancePhase::Failed(error) => SandboxInstancePhaseReport::Failed { error }, - }, - ready: status.ready, - } -} - #[cfg(unix)] fn write_private_file(path: &Path, contents: &[u8]) -> Result<()> { use miette::{IntoDiagnostic as _, WrapErr as _}; From 34d2ee006b19798cf4bfc1f2a5b44ee2fc2764e7 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 21 Oct 2026 18:07:16 -0700 Subject: [PATCH 21/25] fix(supervisor): update Windows host proxy test config Signed-off-by: Drew Newberry --- crates/openshell-supervisor-network/src/host.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-supervisor-network/src/host.rs b/crates/openshell-supervisor-network/src/host.rs index 5f111acb21..45b21127c4 100644 --- a/crates/openshell-supervisor-network/src/host.rs +++ b/crates/openshell-supervisor-network/src/host.rs @@ -258,7 +258,7 @@ mod tests { binary_path, sandbox_id: Some("sandbox-123".to_string()), sandbox_name: Some("agent-box".to_string()), - openshell_endpoint: None, + gateway: None, provider_credentials: None, agent_proposals: AgentProposals::new(true), denial_tx: None, From 58c4c16c12868d44724c4cb84471cfd266960f28 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 21 Oct 2026 21:59:22 -0700 Subject: [PATCH 22/25] refactor(supervisor): assign shared sandbox instances over a supervisor-initiated gRPC stream Replace the private HTTP/JSON session-control API, which the gateway dialed on the shared supervisor, with a supervisor-initiated stream to the gateway. The gateway owns the desired set of sandbox instances; the shared supervisor attaches with a gateway-scoped host certificate, receives complete assignment snapshots with session-scoped revisions, reconciles its instance registry to each one, and reports what it observes. Losing the stream never stops an instance; only a later snapshot that omits it does. - Add the private openshell.supervisor.host.v1 protocol, served on the gateway listener but authorized only for the supervisor host certificate, excluded from SDK generation and reflection, and classified in the schema inventory. - Add SupervisorHostController in openshell-core: uninitialized until the driver derives the desired set once, compare-and-replace full resyncs, capacity bound, session fencing by connection epoch, revision-bounded acks, bounded sends. - Classify TLS peers per connection against the ordinary client CA and the supervisor host CA from one configuration snapshot; a chain matching neither or both is refused, and host connections never become users. - Persist each shared generation's launch material in its supervisor Secret so a restarted gateway rebuilds assignments from Sandbox resources, and withdraw instances on stop, delete, and suspend. - Issue the host identity with `generate-certs --client-only`; the gateway mounts only the CA. Drop the control port, server certificate, and gateway ingress rule from the chart; require gateway TLS for the topology. Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 23 +- Cargo.lock | 1 + crates/openshell-bootstrap/src/pki.rs | 27 +- crates/openshell-core/src/lib.rs | 1 + crates/openshell-core/src/proto/mod.rs | 5 + crates/openshell-core/src/supervisor_host.rs | 802 ++++++++++++++++++ crates/openshell-driver-kubernetes/README.md | 54 +- .../openshell-driver-kubernetes/src/config.rs | 34 +- .../openshell-driver-kubernetes/src/driver.rs | 327 ++++--- .../src/sandbox_runtime.rs | 31 + .../src/shared_supervisor.rs | 234 ++--- crates/openshell-gateway/src/lib.rs | 5 + crates/openshell-sandbox-backend/README.md | 10 +- crates/openshell-sandbox-backend/src/lib.rs | 2 +- .../src/sandbox_instance.rs | 356 ++++++++ .../src/session_control.rs | 285 ------- crates/openshell-server/src/auth/audit.rs | 8 +- .../src/auth/authenticator.rs | 3 + .../src/auth/descriptor_authz.rs | 3 +- crates/openshell-server/src/auth/guard.rs | 9 +- .../openshell-server/src/auth/method_authz.rs | 29 +- crates/openshell-server/src/auth/principal.rs | 12 + .../src/auth/workspace_authz.rs | 8 +- crates/openshell-server/src/certgen.rs | 118 +-- crates/openshell-server/src/compute/mod.rs | 23 + crates/openshell-server/src/grpc/mod.rs | 1 + crates/openshell-server/src/grpc/policy.rs | 4 +- crates/openshell-server/src/grpc/provider.rs | 3 +- .../src/grpc/provider_credentials_tests.rs | 1 + .../src/grpc/supervisor_host.rs | 66 ++ crates/openshell-server/src/grpc/workspace.rs | 4 +- crates/openshell-server/src/lib.rs | 76 +- crates/openshell-server/src/multiplex.rs | 218 ++++- crates/openshell-server/src/storage_proto.rs | 18 +- crates/openshell-server/src/tls.rs | 451 +++++++++- crates/openshell-supervisor/Cargo.toml | 11 +- crates/openshell-supervisor/README.md | 73 +- crates/openshell-supervisor/src/lib.rs | 2 +- crates/openshell-supervisor/src/main.rs | 10 +- .../src/sandbox_instance_host.rs | 52 +- crates/openshell-supervisor/src/shared.rs | 84 +- .../openshell-supervisor/src/shared_host.rs | 648 ++++++++++---- .../openshell/templates/_gateway-workload.tpl | 14 +- deploy/helm/openshell/templates/_helpers.tpl | 9 +- deploy/helm/openshell/templates/certgen.yaml | 11 +- .../helm/openshell/templates/supervisor.yaml | 64 +- .../helm/openshell/tests/supervisor_test.yaml | 60 +- .../tests/supervisor_wiring_test.yaml | 23 +- proto/README.md | 13 + proto/supervisor_host.proto | 129 +++ skills/debug-openshell-cluster/SKILL.md | 1 + 51 files changed, 3396 insertions(+), 1060 deletions(-) create mode 100644 crates/openshell-core/src/supervisor_host.rs create mode 100644 crates/openshell-sandbox-backend/src/sandbox_instance.rs delete mode 100644 crates/openshell-sandbox-backend/src/session_control.rs create mode 100644 crates/openshell-server/src/grpc/supervisor_host.rs create mode 100644 proto/supervisor_host.proto diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 537d390193..9c294000ca 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -70,15 +70,20 @@ gatewayConfig: The gateway chart creates a single-replica shared supervisor Deployment, Service, and NetworkPolicy in the release namespace, regardless of -shared/managed/operator workspace mode, and the certgen hook issues its -session-control Secrets. New sandbox generations run as sandbox instances in -that process. Its process-health `tcpSocket` probes connect to the mTLS -session-control port (5501); there is no separate health port. The chart starts -`--mode=shared`, sets `--gateway-endpoint` to the gateway gRPC URL, and binds -`--address` to this control/probe listener. Build the -gateway, supervisor, and sandbox images from the same branch: -all three must support the routed runtime connection. `workspace` is reserved -and rejected. Omit the field to disable it. `mise run +shared/managed/operator workspace mode, and the certgen hook issues the +`-supervisor-host` Secret: a private CA and the client certificate +the supervisor presents to the gateway. The gateway mounts only that Secret's +`ca.crt` and accepts the certificate solely for the private +`openshell.supervisor.host.v1` assignment stream; the topology therefore +requires gateway TLS (`server.disableTls` is rejected). New sandbox generations +run as sandbox instances in that process. The chart starts `--mode=shared`, +sets `--gateway-endpoint` to the gateway gRPC URL, mounts the host certificate +at `--host-tls-dir`, and binds `--address` (5501) to a plain TCP health +listener for the `tcpSocket` probes; nothing dials the supervisor but sandbox +runtimes (5500) and the kubelet. Build the gateway, supervisor, and sandbox +images from the same branch: all three must support the routed runtime +connection and the assignment stream. `workspace` is reserved and rejected. +Omit the field to disable it. `mise run e2e:kubernetes:shared-supervisor` runs the focused e2e (`ci/values-supervisor.yaml`). The task prepends that overlay to any colon-separated `OPENSHELL_E2E_KUBE_EXTRA_VALUES` list, so layer a diff --git a/Cargo.lock b/Cargo.lock index 3f0749059c..2b85efdc05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5129,6 +5129,7 @@ dependencies = [ "thiserror 2.0.20", "tokio", "tokio-rustls", + "tokio-stream", "tokio-tungstenite 0.26.2", "tonic", "tonic-prost", diff --git a/crates/openshell-bootstrap/src/pki.rs b/crates/openshell-bootstrap/src/pki.rs index 4ade3465f5..a2bd5ea006 100644 --- a/crates/openshell-bootstrap/src/pki.rs +++ b/crates/openshell-bootstrap/src/pki.rs @@ -44,6 +44,10 @@ pub const DEFAULT_SERVER_SANS: &[&str] = &[ "::1", ]; +/// Subject of the shared supervisor host's client certificate. The gateway +/// binds host identity to the issuing CA, not to this name. +pub const SUPERVISOR_HOST_CLIENT_NAME: &str = "openshell-supervisor-host"; + /// Generate a complete PKI bundle: CA, server cert, and client cert. /// /// `extra_sans` are additional Subject Alternative Names to add to the server @@ -53,6 +57,25 @@ pub const DEFAULT_SERVER_SANS: &[&str] = &[ /// never expire. This is appropriate for an internal dev-cluster PKI where certs /// are ephemeral to the cluster's lifetime. pub fn generate_pki(extra_sans: &[String]) -> Result { + generate_pki_with_client(extra_sans, "openshell-client", "openshell-user") +} + +/// Generate the private PKI of a gateway's shared supervisor host: a CA and +/// one client certificate the host presents to the gateway. The server +/// certificate is unused and never written. +pub fn generate_supervisor_host_pki() -> Result { + generate_pki_with_client( + &[], + SUPERVISOR_HOST_CLIENT_NAME, + SUPERVISOR_HOST_CLIENT_NAME, + ) +} + +fn generate_pki_with_client( + extra_sans: &[String], + client_common_name: &str, + client_organizational_unit: &str, +) -> Result { // --- CA --- let ca_key = openshell_crypto::pki::generate_keypair() .into_diagnostic() @@ -102,10 +125,10 @@ pub fn generate_pki(extra_sans: &[String]) -> Result { client_params.use_authority_key_identifier_extension = true; client_params .distinguished_name - .push(DnType::CommonName, "openshell-client"); + .push(DnType::CommonName, client_common_name); client_params .distinguished_name - .push(DnType::OrganizationalUnitName, "openshell-user"); + .push(DnType::OrganizationalUnitName, client_organizational_unit); let client_cert = openshell_crypto::pki::signed_by(client_params, &client_key, &ca_cert, &ca_key) diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 9e70b309c2..c9b351d71e 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -61,6 +61,7 @@ pub mod secrets; pub mod settings; pub mod shell; pub mod spiffe; +pub mod supervisor_host; pub mod telemetry; pub mod time; pub mod token_grant_failure; diff --git a/crates/openshell-core/src/proto/mod.rs b/crates/openshell-core/src/proto/mod.rs index 194a69e44d..fb200cadac 100644 --- a/crates/openshell-core/src/proto/mod.rs +++ b/crates/openshell-core/src/proto/mod.rs @@ -82,6 +82,11 @@ pub mod gateway_interceptor { pub use super::generated::openshell::gateway_interceptor::v1; } +/// Private gateway-to-shared-supervisor assignment protocol. +pub mod supervisor_host { + pub use super::generated::openshell::supervisor::host::v1; +} + pub use datamodel::v1::*; pub use gateway_interceptor::v1::*; pub use middleware::v1::*; diff --git a/crates/openshell-core/src/supervisor_host.rs b/crates/openshell-core/src/supervisor_host.rs new file mode 100644 index 0000000000..436cf13d3e --- /dev/null +++ b/crates/openshell-core/src/supervisor_host.rs @@ -0,0 +1,802 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway side of the private supervisor host protocol +//! (`openshell.supervisor.host.v1`). +//! +//! The gateway owns the desired set of sandbox instances a shared supervisor +//! runs. A compute driver publishes that set here; the gateway's gRPC server +//! hands each authenticated host stream to [`SupervisorHostController::attach`], +//! which sends complete snapshots and records what the host reports. +//! +//! Desired state is uninitialized until the driver has derived it from its +//! durable records once. Nothing is sent before that, so a restarted gateway +//! can never present an empty map as authoritative and stop running +//! instances. An explicit empty snapshot, after initialization, does stop +//! them. Full resyncs are compare-and-replace against the revision the driver +//! read before it started, so a list that raced a lifecycle edit never +//! resurrects a withdrawn instance or discards a new one. +//! +//! Exactly one host serves a gateway: a new attach from a newer connection of +//! the same host process, or from a new host process, supersedes the current +//! session; reports and acks from a superseded stream are dropped, and a +//! delayed hello from an older connection of the current process is refused. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use tokio::sync::{mpsc, watch}; +use tokio_stream::StreamExt as _; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Status, Streaming}; + +use crate::proto::supervisor_host::v1::{ + AssignmentSnapshot, HostAccepted, HostGatewayHeartbeat, HostGatewayMessage, HostHello, + HostMessage, HostStatus, SandboxInstanceAssignment, SandboxInstanceStatus, + host_gateway_message, host_message, +}; + +/// Interval at which both ends send heartbeats on an idle stream. +pub const HOST_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); +/// Silence after which the gateway closes a host stream. +const HOST_SILENCE_TIMEOUT: Duration = Duration::from_mins(1); +/// Bound on the host sending its hello. +const HELLO_TIMEOUT: Duration = Duration::from_secs(10); +/// Bound on queueing one gateway-to-host message; a host that cannot drain +/// its stream this long is disconnected rather than stalling the session. +const SEND_TIMEOUT: Duration = Duration::from_secs(30); +/// Gateway-to-host messages queued before the stream applies backpressure. +const OUTBOUND_QUEUE: usize = 16; +/// Most sandbox instances one shared supervisor host runs. The gateway +/// refuses to assign beyond it, and the host refuses snapshots beyond it, so +/// a sandbox is never created that its host could not accept. +pub const MAX_ASSIGNED_INSTANCES: usize = 256; +/// Most instances one host status report may describe: every assignment plus +/// instances still being removed. +pub const MAX_REPORTED_INSTANCES: usize = 4 * MAX_ASSIGNED_INSTANCES; + +/// Verified identity of a connected supervisor host. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HostIdentity { + /// Subject common name of the host certificate. + pub subject: String, + /// SHA-256 fingerprint of the verified host certificate. + pub certificate_sha256: String, +} + +/// What the gateway knows about one assigned route on the host. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ObservedInstance { + /// No host session is connected. + NoHost, + /// A host is connected but has not reconciled to the snapshot that + /// assigned this route. + Pending, + /// The host reconciled to that snapshot (or the route is not assigned) + /// and does not host the route. + Absent, + /// The host's latest report of the instance. + Reported(SandboxInstanceStatus), +} + +struct Assigned { + assignment: SandboxInstanceAssignment, + /// Snapshot revision that first carried this exact assignment. + revision: u64, +} + +#[derive(Default)] +struct Desired { + initialized: bool, + revision: u64, + assignments: BTreeMap, + /// Edits made before initialization, applied over the first full list. + pending: BTreeMap>, +} + +impl Desired { + fn snapshot(&self) -> Option { + self.initialized.then(|| AssignmentSnapshot { + revision: self.revision, + instances: self + .assignments + .values() + .map(|assigned| assigned.assignment.clone()) + .collect(), + }) + } + + /// Insert or replace one assignment; returns whether anything changed. + fn put(&mut self, assignment: SandboxInstanceAssignment, revision: u64) -> bool { + match self.assignments.get(&assignment.route) { + Some(existing) if existing.assignment == assignment => false, + _ => { + self.assignments.insert( + assignment.route.clone(), + Assigned { + assignment, + revision, + }, + ); + true + } + } + } +} + +struct HostSession { + id: u64, + identity: HostIdentity, + instance_id: String, + connection_epoch: u64, + superseded: watch::Sender, + /// Highest snapshot revision sent on this stream; acks beyond it are + /// impossible and rejected. + sent_revision: u64, +} + +#[derive(Default)] +struct Observed { + session: Option, + /// Latest report of the current session. + status: Option, + next_session: u64, +} + +struct Inner { + trust_anchor: PathBuf, + desired: Mutex, + /// Current desired revision; 0 while uninitialized. + revision: watch::Sender, + observed: Mutex, +} + +/// Gateway-owned desired assignments and observed status of one shared +/// supervisor host. +#[derive(Clone)] +pub struct SupervisorHostController { + inner: Arc, +} + +impl std::fmt::Debug for SupervisorHostController { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SupervisorHostController") + .field("trust_anchor", &self.inner.trust_anchor) + .finish_non_exhaustive() + } +} + +/// Why a change to the desired set was refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum AssignmentError { + /// A full resync was derived from a desired set that has since changed: + /// `baseline` is the revision the caller read before deriving its list and + /// `current` the one now in force; derive the list again from it. + #[error("desired assignments changed from revision {baseline} to {current} during the resync")] + Conflict { baseline: u64, current: u64 }, + /// The change would assign more instances than one host runs. + #[error("the shared supervisor hosts at most {MAX_ASSIGNED_INSTANCES} sandbox instances")] + Capacity, +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +impl SupervisorHostController { + /// A controller whose hosts authenticate with certificates issued by the + /// CA at `trust_anchor`. + #[must_use] + pub fn new(trust_anchor: PathBuf) -> Self { + Self { + inner: Arc::new(Inner { + trust_anchor, + desired: Mutex::new(Desired::default()), + revision: watch::Sender::new(0), + observed: Mutex::new(Observed::default()), + }), + } + } + + /// PEM CA certificate that issues supervisor host client certificates. + #[must_use] + pub fn trust_anchor_path(&self) -> &Path { + &self.inner.trust_anchor + } + + /// The driver has derived the complete desired set at least once. + #[must_use] + pub fn is_initialized(&self) -> bool { + lock(&self.inner.desired).initialized + } + + /// Current desired revision: 0 while uninitialized. Read it before + /// deriving a full list for [`Self::replace_assignments`]. + #[must_use] + pub fn revision(&self) -> u64 { + lock(&self.inner.desired).revision + } + + /// The current assignment of `route`, if any. + #[must_use] + pub fn assignment(&self, route: &str) -> Option { + lock(&self.inner.desired) + .assignments + .get(route) + .map(|assigned| assigned.assignment.clone()) + } + + /// Replace the whole desired set with `assignments`, derived from the + /// driver's durable records while the revision was `baseline`. + /// + /// Initializes the controller; edits made before initialization are + /// applied on top of the list. After initialization the replacement is + /// compare-and-replace: when the revision moved past `baseline` while + /// the list was being derived, nothing changes and the conflict is + /// returned so the caller derives the list again. + pub fn replace_assignments( + &self, + baseline: u64, + assignments: impl IntoIterator, + ) -> Result { + let mut desired = lock(&self.inner.desired); + if desired.initialized && desired.revision != baseline { + return Err(AssignmentError::Conflict { + baseline, + current: desired.revision, + }); + } + // Resolve the list and the pending overlay first, so a refused + // replacement leaves the current set untouched. + let mut resolved = BTreeMap::new(); + for assignment in assignments { + resolved.insert(assignment.route.clone(), assignment); + } + for (route, edit) in &desired.pending { + match edit { + Some(assignment) => { + resolved.insert(route.clone(), assignment.clone()); + } + None => { + resolved.remove(route); + } + } + } + if resolved.len() > MAX_ASSIGNED_INSTANCES { + return Err(AssignmentError::Capacity); + } + desired.pending.clear(); + let next = desired.revision + 1; + let mut changed = !desired.initialized; + for assignment in resolved.values() { + changed |= desired.put(assignment.clone(), next); + } + let before = desired.assignments.len(); + desired + .assignments + .retain(|route, _| resolved.contains_key(route)); + changed |= desired.assignments.len() != before; + desired.initialized = true; + if changed { + desired.revision = next; + self.inner.revision.send_replace(next); + } + Ok(desired.revision) + } + + /// Assign one instance. Before initialization the edit is kept until the + /// first full list arrives. Refused when it would exceed the host's + /// capacity, so the caller fails its create before anything runs. + pub fn assign(&self, assignment: SandboxInstanceAssignment) -> Result<(), AssignmentError> { + let mut desired = lock(&self.inner.desired); + if !desired.initialized { + let pending_additions = desired + .pending + .values() + .filter(|edit| edit.is_some()) + .count(); + if !desired.pending.contains_key(&assignment.route) + && pending_additions >= MAX_ASSIGNED_INSTANCES + { + return Err(AssignmentError::Capacity); + } + desired + .pending + .insert(assignment.route.clone(), Some(assignment)); + return Ok(()); + } + if !desired.assignments.contains_key(&assignment.route) + && desired.assignments.len() >= MAX_ASSIGNED_INSTANCES + { + return Err(AssignmentError::Capacity); + } + let next = desired.revision + 1; + if desired.put(assignment, next) { + desired.revision = next; + self.inner.revision.send_replace(next); + } + Ok(()) + } + + /// Withdraw one instance so the host stops it. + pub fn unassign(&self, route: &str) { + let mut desired = lock(&self.inner.desired); + if !desired.initialized { + desired.pending.insert(route.to_string(), None); + return; + } + if desired.assignments.remove(route).is_some() { + desired.revision += 1; + let revision = desired.revision; + self.inner.revision.send_replace(revision); + } + } + + /// Whether a host session is currently connected. + #[must_use] + pub fn host_connected(&self) -> bool { + lock(&self.inner.observed).session.is_some() + } + + /// What the connected host reports for `route`. + #[must_use] + pub fn observed(&self, route: &str) -> ObservedInstance { + let desired = lock(&self.inner.desired); + let observed = lock(&self.inner.observed); + if observed.session.is_none() { + return ObservedInstance::NoHost; + } + let Some(status) = &observed.status else { + return ObservedInstance::Pending; + }; + if let Some(instance) = status + .instances + .iter() + .find(|instance| instance.route == route) + { + return ObservedInstance::Reported(instance.clone()); + } + match desired.assignments.get(route) { + Some(assigned) if assigned.revision > status.applied_revision => { + ObservedInstance::Pending + } + _ => ObservedInstance::Absent, + } + } + + /// Serve one authenticated host stream. Reads the hello, supersedes any + /// current session, and returns the gateway-to-host stream. + pub async fn attach( + &self, + identity: HostIdentity, + mut inbound: Streaming, + ) -> Result>, Status> { + let hello = match tokio::time::timeout(HELLO_TIMEOUT, inbound.next()).await { + Ok(Some(Ok(HostMessage { + payload: Some(host_message::Payload::Hello(hello)), + }))) => hello, + Ok(Some(Ok(_))) => { + return Err(Status::invalid_argument( + "supervisor host stream must start with a hello", + )); + } + Ok(Some(Err(status))) => return Err(status), + Ok(None) => { + return Err(Status::invalid_argument( + "supervisor host stream ended before its hello", + )); + } + Err(_) => { + return Err(Status::deadline_exceeded( + "supervisor host did not send its hello in time", + )); + } + }; + if hello.instance_id.is_empty() { + return Err(Status::invalid_argument( + "supervisor host hello names no instance", + )); + } + let (tx, rx) = mpsc::channel(OUTBOUND_QUEUE); + let (session_id, superseded) = self.begin_session(identity, &hello)?; + tx.send(Ok(HostGatewayMessage { + payload: Some(host_gateway_message::Payload::Accepted(HostAccepted { + session_id: session_id.to_string(), + heartbeat_interval: Some(prost_types::Duration { + seconds: i64::try_from(HOST_HEARTBEAT_INTERVAL.as_secs()).unwrap_or(15), + nanos: 0, + }), + })), + })) + .await + .map_err(|_| Status::unavailable("supervisor host stream closed"))?; + let controller = self.clone(); + tokio::spawn(async move { + controller + .run_session(session_id, inbound, tx, superseded) + .await; + controller.end_session(session_id); + }); + Ok(ReceiverStream::new(rx)) + } + + /// Start a session for `hello`, superseding the current one. A hello from + /// the current host process with a connection epoch that is not newer is + /// a delayed or replayed connection and is refused. + fn begin_session( + &self, + identity: HostIdentity, + hello: &HostHello, + ) -> Result<(u64, watch::Receiver), Status> { + let mut observed = lock(&self.inner.observed); + if let Some(current) = &observed.session + && current.instance_id == hello.instance_id + && hello.connection_epoch <= current.connection_epoch + { + return Err(Status::failed_precondition( + "supervisor host connection epoch is not newer than the current session", + )); + } + observed.next_session += 1; + let id = observed.next_session; + if let Some(previous) = observed.session.take() { + tracing::info!( + subject = %previous.identity.subject, + instance_id = %previous.instance_id, + connection_epoch = previous.connection_epoch, + "superseding supervisor host session" + ); + previous.superseded.send_replace(true); + } + tracing::info!( + subject = %identity.subject, + instance_id = %hello.instance_id, + connection_epoch = hello.connection_epoch, + session = id, + "supervisor host attached" + ); + let (superseded, watch) = watch::channel(false); + observed.session = Some(HostSession { + id, + identity, + instance_id: hello.instance_id.clone(), + connection_epoch: hello.connection_epoch, + superseded, + sent_revision: 0, + }); + observed.status = None; + Ok((id, watch)) + } + + fn end_session(&self, id: u64) { + let mut observed = lock(&self.inner.observed); + if observed + .session + .as_ref() + .is_some_and(|session| session.id == id) + { + let session = observed.session.take(); + observed.status = None; + if let Some(session) = session { + tracing::info!( + subject = %session.identity.subject, + instance_id = %session.instance_id, + session = id, + "supervisor host detached; hosted instances keep running" + ); + } + } + } + + /// Note that `revision` was sent on session `id`. + fn record_sent(&self, id: u64, revision: u64) { + let mut observed = lock(&self.inner.observed); + if let Some(session) = observed.session.as_mut().filter(|session| session.id == id) { + session.sent_revision = session.sent_revision.max(revision); + } + } + + /// Record a report from session `id`. Reports from other sessions, acks + /// of revisions never sent on this session, regressing acks, and + /// oversized reports are dropped. + fn record_status(&self, id: u64, status: HostStatus) -> Result<(), &'static str> { + let mut observed = lock(&self.inner.observed); + let Some(session) = observed.session.as_ref().filter(|session| session.id == id) else { + return Err("report from a superseded supervisor host session"); + }; + if status.applied_revision > session.sent_revision { + return Err("supervisor host acknowledged a revision it was never sent"); + } + if status.instances.len() > MAX_REPORTED_INSTANCES { + return Err("supervisor host status reports too many instances"); + } + if observed + .status + .as_ref() + .is_some_and(|previous| status.applied_revision < previous.applied_revision) + { + return Err("supervisor host acknowledged an older revision than before"); + } + observed.status = Some(status); + Ok(()) + } + + fn snapshot(&self) -> Option { + lock(&self.inner.desired).snapshot() + } + + async fn run_session( + &self, + session_id: u64, + mut inbound: Streaming, + tx: mpsc::Sender>, + mut superseded: watch::Receiver, + ) { + let mut revision = self.inner.revision.subscribe(); + let mut sent_revision = 0; + let mut last_heard = tokio::time::Instant::now(); + let mut heartbeat = tokio::time::interval(HOST_HEARTBEAT_INTERVAL); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // A send that cannot complete within the bound ends the session, so a + // stalled host can neither block supersession nor delay heartbeats. + let send = |message: host_gateway_message::Payload| { + let tx = tx.clone(); + async move { + tx.send_timeout( + Ok(HostGatewayMessage { + payload: Some(message), + }), + SEND_TIMEOUT, + ) + .await + .is_ok() + } + }; + loop { + let current = *revision.borrow_and_update(); + if current > sent_revision { + let Some(snapshot) = self.snapshot() else { + continue; + }; + tracing::debug!( + session = session_id, + revision = snapshot.revision, + instances = snapshot.instances.len(), + "sending supervisor host assignment snapshot" + ); + let revision_sent = snapshot.revision; + if !send(host_gateway_message::Payload::Assignments(snapshot)).await { + return; + } + self.record_sent(session_id, revision_sent); + sent_revision = current; + } + tokio::select! { + changed = revision.changed() => { + if changed.is_err() { + return; + } + } + message = inbound.next() => match message { + Some(Ok(HostMessage { payload: Some(host_message::Payload::Status(status)) })) => { + last_heard = tokio::time::Instant::now(); + if let Err(reason) = self.record_status(session_id, status) { + tracing::warn!(session = session_id, reason, "dropped supervisor host status report"); + } + } + Some(Ok(HostMessage { payload: Some(host_message::Payload::Heartbeat(_)) })) => { + last_heard = tokio::time::Instant::now(); + } + Some(Ok(_)) => { + let _ = tx + .send_timeout( + Err(Status::invalid_argument( + "unexpected message on supervisor host stream", + )), + SEND_TIMEOUT, + ) + .await; + return; + } + Some(Err(status)) => { + tracing::debug!(session = session_id, %status, "supervisor host stream failed"); + return; + } + None => return, + }, + _ = heartbeat.tick() => { + if last_heard.elapsed() > HOST_SILENCE_TIMEOUT { + tracing::warn!(session = session_id, "supervisor host stream went silent; closing it"); + return; + } + if !send(host_gateway_message::Payload::Heartbeat(HostGatewayHeartbeat {})).await { + return; + } + } + _ = superseded.changed() => return, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assignment(route: &str, material: &[u8]) -> SandboxInstanceAssignment { + SandboxInstanceAssignment { + route: route.to_string(), + sandbox_id: format!("sandbox-{route}"), + generation: "gen".to_string(), + launch_material: material.to_vec(), + } + } + + fn status(applied_revision: u64, routes: &[&str]) -> HostStatus { + HostStatus { + applied_revision, + instances: routes + .iter() + .map(|route| SandboxInstanceStatus { + route: (*route).to_string(), + sandbox_id: format!("sandbox-{route}"), + phase: 2, + ready: true, + exit_code: 0, + error: String::new(), + }) + .collect(), + } + } + + #[test] + fn full_resyncs_initialize_once_and_then_compare_and_replace() { + let controller = SupervisorHostController::new(PathBuf::from("/ca.crt")); + let revision = controller.inner.revision.subscribe(); + // Edits before the first full list wait for it. + controller + .assign(assignment("rt-created-before-list", b"a")) + .unwrap(); + controller.unassign("rt-stopped-before-list"); + assert!(!controller.is_initialized()); + assert!(controller.snapshot().is_none()); + assert!(!revision.has_changed().unwrap()); + + let baseline = controller.revision(); + assert_eq!( + controller.replace_assignments( + baseline, + [ + assignment("rt-stopped-before-list", b"old"), + assignment("rt-listed", b"b"), + ], + ), + Ok(1) + ); + let snapshot = controller.snapshot().unwrap(); + assert_eq!(snapshot.revision, 1); + assert_eq!( + snapshot + .instances + .iter() + .map(|instance| instance.route.as_str()) + .collect::>(), + ["rt-created-before-list", "rt-listed"] + ); + assert!(revision.has_changed().unwrap()); + + // An identical list changes nothing. + assert_eq!( + controller.replace_assignments( + 1, + [ + assignment("rt-created-before-list", b"a"), + assignment("rt-listed", b"b"), + ], + ), + Ok(1) + ); + controller.assign(assignment("rt-listed", b"b")).unwrap(); + assert_eq!(controller.revision(), 1); + + // A list derived at revision 1 that races a stop cannot resurrect the + // stopped instance; the caller derives it again. + controller.unassign("rt-listed"); + assert_eq!(controller.revision(), 2); + assert_eq!( + controller.replace_assignments( + 1, + [ + assignment("rt-created-before-list", b"a"), + assignment("rt-listed", b"b"), + ], + ), + Err(AssignmentError::Conflict { + baseline: 1, + current: 2 + }) + ); + assert!(controller.assignment("rt-listed").is_none()); + assert_eq!(controller.replace_assignments(2, []), Ok(3)); + assert!(controller.snapshot().unwrap().instances.is_empty()); + + // The gateway refuses the assignment its host could not accept. + let full = + (0..MAX_ASSIGNED_INSTANCES).map(|index| assignment(&format!("rt-{index}"), b"x")); + assert_eq!(controller.replace_assignments(3, full), Ok(4)); + assert_eq!( + controller.assign(assignment("rt-one-too-many", b"x")), + Err(AssignmentError::Capacity) + ); + assert_eq!(controller.assign(assignment("rt-0", b"x")), Ok(())); + } + + #[test] + fn sessions_fence_stale_streams_epochs_and_acks() { + let controller = SupervisorHostController::new(PathBuf::from("/ca.crt")); + controller + .replace_assignments(0, [assignment("rt-a", b"a")]) + .unwrap(); + assert_eq!(controller.observed("rt-a"), ObservedInstance::NoHost); + + let identity = HostIdentity { + subject: "host".to_string(), + certificate_sha256: "00".to_string(), + }; + let hello = |instance_id: &str, connection_epoch| HostHello { + instance_id: instance_id.to_string(), + connection_epoch, + }; + let (first, superseded) = controller + .begin_session(identity.clone(), &hello("boot-1", 1)) + .unwrap(); + assert_eq!(controller.observed("rt-a"), ObservedInstance::Pending); + + // A delayed hello from an older connection of the same process is + // refused; a newer one supersedes the stream. + assert!( + controller + .begin_session(identity.clone(), &hello("boot-1", 1)) + .is_err() + ); + let (current, _) = controller + .begin_session(identity.clone(), &hello("boot-1", 2)) + .unwrap(); + assert!(*superseded.borrow()); + assert!(controller.record_status(first, status(0, &[])).is_err()); + + // Acks are bounded by what this stream was sent and never regress. + assert!(controller.record_status(current, status(1, &[])).is_err()); + controller.record_sent(current, 1); + controller + .record_status(current, status(1, &["rt-a"])) + .unwrap(); + assert!(controller.record_status(current, status(0, &[])).is_err()); + assert!(matches!( + controller.observed("rt-a"), + ObservedInstance::Reported(status) if status.ready + )); + + // A newer assignment is pending until the host reconciles past it. + controller.assign(assignment("rt-b", b"b")).unwrap(); + assert_eq!(controller.observed("rt-b"), ObservedInstance::Pending); + assert_eq!(controller.observed("rt-unknown"), ObservedInstance::Absent); + controller.record_sent(current, 2); + controller.record_status(current, status(2, &[])).unwrap(); + assert_eq!(controller.observed("rt-b"), ObservedInstance::Absent); + + // A new host process always takes over; ending a stale session is a no-op. + let (replacement, _) = controller + .begin_session(identity, &hello("boot-2", 1)) + .unwrap(); + controller.end_session(current); + assert!(controller.host_connected()); + controller.end_session(replacement); + assert_eq!(controller.observed("rt-a"), ObservedInstance::NoHost); + } +} diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index ac67e373ec..a91b57b7a2 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -192,27 +192,45 @@ keeps its established name) when the generation starts. Every later lifecycle de never reinterprets a running generation, and an unknown or incomplete record fails closed. +- **Assignments.** The gateway owns which instances the shared supervisor + hosts. The driver publishes that set into a + `openshell_core::supervisor_host::SupervisorHostController`, installed into + the gateway through the compute-driver build context, and the gateway serves + it to the supervisor over the private `openshell.supervisor.host.v1` stream. + The supervisor dials the gateway with a client certificate issued by the CA + at `shared_supervisor.host_ca_file`; that identity authorizes only the + assignment stream. - **Create and start.** The driver creates a per-sandbox egress allowance `os-shared-control-` from the workload Pod to the Pods the shared supervisor's Service selects, on the runtime port, binds its UID and - resourceVersion into the boundary evidence, and registers the sandbox - instance over session control before releasing the workload Pod. - Registration does not wait for the runtime, so create still returns before - the gateway persists the generation. The runtime identity's last component is - the instance route, which no Pod UID can equal, so `authenticate_sandbox` cannot vouch for a shared - generation. -- **Readiness.** A shared generation is available while its sandbox instance - is registered for the sandbox and accepted by the gateway and its fence and - egress allowance exist. A missing or ended instance is unavailable, and - reconciliation suspends the workload; an unreachable supervisor is unknown. -- **Stop and delete.** The driver removes the sandbox instance before - suspending or deleting the workload. Removal is best effort because ending the - workload ends supervision anyway. Reconciliation removes instances that no - Sandbox's current generation may run; it lists instances before Sandboxes, - because a generation records its route before registering it. -- **Isolation.** The admission credential is separate from every instance's - credentials. Sandboxes cannot reach session control, and the routing preface - only selects which generation-pinned TLS identity the supervisor demands. + resourceVersion into the boundary evidence, persists the instance's launch + material in the generation's `os-supervisor--` Secret (key + `instance.json`, owned by the workload Pod), and then assigns the instance + before releasing the workload Pod. Assignment does not wait for the + supervisor, so create still returns before the gateway persists the + generation. The runtime identity's last component is the instance route, + which no Pod UID can equal, so `authenticate_sandbox` cannot vouch for a + shared generation. +- **Readiness.** A shared generation is available while the supervisor + reports its instance as accepted by the gateway and its fence and egress + allowance exist. An instance the supervisor has reconciled past and does not + host, or reports as ended, is unavailable, and reconciliation suspends the + workload. A disconnected supervisor is unknown and a supervisor that has not + yet reconciled to the assignment is degraded; neither suspends anything. +- **Stop and delete.** The driver withdraws the instance before suspending or + deleting the workload, and the generation cleanup deletes its launch + Secret. Every reconciliation also rebuilds the desired set from the Sandbox + resources that may run, reading launch material from the current assignment + or the launch Secret, so instances of deleted Sandboxes and of stops made + while the gateway was down are withdrawn, and a restarted gateway recovers + its assignments before it sends anything. The rebuild is compare-and-replace + against the revision read before the list, so it never undoes a lifecycle + edit that raced it. +- **Isolation.** The host certificate is separate from every instance's + credentials and from the ordinary client CA; the gateway refuses it on every + other RPC and refuses every other identity on the assignment stream. + Sandboxes cannot reach the stream, and the routing preface only selects + which generation-pinned TLS identity the supervisor demands. The topology does not support upstream proxies or provider SPIFFE identity, and the shared supervisor is a single point of failure for the sandboxes it hosts. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 58fabeba7f..d2f3b5b8c5 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -202,7 +202,7 @@ pub enum SupervisorTopology { Workspace, } -/// Connection to the supervisor Helm deploys for the gateway topology. +/// The supervisor Helm deploys for the gateway topology. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SharedSupervisorConfig { @@ -211,11 +211,10 @@ pub struct SharedSupervisorConfig { pub service_namespace: String, /// Service port for runtime-initiated Sandbox Protocol connections. pub runtime_port: u16, - /// Service port for the mutually authenticated session-control API. - pub control_port: u16, - /// Directory with the session-control client `tls.crt` and `tls.key` and - /// the CA `ca.crt` that issued the supervisor's certificate. - pub control_tls_dir: String, + /// PEM CA certificate that issued the supervisor host's client + /// certificate. The gateway accepts that certificate on its own listener + /// and classifies the connection as the supervisor host. + pub host_ca_file: String, } impl SharedSupervisorConfig { @@ -225,22 +224,16 @@ impl SharedSupervisorConfig { "shared_supervisor service_name and service_namespace must be DNS labels".into(), ); } - if self.runtime_port == 0 - || self.control_port == 0 - || self.runtime_port == self.control_port - { - return Err( - "shared_supervisor runtime_port and control_port must be distinct and nonzero" - .into(), - ); + if self.runtime_port == 0 { + return Err("shared_supervisor runtime_port must be nonzero".into()); } - if !Path::new(&self.control_tls_dir).is_absolute() { - return Err("shared_supervisor control_tls_dir must be absolute".into()); + if !Path::new(&self.host_ca_file).is_absolute() { + return Err("shared_supervisor host_ca_file must be absolute".into()); } Ok(()) } - /// DNS name in the supervisor's session-control certificate. + /// DNS name of the shared supervisor Service, which sandbox runtimes dial. #[must_use] pub fn service_host(&self) -> String { format!("{}.{}.svc", self.service_name, self.service_namespace) @@ -1111,8 +1104,7 @@ mod tests { service_name: "openshell-supervisor".to_string(), service_namespace: "openshell".to_string(), runtime_port: 8443, - control_port: 9443, - control_tls_dir: "/etc/openshell-tls/session-control".to_string(), + host_ca_file: "/etc/openshell-tls/supervisor-host/ca.crt".to_string(), }; let gateway = |shared: Option| KubernetesComputeConfig { supervisor_topology: Some(SupervisorTopology::Gateway), @@ -1133,7 +1125,7 @@ mod tests { assert!(unset.validate_supervisor_topology().is_err()); for invalid in [ SharedSupervisorConfig { - control_port: 8443, + runtime_port: 0, ..shared() }, SharedSupervisorConfig { @@ -1141,7 +1133,7 @@ mod tests { ..shared() }, SharedSupervisorConfig { - control_tls_dir: "relative".to_string(), + host_ca_file: "relative/ca.crt".to_string(), ..shared() }, ] { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f0dbe6e210..d06ec35fb4 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -16,15 +16,15 @@ use crate::isolation::{ use crate::lifecycle::LifecycleGates; use crate::sandbox_runtime::{ BOUNDARY_CERTIFICATE_PATH, BOUNDARY_CONFIG_PATH, BOUNDARY_PRIVATE_KEY_PATH, ClientTlsMaterial, - SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, SupervisorClientTls, - boundary_service, boundary_service_matches_mode, control_policy, generate_proxy_ca_material, - image_pull_secret_copy, sandbox_bootstrap_secret, + SHARED_INSTANCE_KEY, SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, + SupervisorClientTls, boundary_service, boundary_service_matches_mode, control_policy, + generate_proxy_ca_material, image_pull_secret_copy, sandbox_bootstrap_secret, sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, shared_control_policy, - supervisor_bootstrap_secret, supervisor_pod, workload_fence, + shared_instance_secret, supervisor_bootstrap_secret, supervisor_pod, workload_fence, }; use crate::shared_supervisor::{ ANNOTATION_SUPERVISOR_SESSION, ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY, - SharedSupervisor, SupervisorPlacement, supervisor_placement, + SharedSupervisor, SupervisorPlacement, reported, supervisor_placement, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::authentication::v1::{ @@ -69,6 +69,7 @@ use openshell_core::proto::compute::v1::{ WatchSandboxesSandboxEvent, watch_sandboxes_event, }; use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; +use openshell_core::supervisor_host::{AssignmentError, ObservedInstance}; use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use openshell_sandbox_backend::boundary_protocol::{ GatewayVerificationKey, SandboxTlsClientConfig, SandboxTlsServerConfig, @@ -416,7 +417,6 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError { } const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); /// Unsupervised shared sandbox instances removed at once by one reconcile pass. -const SHARED_INSTANCE_GC_CONCURRENCY: usize = 4; /// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5); @@ -892,7 +892,8 @@ pub struct KubernetesComputeDriver { lifecycle_gates: Arc, config: KubernetesComputeConfig, operator_allowlist: Option, - /// Session-control client while new generations use the gateway topology. + /// Desired shared sandbox instances while new generations use the gateway + /// topology. shared_supervisor: Option, } @@ -2368,7 +2369,7 @@ impl KubernetesComputeDriver { Ok(runtime_identity) => runtime_identity, Err(error) => { warn!(sandbox_id = %sandbox.id, %error, "sandbox-runtime provisioning failed; rolling back Sandbox CR"); - self.remove_shared_instance(&placement).await; + self.remove_shared_instance(&placement); let _ = agent_sandbox_api .api .delete(&kube_name, &DeleteParams::default()) @@ -3025,25 +3026,34 @@ impl KubernetesComputeDriver { } } - /// Remove a shared generation's sandbox instance, if any. Instance loss - /// also ends supervision when the workload is suspended, so failures only - /// log. - async fn remove_shared_instance(&self, placement: &SupervisorPlacement) { + /// Withdraw a shared generation's sandbox instance, if any, so the host + /// stops it. Instance loss also ends supervision when the workload is + /// suspended, so a missing configuration only logs. + fn remove_shared_instance(&self, placement: &SupervisorPlacement) { let SupervisorPlacement::Shared { route } = placement else { return; }; - let Some(shared) = &self.shared_supervisor else { + if let Some(shared) = &self.shared_supervisor { + shared.withdraw(route); + } else { warn!( route, - "shared supervisor is not configured; its sandbox instance cannot be removed" + "shared supervisor is not configured; its sandbox instance cannot be withdrawn" ); - return; - }; - if let Err(error) = shared.remove(route).await { - warn!(route, %error, "could not remove shared supervisor sandbox instance"); } } + /// The gateway-owned assignments of the shared supervisor host, when the + /// gateway topology is configured. The gateway serves them to the host. + #[must_use] + pub fn supervisor_host_controller( + &self, + ) -> Option { + self.shared_supervisor + .as_ref() + .map(|shared| shared.controller().clone()) + } + /// Read operator-provided bootstrap material once per generation, so the /// whole generation stages, mounts, and references the same bytes, and so /// unreadable material fails before any Service, Pod, or Secret is created. @@ -3475,6 +3485,7 @@ impl KubernetesComputeDriver { .provision() .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; let workload_owner = runtime_pod_owner(&workload_pod_name, &workload_pod_uid); + let instance_owner = workload_owner.clone(); let sandbox_secret = sandbox_bootstrap_secret( namespace, names, @@ -3542,19 +3553,37 @@ impl KubernetesComputeDriver { vec![workload_owner], ) .await?; - // Registration only admits the instance; it does not wait for - // the runtime, so the gateway can persist this generation - // before the supervisor reads its configuration. - shared - .register(&shared_instance_registration( - &admission_object, - route, - sandbox_id, - provisioned.runtime_descriptor, - launch_authentication, - proxy_ca, - )?) - .await?; + let registration = shared_instance_registration( + &admission_object, + route, + sandbox_id, + provisioned.runtime_descriptor, + launch_authentication, + proxy_ca, + )?; + // The launch material is persisted with the generation first, + // so a restarted gateway rebuilds the host's assignments from + // the generation's own resources; assignment then publishes it + // to the host without waiting for the runtime. + let launch_material = serde_json::to_vec(®istration).map_err(|error| { + KubernetesDriverError::Message(format!( + "encode sandbox instance registration: {error}" + )) + })?; + secrets + .create( + &PostParams::default(), + &shared_instance_secret( + namespace, + names, + sandbox_id, + launch_material, + instance_owner, + ), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + shared.assign(®istration)?; } } pods.patch( @@ -4040,7 +4069,7 @@ impl KubernetesComputeDriver { if let Err(error) = restart { // The restart patch may not have recorded the sandbox instance yet. - self.remove_shared_instance(&placement).await; + self.remove_shared_instance(&placement); if let Err(rollback_error) = self.stop_sandbox_inner(sandbox_id).await { return Err(KubernetesDriverError::Message(format!( "restart failed: {error}; rollback remains pending: {rollback_error}" @@ -4165,7 +4194,7 @@ impl KubernetesComputeDriver { .await } SupervisorPlacement::Shared { .. } => { - self.remove_shared_instance(placement).await; + self.remove_shared_instance(placement); Ok(()) } } @@ -4361,8 +4390,7 @@ impl KubernetesComputeDriver { // stop it gracefully; end the instance without delaying the // delete. if let Some(placement @ SupervisorPlacement::Shared { .. }) = placement { - let driver = self.clone(); - tokio::spawn(async move { driver.remove_shared_instance(&placement).await }); + self.remove_shared_instance(&placement); } { let pod_api = Api::::namespaced(self.client.clone(), &obj_namespace); @@ -4458,20 +4486,12 @@ impl KubernetesComputeDriver { return; } }; - // List sandbox instances before Sandboxes: a generation records its - // route on its Sandbox before registering, so every instance listed - // here whose Sandbox still supervises it appears in the Sandbox list - // below. - let instances = match &self.shared_supervisor { - Some(shared) => match shared.list().await { - Ok(list) => Some(list.sessions), - Err(error) => { - warn!(%error, "could not list shared supervisor sandbox instances; reconciliation will retry"); - None - } - }, - None => None, - }; + // Read the desired revision before the list so a lifecycle edit that + // lands in between cannot be undone by the resync below. + let shared_baseline = self + .shared_supervisor + .as_ref() + .map(SharedSupervisor::revision); let list = match tokio::time::timeout( KUBE_API_TIMEOUT, lookup_api @@ -4491,9 +4511,11 @@ impl KubernetesComputeDriver { } }; - if let Some(instances) = instances { - self.remove_unsupervised_instances(instances, &list.items) - .await; + // The shared supervisor's desired set is derived from the Sandbox + // resources that may run, so instances of deleted Sandboxes and of + // stops that happened while this gateway was down are withdrawn here. + if let Some(baseline) = shared_baseline { + self.resync_shared_instances(baseline, &list.items).await; } for object in list.items { @@ -4747,33 +4769,107 @@ impl KubernetesComputeDriver { } } - /// Remove shared sandbox instances that no Sandbox's current generation may - /// still run, such as those of deleted Sandboxes or of stops that could - /// not reach the shared supervisor. - async fn remove_unsupervised_instances( - &self, - instances: Vec, - sandboxes: &[DynamicObject], - ) { + /// Rebuild the shared supervisor's desired set from the generations that + /// may run: their launch material comes from the current assignment or + /// from the generation's launch Secret. A failed read keeps the previous + /// set, because an incomplete set would stop running instances, and a + /// list that raced a lifecycle edit is refused by the controller. + async fn resync_shared_instances(&self, baseline: u64, sandboxes: &[DynamicObject]) { let Some(shared) = &self.shared_supervisor else { return; }; - let supervised = supervised_shared_routes(sandboxes); - let orphans = instances - .into_iter() - .filter(|instance| !supervised.contains(&instance.route)); - // Removals hold session-control connections while workloads stop. - futures::stream::iter(orphans) - .for_each_concurrent(SHARED_INSTANCE_GC_CONCURRENCY, |instance| async move { - info!( - sandbox_id = instance.sandbox_id, - "removing shared supervisor sandbox instance that no sandbox generation supervises" - ); - if let Err(error) = shared.remove(&instance.route).await { - warn!(sandbox_id = instance.sandbox_id, %error, "could not remove unsupervised shared sandbox instance"); + let mut assignments = Vec::new(); + for (object, route) in runnable_shared_generations(sandboxes) { + if let Some(assignment) = shared.assignment(&route) { + assignments.push(assignment); + continue; + } + let (Ok(sandbox_id), Some(generation)) = ( + sandbox_id_from_object(object), + sandbox_runtime_generation(object), + ) else { + continue; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + match self + .read_shared_instance_registration(namespace, &sandbox_id, generation, &route) + .await + .and_then(|registration| { + registration + .map(|registration| registration.to_assignment()) + .transpose() + .map_err(|error| KubernetesDriverError::Message(error.to_string())) + }) { + Ok(Some(assignment)) => assignments.push(assignment), + Ok(None) => warn!( + sandbox_id, + route, "shared sandbox instance has no launch material; it stays unassigned" + ), + Err(error) => { + warn!(sandbox_id, route, %error, "could not read shared sandbox instance launch material; keeping the previous assignments"); + return; } - }) - .await; + } + } + match shared.resync(baseline, assignments) { + Ok(_) | Err(AssignmentError::Conflict { .. }) => {} + Err(error) => { + warn!(%error, "could not publish the shared sandbox instances; keeping the previous assignments"); + } + } + } + + /// Launch material persisted with a shared generation, or `None` when its + /// Secret is gone or belongs to a dedicated generation. + async fn read_shared_instance_registration( + &self, + namespace: &str, + sandbox_id: &str, + generation: &str, + route: &str, + ) -> Result< + Option, + KubernetesDriverError, + > { + let names = SandboxRuntimeNames::for_generation(sandbox_id, generation); + let secrets = Api::::namespaced(self.client.clone(), namespace); + let secret = + tokio::time::timeout(KUBE_API_TIMEOUT, secrets.get_opt(&names.supervisor_secret)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading the shared sandbox instance launch Secret".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + let Some(material) = secret + .as_ref() + .and_then(|secret| secret.data.as_ref()) + .and_then(|data| data.get(SHARED_INSTANCE_KEY)) + else { + return Ok(None); + }; + // Decoder errors can quote field values, including credentials. + let registration: openshell_sandbox_backend::sandbox_instance::SandboxInstanceRegistration = + serde_json::from_slice(&material.0).map_err(|_| { + KubernetesDriverError::Message( + "shared sandbox instance launch Secret is not a valid registration".to_string(), + ) + })?; + if registration.route != route || registration.sandbox_id != sandbox_id { + return Err(KubernetesDriverError::Precondition( + "shared sandbox instance launch Secret does not match the recorded generation" + .to_string(), + )); + } + registration + .validate() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + Ok(Some(registration)) } async fn reconcile_sandbox_runtime_suspending( @@ -4800,9 +4896,9 @@ impl KubernetesComputeDriver { return; } } - // Completion forgets the placement, so end a shared sandbox instance now. + // Completion forgets the placement, so withdraw a shared sandbox instance now. if let Ok(placement) = supervisor_placement(object) { - self.remove_shared_instance(&placement).await; + self.remove_shared_instance(&placement); } if let Err(error) = self .delete_sandbox_runtime_supervisor( @@ -5288,15 +5384,16 @@ fn placement_for_release(object: &DynamicObject, sandbox_id: &str) -> Supervisor }) } -/// Sandbox instance routes of the shared generations that may currently run. -fn supervised_shared_routes(sandboxes: &[DynamicObject]) -> HashSet { +/// Shared generations that may currently run, with their sandbox instance +/// routes: the desired set of the shared supervisor. +fn runnable_shared_generations(sandboxes: &[DynamicObject]) -> Vec<(&DynamicObject, String)> { sandboxes .iter() .filter(|object| { sandbox_runtime_should_run(object) || sandbox_runtime_bootstrap_in_progress(object) }) .filter_map(|object| match supervisor_placement(object) { - Ok(SupervisorPlacement::Shared { route }) => Some(route), + Ok(SupervisorPlacement::Shared { route }) => Some((object, route)), _ => None, }) .collect() @@ -5328,10 +5425,10 @@ fn shared_instance_registration( launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, proxy_ca: crate::sandbox_runtime::ProxyCaMaterial, ) -> Result< - openshell_sandbox_backend::session_control::SandboxInstanceRegistration, + openshell_sandbox_backend::sandbox_instance::SandboxInstanceRegistration, KubernetesDriverError, > { - use openshell_sandbox_backend::session_control::{ + use openshell_sandbox_backend::sandbox_instance::{ ProxyCaMaterial, RegistrationTraceContext, SandboxInstanceRegistration, SecretPem, }; let trace = openshell_otel::current_trace_context_environment(); @@ -5417,11 +5514,20 @@ impl BootstrapControl { } } Self::SharedInstance { shared, route } => loop { - match shared.status(&route).await { - Ok(Some(status)) if status.ready => return true, - Ok(Some(status)) if status.phase.is_terminal() => return false, - Ok(None) => return false, - Ok(Some(_)) | Err(_) => {} + match shared.observed(&route) { + ObservedInstance::Reported(status) => { + let status = reported(&status); + if status.ready { + return true; + } + if status.phase.is_terminal() { + return false; + } + } + ObservedInstance::Absent => return false, + // A disconnected or still-reconciling host is not a + // verdict; keep waiting within the bootstrap grace. + ObservedInstance::Pending | ObservedInstance::NoHost => {} } tokio::time::sleep(Duration::from_secs(1)).await; }, @@ -6429,26 +6535,37 @@ async fn shared_instance_control_availability( } } }; - let (instance, fence, control) = tokio::join!( - shared.status(route), + // Only a host that has reconciled to the assigning snapshot can render a + // verdict on the instance; a disconnected or still-reconciling host must + // not read as a terminal failure and suspend a running sandbox. + let instance = match shared.observed(route) { + ObservedInstance::Reported(status) => { + let status = reported(&status); + if status.sandbox_id != sandbox_id || status.phase.is_terminal() { + SandboxRuntimeControlAvailability::Unavailable + } else if status.ready { + SandboxRuntimeControlAvailability::Available + } else { + SandboxRuntimeControlAvailability::Degraded + } + } + ObservedInstance::Pending => SandboxRuntimeControlAvailability::Degraded, + ObservedInstance::Absent => SandboxRuntimeControlAvailability::Unavailable, + ObservedInstance::NoHost => { + warn!( + sandbox_id, + "no shared supervisor host is connected; sandbox instance availability is unknown" + ); + SandboxRuntimeControlAvailability::Unknown + } + }; + let (fence, control) = tokio::join!( tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(&names.workload_policy)), tokio::time::timeout( KUBE_API_TIMEOUT, policies.get_opt(&names.shared_control_policy) ), ); - let instance = match instance { - Ok(Some(status)) if status.sandbox_id != sandbox_id || status.phase.is_terminal() => { - SandboxRuntimeControlAvailability::Unavailable - } - Ok(Some(status)) if status.ready => SandboxRuntimeControlAvailability::Available, - Ok(Some(_)) => SandboxRuntimeControlAvailability::Degraded, - Ok(None) => SandboxRuntimeControlAvailability::Unavailable, - Err(error) => { - warn!(sandbox_id, %error, "could not determine shared supervisor sandbox instance availability"); - SandboxRuntimeControlAvailability::Unknown - } - }; combine_sandbox_runtime_control_availability([ instance, exists(fence, "workload fence"), @@ -9462,7 +9579,7 @@ mod tests { let stopped = "rt-stopped-route-00001"; let mut dedicated = sandbox_object_for_test("dedicated", "dedicated"); dedicated.data = serde_json::json!({"spec": {"operatingMode": "Running"}}); - let routes = supervised_shared_routes(&[ + let sandboxes = [ shared(running, "Running", &[]), // A stop in progress still ends its sandbox instance itself. shared( @@ -9472,7 +9589,11 @@ mod tests { ), shared(stopped, "Suspended", &[]), dedicated, - ]); + ]; + let routes = runnable_shared_generations(&sandboxes) + .into_iter() + .map(|(_, route)| route) + .collect::>(); assert_eq!( routes, HashSet::from([running.to_string(), stopping.to_string()]) diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 94a5dd1c65..09843f1d74 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -43,6 +43,8 @@ pub const PROXY_CA_PRIVATE_KEY: &str = "proxy-ca.key"; /// bundle (`proxy_ca_bundle`). Distinct from [`PROXY_CA_CERTIFICATE_KEY`], /// which is the sandbox's own generated TLS-interception CA. pub const UPSTREAM_PROXY_CA_BUNDLE_KEY: &str = "upstream-proxy-ca.pem"; +/// JSON sandbox instance registration of a generation on the shared supervisor. +pub const SHARED_INSTANCE_KEY: &str = "instance.json"; pub const CLIENT_TLS_CA_KEY: &str = "client-ca.crt"; pub const SANDBOX_BOOTSTRAP_INPUT_PATH: &str = "/.openshell/bootstrap-input"; pub const BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json"; @@ -688,6 +690,35 @@ pub fn supervisor_bootstrap_secret( } } +/// Durable launch material of one shared-supervisor sandbox instance, under +/// the generation's supervisor Secret name so the existing generation cleanup +/// deletes it. The gateway reads it back to rebuild the host's assignments. +#[must_use] +pub fn shared_instance_secret( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + registration: Vec, + owner: OwnerReference, +) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(names.supervisor_secret.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(common_labels(sandbox_id, SUPERVISOR_SECRET_COMPONENT)), + ..Default::default() + }, + data: Some(BTreeMap::from([( + SHARED_INSTANCE_KEY.to_string(), + ByteString(registration), + )])), + immutable: Some(true), + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + /// Copy a configured image-pull Secret into an immutable Secret for one /// runtime generation. #[must_use] diff --git a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs index 9b03574f8d..b549b55ce6 100644 --- a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs +++ b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs @@ -6,15 +6,26 @@ //! Each runtime generation records where its supervisor runs when it starts. //! Every later lifecycle decision reads that record, so changing the gateway //! topology setting never reinterprets an existing generation. +//! +//! The gateway owns which sandbox instances the shared supervisor hosts. The +//! driver publishes that set into a [`SupervisorHostController`], which the +//! gateway serves to the supervisor over the private supervisor host stream, +//! and reads the host's observations back from it. The durable source of the +//! set is the generation's Sandbox resource plus its launch Secret, so a +//! restarted gateway rebuilds it before publishing anything, and a full +//! resync is compare-and-replace against the revision it started from. -use std::time::Duration; +use std::path::PathBuf; -use bytes::Bytes; -use http_body_util::{BodyExt as _, Full}; use kube::core::DynamicObject; -use openshell_sandbox_backend::session_control::{ - INSTANCE_REMOVAL_TIMEOUT, SESSIONS_PATH, SandboxInstanceList, SandboxInstanceRegistration, - SandboxInstanceStatusReport, +use openshell_core::proto::supervisor_host::v1::{ + SandboxInstanceAssignment, SandboxInstanceStatus, +}; +use openshell_core::supervisor_host::{ + AssignmentError, ObservedInstance, SupervisorHostController, +}; +use openshell_sandbox_backend::sandbox_instance::{ + SandboxInstanceRegistration, SandboxInstanceStatusReport, }; use crate::config::SharedSupervisorConfig; @@ -28,11 +39,6 @@ pub const ANNOTATION_SUPERVISOR_TOPOLOGY: &str = "openshell.ai/sandbox-runtime-s pub const ANNOTATION_SUPERVISOR_SESSION: &str = "openshell.ai/sandbox-runtime-supervisor-session"; pub const GATEWAY_TOPOLOGY: &str = "gateway"; -/// Bound on one session-control request, including connection and TLS. -/// Removals wait for the sandbox instance's workload to stop and use the -/// contract's longer bound instead. -const SESSION_CONTROL_TIMEOUT: Duration = Duration::from_secs(15); - /// Where a generation's supervisor runs, as recorded when it started. #[derive(Clone, Debug, PartialEq, Eq)] pub enum SupervisorPlacement { @@ -71,187 +77,89 @@ pub fn supervisor_placement( } } -/// Client for the shared supervisor's session-control API. It authenticates -/// with the gateway's admission certificate, which only Helm issues. +/// The gateway's desired set of shared sandbox instances and the host's +/// observations of them. The controller is the only owner of that state; +/// this type only translates between the driver's launch material and its +/// wire form. #[derive(Clone, Debug)] pub struct SharedSupervisor { config: SharedSupervisorConfig, + controller: SupervisorHostController, } impl SharedSupervisor { pub fn new(config: SharedSupervisorConfig) -> Self { - Self { config } + let controller = SupervisorHostController::new(PathBuf::from(&config.host_ca_file)); + Self { config, controller } } pub fn config(&self) -> &SharedSupervisorConfig { &self.config } - /// Register a generation; an identical registration is accepted again. - pub async fn register( - &self, - registration: &SandboxInstanceRegistration, - ) -> Result { - let body = serde_json::to_vec(registration).map_err(|error| { - KubernetesDriverError::Message(format!("encode sandbox instance registration: {error}")) - })?; - let (status, body) = self - .request( - http::Method::PUT, - &format!("{SESSIONS_PATH}/{}", registration.route), - Some(body), - ) - .await?; - match status { - http::StatusCode::CREATED | http::StatusCode::OK => decode(&body), - http::StatusCode::CONFLICT => Err(KubernetesDriverError::Precondition( - "shared supervisor already hosts this route with different launch material" - .to_string(), - )), - status => Err(rejected("register sandbox instance", status, &body)), - } + /// The controller the gateway serves to the supervisor host. + pub fn controller(&self) -> &SupervisorHostController { + &self.controller } - /// Status of one sandbox instance, or `None` when it is not registered. - pub async fn status( + /// Assign a generation's sandbox instance; an identical assignment + /// changes nothing. Refused as a precondition when the host is at + /// capacity, so the create fails before its workload runs. + pub fn assign( &self, - route: &str, - ) -> Result, KubernetesDriverError> { - let (status, body) = self - .request(http::Method::GET, &format!("{SESSIONS_PATH}/{route}"), None) - .await?; - match status { - http::StatusCode::OK => decode(&body).map(Some), - http::StatusCode::NOT_FOUND => Ok(None), - status => Err(rejected("read sandbox instance status", status, &body)), - } + registration: &SandboxInstanceRegistration, + ) -> Result<(), KubernetesDriverError> { + let assignment = registration + .to_assignment() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + self.controller + .assign(assignment) + .map_err(|error| KubernetesDriverError::Precondition(error.to_string())) } - /// Stop and forget one sandbox instance. A missing instance is already - /// removed. - pub async fn remove(&self, route: &str) -> Result<(), KubernetesDriverError> { - let (status, body) = self - .request( - http::Method::DELETE, - &format!("{SESSIONS_PATH}/{route}"), - None, - ) - .await?; - match status { - http::StatusCode::OK | http::StatusCode::NOT_FOUND => Ok(()), - status => Err(rejected("remove sandbox instance", status, &body)), - } + /// Withdraw a generation's sandbox instance so the host stops it. + pub fn withdraw(&self, route: &str) { + self.controller.unassign(route); } - /// Every sandbox instance the shared supervisor hosts. - pub async fn list(&self) -> Result { - let (status, body) = self.request(http::Method::GET, SESSIONS_PATH, None).await?; - match status { - http::StatusCode::OK => decode(&body), - status => Err(rejected("list sandbox instances", status, &body)), - } + /// Current desired revision, read before deriving a full list for + /// [`Self::resync`]. + #[must_use] + pub fn revision(&self) -> u64 { + self.controller.revision() } - async fn request( - &self, - method: http::Method, - path: &str, - body: Option>, - ) -> Result<(http::StatusCode, Bytes), KubernetesDriverError> { - let deadline = if method == http::Method::DELETE { - INSTANCE_REMOVAL_TIMEOUT - } else { - SESSION_CONTROL_TIMEOUT - }; - tokio::time::timeout(deadline, self.request_once(method, path, body)) - .await - .map_err(|_| { - KubernetesDriverError::Message( - "timed out calling the shared supervisor session-control API".to_string(), - ) - })? + /// The current assignment of a route, if any, so a resync keeps the + /// launch material it already holds instead of reading the Secret again. + #[must_use] + pub fn assignment(&self, route: &str) -> Option { + self.controller.assignment(route) } - async fn request_once( + /// Replace the desired set with the generations that may run, derived + /// from Sandbox resources and their launch Secrets while the revision was + /// `baseline`. The first call initializes the gateway's view; until then + /// nothing is sent to the host. A list that raced a lifecycle edit is + /// refused and derived again on the next reconciliation. + pub fn resync( &self, - method: http::Method, - path: &str, - body: Option>, - ) -> Result<(http::StatusCode, Bytes), KubernetesDriverError> { - let unavailable = |what: &str, error: &dyn std::fmt::Display| { - KubernetesDriverError::Message(format!("shared supervisor {what}: {error}")) - }; - // Read the admission material per request so rotated Secrets apply. - let directory = std::path::Path::new(&self.config.control_tls_dir); - let read = |name: &str| { - std::fs::read(directory.join(name)) - .map_err(|error| unavailable(&format!("session-control TLS file {name}"), &error)) - }; - let tls = openshell_sandbox_backend::session_control::client_tls_config( - &read("tls.crt")?, - &read("tls.key")?, - &read("ca.crt")?, - ) - .map_err(|error| unavailable("session-control TLS", &error))?; - let host = self.config.service_host(); - let server_name = rustls::pki_types::ServerName::try_from(host.clone()) - .map_err(|error| unavailable("session-control server name", &error))?; - let addresses = tokio::net::lookup_host((host.as_str(), self.config.control_port)) - .await - .map_err(|error| unavailable("resolve", &error))? - .collect::>(); - let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&addresses) - .await - .map_err(|error| unavailable("connect", &error))?; - let stream = tokio_rustls::TlsConnector::from(tls) - .connect(server_name, stream) - .await - .map_err(|error| unavailable("TLS", &error))?; - let (mut sender, connection) = - hyper::client::conn::http1::handshake(hyper_util::rt::TokioIo::new(stream)) - .await - .map_err(|error| unavailable("HTTP handshake", &error))?; - tokio::spawn(connection); - let request = http::Request::builder() - .method(method) - .uri(path) - .header(http::header::HOST, host) - .header(http::header::CONTENT_TYPE, "application/json") - .body(Full::new(Bytes::from(body.unwrap_or_default()))) - .map_err(|error| unavailable("request", &error))?; - let response = sender - .send_request(request) - .await - .map_err(|error| unavailable("request", &error))?; - let status = response.status(); - let body = response - .into_body() - .collect() - .await - .map_err(|error| unavailable("response", &error))? - .to_bytes(); - Ok((status, body)) + baseline: u64, + assignments: Vec, + ) -> Result { + self.controller.replace_assignments(baseline, assignments) } -} -fn decode(body: &[u8]) -> Result { - serde_json::from_slice(body).map_err(|error| { - KubernetesDriverError::Message(format!("decode shared supervisor response: {error}")) - }) + /// The host's view of one route. + #[must_use] + pub fn observed(&self, route: &str) -> ObservedInstance { + self.controller.observed(route) + } } -fn rejected(what: &str, status: http::StatusCode, body: &[u8]) -> KubernetesDriverError { - let detail = serde_json::from_slice::(body) - .ok() - .and_then(|value| { - value - .get("error") - .and_then(|error| error.as_str().map(str::to_string)) - }) - .unwrap_or_default(); - KubernetesDriverError::Message(format!( - "shared supervisor rejected {what} ({status}): {detail}" - )) +/// The host's report of one instance, in the driver's terms. +#[must_use] +pub fn reported(status: &SandboxInstanceStatus) -> SandboxInstanceStatusReport { + SandboxInstanceStatusReport::from_proto(status) } #[cfg(test)] diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index d353cb4f1b..b5f44bc5e8 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -291,6 +291,11 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { ) .await .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + // With the gateway supervisor topology, the driver owns the shared + // supervisor's assignments and the gateway serves them to the host. + if let Some(controller) = driver.supervisor_host_controller() { + context.install_supervisor_host(controller)?; + } let driver = openshell_driver_kubernetes::ComputeDriverService::new_in_process(driver); Ok(openshell_server::ComputeDriverInstance::InProcess( std::sync::Arc::new(driver), diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index ccf9c1021b..93b24ad451 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -136,10 +136,12 @@ handshake. Handshakes are bounded globally and per route, and an authenticated connection only fills its own instance's spare queue. A routed instance waits up to five minutes for its first connection, since it is registered before its workload Pod is scheduled, and the usual connect window -afterwards. `session_control` defines the registration, status, and removal -contract between a compute driver and the shared supervisor, including its -mutually authenticated TLS configuration; its `/v1/sessions` paths and JSON -fields keep their established wire names. +afterwards. `sandbox_instance` defines the launch material and status of one +hosted instance: `SandboxInstanceRegistration` is carried as the JSON +`launch_material` of a `SandboxInstanceAssignment` on the gateway's private +`openshell.supervisor.host.v1` stream, validated against the assignment's +route, sandbox, and generation, and `SandboxInstanceStatusReport` converts to +and from the stream's status message. Runtime-initiated transport supports TCP, protected Unix sockets, and host vsock listeners. These share bounded TLS handshakes, the authenticated spare diff --git a/crates/openshell-sandbox-backend/src/lib.rs b/crates/openshell-sandbox-backend/src/lib.rs index afb5e6561a..2fda81f66d 100644 --- a/crates/openshell-sandbox-backend/src/lib.rs +++ b/crates/openshell-sandbox-backend/src/lib.rs @@ -20,7 +20,7 @@ mod runtime; mod runtime_initiated; pub mod runtime_routed; pub mod sandbox_auth; -pub mod session_control; +pub mod sandbox_instance; pub mod stream_adapter; pub use openshell_core::proto::sandbox::backend::v1 as backend_proto; diff --git a/crates/openshell-sandbox-backend/src/sandbox_instance.rs b/crates/openshell-sandbox-backend/src/sandbox_instance.rs new file mode 100644 index 0000000000..6ce1606842 --- /dev/null +++ b/crates/openshell-sandbox-backend/src/sandbox_instance.rs @@ -0,0 +1,356 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Launch material and status of a sandbox instance hosted by a supervisor +//! shared by many sandboxes. +//! +//! A sandbox instance is one running generation of a stable Sandbox resource. +//! The gateway assigns instances to the shared supervisor over the private +//! `openshell.supervisor.host.v1` stream; each assignment carries one +//! instance's [`SandboxInstanceRegistration`] as JSON launch material, and the +//! host reports each instance's [`SandboxInstanceStatusReport`] back. A +//! registration grants the supervisor exactly the per-generation gateway and +//! Sandbox Protocol credentials it carries; it confers no other identity and +//! is never available to a sandbox. + +use std::time::Duration; + +use openshell_core::proto::supervisor_host::v1::{ + SandboxInstanceAssignment, SandboxInstancePhase, SandboxInstanceStatus, +}; +use openshell_isolation_interface::contract::BackendError; +use serde::{Deserialize, Serialize}; + +use crate::boundary_protocol::{SandboxRuntimeDescriptor, SandboxTransport}; + +/// Largest launch material one assignment may carry. +pub const MAX_LAUNCH_MATERIAL_BYTES: usize = 256 * 1024; +/// Time a shared supervisor gives a withdrawn sandbox instance's workload to +/// stop before cancelling the instance. +pub const INSTANCE_REMOVAL_GRACE: Duration = Duration::from_secs(20); + +/// PEM private key that never appears in `Debug` output. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct SecretPem(String); + +impl SecretPem { + #[must_use] + pub fn new(pem: String) -> Self { + Self(pem) + } + + #[must_use] + pub fn expose_secret(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Debug for SecretPem { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("SecretPem([REDACTED])") + } +} + +/// Proxy interception CA issued for one generation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProxyCaMaterial { + pub certificate_pem: String, + pub private_key_pem: SecretPem, +} + +/// W3C trace context of the operation that created the generation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RegistrationTraceContext { + pub traceparent: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tracestate: Option, +} + +/// Trusted launch material for one sandbox instance on a shared supervisor. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SandboxInstanceRegistration { + /// Route handle the runtime names; also the instance's key on the host. + pub route: String, + pub sandbox_id: String, + pub sandbox_name: String, + /// Supervisor-side launch descriptor using the routed transport. + pub runtime_descriptor: SandboxRuntimeDescriptor, + /// Gateway-issued credentials for exactly this generation. + pub auth: openshell_core::jwt::SupervisorAuthBundle, + /// Encoded `MainProcessConfig` for the canonical process. + pub main_process_spec: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workdir: Option, + /// Workload image recorded in the instance's OCSF events. + #[serde(default)] + pub container_image: String, + pub proxy_ca: ProxyCaMaterial, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trace_context: Option, +} + +impl std::fmt::Debug for SandboxInstanceRegistration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SandboxInstanceRegistration") + .field("route", &self.route) + .field("sandbox_id", &self.sandbox_id) + .field("sandbox_name", &self.sandbox_name) + .field("runtime_descriptor", &self.runtime_descriptor) + .field("auth", &self.auth) + .field("workdir", &self.workdir) + .field("container_image", &self.container_image) + .finish_non_exhaustive() + } +} + +impl SandboxInstanceRegistration { + /// Check that every part of the registration names the same sandbox, + /// generation, launch session, and route. + pub fn validate(&self) -> Result<(), BackendError> { + crate::runtime_routed::validate_route_handle(&self.route)?; + let descriptor = &self.runtime_descriptor; + match &descriptor.transport { + SandboxTransport::RuntimeInitiatedRouted { route, .. } if *route == self.route => {} + _ => { + return Err(BackendError::Descriptor( + "shared supervisor sandbox instances require this route's routed transport" + .to_string(), + )); + } + } + crate::runtime::validate_runtime_coordinates(descriptor)?; + if self.sandbox_id.is_empty() || descriptor.boundary_id != self.sandbox_id { + return Err(BackendError::Descriptor( + "runtime descriptor does not match the registered sandbox".to_string(), + )); + } + if descriptor.generation != self.auth.runtime_generation.as_str() + || descriptor.session_id != self.auth.session_id + { + return Err(BackendError::Descriptor( + "launch credentials do not match the runtime generation".to_string(), + )); + } + self.auth + .validate() + .map_err(|error| BackendError::Descriptor(error.to_string()))?; + openshell_core::sandbox_env::MainProcessConfig::decode(&self.main_process_spec) + .map_err(BackendError::Descriptor)?; + if self.proxy_ca.certificate_pem.is_empty() + || self.proxy_ca.private_key_pem.expose_secret().is_empty() + { + return Err(BackendError::Descriptor( + "proxy CA material is required".to_string(), + )); + } + Ok(()) + } + + /// Encode this registration as the launch material of one assignment. + pub fn to_assignment(&self) -> Result { + let launch_material = serde_json::to_vec(self).map_err(|error| { + BackendError::Descriptor(format!("encode sandbox instance registration: {error}")) + })?; + if launch_material.len() > MAX_LAUNCH_MATERIAL_BYTES { + return Err(BackendError::Descriptor( + "sandbox instance launch material is too large".to_string(), + )); + } + Ok(SandboxInstanceAssignment { + route: self.route.clone(), + sandbox_id: self.sandbox_id.clone(), + generation: self.auth.runtime_generation.to_string(), + launch_material, + }) + } + + /// Decode and validate one assignment's launch material, checking that + /// the assignment's own identifiers agree with it. + pub fn from_assignment(assignment: &SandboxInstanceAssignment) -> Result { + if assignment.launch_material.len() > MAX_LAUNCH_MATERIAL_BYTES { + return Err(BackendError::Descriptor( + "sandbox instance launch material is too large".to_string(), + )); + } + // Decoder errors can quote field values, including credentials. + let registration: Self = + serde_json::from_slice(&assignment.launch_material).map_err(|_| { + BackendError::Descriptor( + "sandbox instance launch material is not a valid registration".to_string(), + ) + })?; + if registration.route != assignment.route + || registration.sandbox_id != assignment.sandbox_id + || registration.auth.runtime_generation.as_str() != assignment.generation + { + return Err(BackendError::Descriptor( + "sandbox instance assignment does not match its launch material".to_string(), + )); + } + registration.validate()?; + Ok(registration) + } +} + +/// Lifecycle phase of a hosted sandbox instance. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case")] +pub enum SandboxInstancePhaseReport { + Starting, + Supervising, + Stopping, + Exited { exit_code: i32 }, + Failed { error: String }, +} + +impl SandboxInstancePhaseReport { + /// Supervision has ended and cannot resume for this generation. + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Exited { .. } | Self::Failed { .. }) + } +} + +/// Status of one hosted sandbox instance. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxInstanceStatusReport { + pub route: String, + pub sandbox_id: String, + #[serde(flatten)] + pub phase: SandboxInstancePhaseReport, + /// The gateway currently accepts this instance's supervision. + pub ready: bool, +} + +impl SandboxInstanceStatusReport { + /// The wire form of this report. + #[must_use] + pub fn to_proto(&self) -> SandboxInstanceStatus { + let (phase, exit_code, error) = match &self.phase { + SandboxInstancePhaseReport::Starting => { + (SandboxInstancePhase::Starting, 0, String::new()) + } + SandboxInstancePhaseReport::Supervising => { + (SandboxInstancePhase::Supervising, 0, String::new()) + } + SandboxInstancePhaseReport::Stopping => { + (SandboxInstancePhase::Stopping, 0, String::new()) + } + SandboxInstancePhaseReport::Exited { exit_code } => { + (SandboxInstancePhase::Exited, *exit_code, String::new()) + } + SandboxInstancePhaseReport::Failed { error } => { + (SandboxInstancePhase::Failed, 0, error.clone()) + } + }; + SandboxInstanceStatus { + route: self.route.clone(), + sandbox_id: self.sandbox_id.clone(), + phase: phase.into(), + ready: self.ready, + exit_code, + error, + } + } + + /// Read a wire status. An unknown phase reads as a failure, so a newer + /// host can never look healthy to an older gateway. + #[must_use] + pub fn from_proto(status: &SandboxInstanceStatus) -> Self { + let phase = match SandboxInstancePhase::try_from(status.phase) { + Ok(SandboxInstancePhase::Starting) => SandboxInstancePhaseReport::Starting, + Ok(SandboxInstancePhase::Supervising) => SandboxInstancePhaseReport::Supervising, + Ok(SandboxInstancePhase::Stopping) => SandboxInstancePhaseReport::Stopping, + Ok(SandboxInstancePhase::Exited) => SandboxInstancePhaseReport::Exited { + exit_code: status.exit_code, + }, + Ok(SandboxInstancePhase::Failed) => SandboxInstancePhaseReport::Failed { + error: status.error.clone(), + }, + Ok(SandboxInstancePhase::Unspecified) | Err(_) => SandboxInstancePhaseReport::Failed { + error: "sandbox instance reported an unknown phase".to_string(), + }, + }; + let ready = status.ready && !phase.is_terminal(); + Self { + route: status.route.clone(), + sandbox_id: status.sandbox_id.clone(), + phase, + ready, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_reports_round_trip_and_unknown_phases_fail_closed() { + for report in [ + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Supervising, + ready: true, + }, + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Exited { exit_code: 3 }, + ready: false, + }, + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Failed { + error: "boom".to_string(), + }, + ready: false, + }, + ] { + assert_eq!( + SandboxInstanceStatusReport::from_proto(&report.to_proto()), + report + ); + } + let unknown = SandboxInstanceStatus { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: 99, + ready: true, + exit_code: 0, + error: String::new(), + }; + let report = SandboxInstanceStatusReport::from_proto(&unknown); + assert!(report.phase.is_terminal()); + assert!(!report.ready); + } + + #[test] + fn assignments_must_agree_with_their_launch_material() { + let mismatched = SandboxInstanceAssignment { + route: "rt-0123456789abcdef".to_string(), + sandbox_id: "sandbox".to_string(), + generation: "gen".to_string(), + launch_material: b"{}".to_vec(), + }; + let error = SandboxInstanceRegistration::from_assignment(&mismatched).unwrap_err(); + assert!(error.to_string().contains("not a valid registration")); + let oversized = SandboxInstanceAssignment { + launch_material: vec![b' '; MAX_LAUNCH_MATERIAL_BYTES + 1], + ..mismatched + }; + assert!( + SandboxInstanceRegistration::from_assignment(&oversized) + .unwrap_err() + .to_string() + .contains("too large") + ); + } +} diff --git a/crates/openshell-sandbox-backend/src/session_control.rs b/crates/openshell-sandbox-backend/src/session_control.rs deleted file mode 100644 index 56aea956fd..0000000000 --- a/crates/openshell-sandbox-backend/src/session_control.rs +++ /dev/null @@ -1,285 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Contract between a compute driver and a supervisor shared by sandbox -//! instances. -//! -//! A sandbox instance is one running generation of a stable Sandbox resource. -//! The driver registers one instance with trusted launch material the gateway -//! issued for its generation, reads that instance's status, and removes it. -//! "Session control" and the `/v1/sessions` paths and JSON fields are the -//! protocol's established wire names; each entry they address is a sandbox -//! instance. The exchange runs over mutually authenticated TLS with a -//! credential reserved for instance admission; it is never available to a -//! sandbox. A registration grants the supervisor exactly the per-generation -//! gateway and Sandbox Protocol credentials it carries; it confers no other -//! identity. - -use std::sync::Arc; -use std::time::Duration; - -use openshell_isolation_interface::contract::BackendError; -use serde::{Deserialize, Serialize}; - -use crate::boundary_protocol::{SandboxRuntimeDescriptor, SandboxTransport}; - -/// Collection path of the session-control API; a sandbox instance is -/// `{prefix}/{route}`. The path keeps its established wire name. -pub const SESSIONS_PATH: &str = "/v1/sessions"; -/// Largest registration body a supervisor accepts. -pub const MAX_REGISTRATION_BYTES: usize = 256 * 1024; -/// Time a shared supervisor gives a removed sandbox instance's workload to -/// stop before cancelling the instance. -pub const INSTANCE_REMOVAL_GRACE: Duration = Duration::from_secs(20); -/// Bound on answering a removal request: the grace period, the instance -/// runtime's shutdown, file cleanup, and the connection itself. A removal -/// continues on the supervisor if its caller stops waiting. -pub const INSTANCE_REMOVAL_TIMEOUT: Duration = Duration::from_secs(45); - -/// PEM private key that never appears in `Debug` output. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(transparent)] -pub struct SecretPem(String); - -impl SecretPem { - #[must_use] - pub fn new(pem: String) -> Self { - Self(pem) - } - - #[must_use] - pub fn expose_secret(&self) -> &str { - &self.0 - } -} - -impl std::fmt::Debug for SecretPem { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("SecretPem([REDACTED])") - } -} - -/// Proxy interception CA issued for one generation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ProxyCaMaterial { - pub certificate_pem: String, - pub private_key_pem: SecretPem, -} - -/// W3C trace context of the operation that created the generation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RegistrationTraceContext { - pub traceparent: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tracestate: Option, -} - -/// Trusted launch material for one sandbox instance on a shared supervisor. -#[derive(Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SandboxInstanceRegistration { - /// Route handle the runtime names; also the instance's key on the host. - pub route: String, - pub sandbox_id: String, - pub sandbox_name: String, - /// Supervisor-side launch descriptor using the routed transport. - pub runtime_descriptor: SandboxRuntimeDescriptor, - /// Gateway-issued credentials for exactly this generation. - pub auth: openshell_core::jwt::SupervisorAuthBundle, - /// Encoded `MainProcessConfig` for the canonical process. - pub main_process_spec: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub workdir: Option, - /// Workload image recorded in the instance's OCSF events. - #[serde(default)] - pub container_image: String, - pub proxy_ca: ProxyCaMaterial, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trace_context: Option, -} - -impl std::fmt::Debug for SandboxInstanceRegistration { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("SandboxInstanceRegistration") - .field("route", &self.route) - .field("sandbox_id", &self.sandbox_id) - .field("sandbox_name", &self.sandbox_name) - .field("runtime_descriptor", &self.runtime_descriptor) - .field("auth", &self.auth) - .field("workdir", &self.workdir) - .field("container_image", &self.container_image) - .finish_non_exhaustive() - } -} - -impl SandboxInstanceRegistration { - /// Check that every part of the registration names the same sandbox, - /// generation, launch session, and route. - pub fn validate(&self) -> Result<(), BackendError> { - crate::runtime_routed::validate_route_handle(&self.route)?; - let descriptor = &self.runtime_descriptor; - match &descriptor.transport { - SandboxTransport::RuntimeInitiatedRouted { route, .. } if *route == self.route => {} - _ => { - return Err(BackendError::Descriptor( - "shared supervisor sandbox instances require this route's routed transport" - .to_string(), - )); - } - } - crate::runtime::validate_runtime_coordinates(descriptor)?; - if self.sandbox_id.is_empty() || descriptor.boundary_id != self.sandbox_id { - return Err(BackendError::Descriptor( - "runtime descriptor does not match the registered sandbox".to_string(), - )); - } - if descriptor.generation != self.auth.runtime_generation.as_str() - || descriptor.session_id != self.auth.session_id - { - return Err(BackendError::Descriptor( - "launch credentials do not match the runtime generation".to_string(), - )); - } - self.auth - .validate() - .map_err(|error| BackendError::Descriptor(error.to_string()))?; - openshell_core::sandbox_env::MainProcessConfig::decode(&self.main_process_spec) - .map_err(BackendError::Descriptor)?; - if self.proxy_ca.certificate_pem.is_empty() - || self.proxy_ca.private_key_pem.expose_secret().is_empty() - { - return Err(BackendError::Descriptor( - "proxy CA material is required".to_string(), - )); - } - Ok(()) - } -} - -/// Lifecycle phase of a hosted sandbox instance. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "phase", rename_all = "snake_case")] -pub enum SandboxInstancePhaseReport { - Starting, - Supervising, - Stopping, - Exited { exit_code: i32 }, - Failed { error: String }, -} - -impl SandboxInstancePhaseReport { - /// Supervision has ended and cannot resume for this generation. - #[must_use] - pub fn is_terminal(&self) -> bool { - matches!(self, Self::Exited { .. } | Self::Failed { .. }) - } -} - -/// Status of one hosted sandbox instance. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SandboxInstanceStatusReport { - pub route: String, - pub sandbox_id: String, - #[serde(flatten)] - pub phase: SandboxInstancePhaseReport, - /// The gateway currently accepts this instance's supervision. - pub ready: bool, -} - -/// Every sandbox instance a shared supervisor hosts. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SandboxInstanceList { - /// Hosted sandbox instances; `sessions` is the established wire name. - pub sessions: Vec, -} - -fn pem_certificates( - pem: &[u8], - what: &str, -) -> Result>, BackendError> { - let certificates = rustls_pemfile::certs(&mut &*pem) - .collect::, _>>() - .map_err(|error| BackendError::Descriptor(format!("parse {what}: {error}")))?; - if certificates.is_empty() { - return Err(BackendError::Descriptor(format!( - "{what} contains no certificates" - ))); - } - Ok(certificates) -} - -fn pem_private_key(pem: &[u8]) -> Result, BackendError> { - rustls_pemfile::private_key(&mut &*pem) - .map_err(|error| { - BackendError::Descriptor(format!("parse session-control private key: {error}")) - })? - .ok_or_else(|| { - BackendError::Descriptor("session-control private key PEM is empty".to_string()) - }) -} - -fn root_store(ca_pem: &[u8]) -> Result { - let mut roots = rustls::RootCertStore::empty(); - for certificate in pem_certificates(ca_pem, "session-control CA")? { - roots.add(certificate).map_err(|error| { - BackendError::Descriptor(format!("load session-control CA: {error}")) - })?; - } - Ok(roots) -} - -/// Server TLS for a shared supervisor: TLS 1.3, and only clients presenting a -/// certificate issued by the session-control CA. -pub fn server_tls_config( - certificate_pem: &[u8], - private_key_pem: &[u8], - client_ca_pem: &[u8], -) -> Result, BackendError> { - let provider = openshell_crypto::tls::configuration_provider(); - let verifier = rustls::server::WebPkiClientVerifier::builder_with_provider( - Arc::new(root_store(client_ca_pem)?), - provider.clone(), - ) - .build() - .map_err(|error| { - BackendError::Descriptor(format!("build session-control client verifier: {error}")) - })?; - let config = rustls::ServerConfig::builder_with_provider(provider) - .with_protocol_versions(&[&rustls::version::TLS13]) - .map_err(|error| BackendError::Descriptor(error.to_string()))? - .with_client_cert_verifier(verifier) - .with_single_cert( - pem_certificates(certificate_pem, "session-control certificate")?, - pem_private_key(private_key_pem)?, - ) - .map_err(|error| { - BackendError::Descriptor(format!("load session-control certificate: {error}")) - })?; - Ok(Arc::new(config)) -} - -/// Client TLS for a compute driver: TLS 1.3, trusting only the -/// session-control CA, and presenting the admission client certificate. -pub fn client_tls_config( - certificate_pem: &[u8], - private_key_pem: &[u8], - server_ca_pem: &[u8], -) -> Result, BackendError> { - let provider = openshell_crypto::tls::configuration_provider(); - let config = rustls::ClientConfig::builder_with_provider(provider) - .with_protocol_versions(&[&rustls::version::TLS13]) - .map_err(|error| BackendError::Descriptor(error.to_string()))? - .with_root_certificates(root_store(server_ca_pem)?) - .with_client_auth_cert( - pem_certificates(certificate_pem, "session-control client certificate")?, - pem_private_key(private_key_pem)?, - ) - .map_err(|error| { - BackendError::Descriptor(format!("load session-control client certificate: {error}")) - })?; - Ok(Arc::new(config)) -} diff --git a/crates/openshell-server/src/auth/audit.rs b/crates/openshell-server/src/auth/audit.rs index 4d4248ceb1..778f8b725f 100644 --- a/crates/openshell-server/src/auth/audit.rs +++ b/crates/openshell-server/src/auth/audit.rs @@ -216,7 +216,10 @@ fn authenticated(principal: &Principal) -> bool { match principal { Principal::User(user) => user.identity.provider != IdentityProvider::LocalDev, Principal::Anonymous => false, - Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) => true, + Principal::Operator(_) + | Principal::Sandbox(_) + | Principal::Peer(_) + | Principal::SupervisorHost(_) => true, } } @@ -234,6 +237,7 @@ fn mechanism(principal: &Principal) -> Option { SandboxIdentitySource::ComputeDriver { .. } => AuthMechanism::ComputeDriver, }), Principal::Peer(_) => Some(AuthMechanism::GatewayPeer), + Principal::SupervisorHost(_) => Some(AuthMechanism::SupervisorHost), Principal::Anonymous => None, } } @@ -250,6 +254,7 @@ fn credential_digest( ) -> Option<[u8; 32]> { let credential = match principal { Principal::Operator(operator) => operator.certificate_sha256.as_str(), + Principal::SupervisorHost(host) => host.identity.certificate_sha256.as_str(), Principal::User(user) if user.identity.provider == IdentityProvider::Mtls => { peer_certificate_sha256.unwrap_or_default() } @@ -506,6 +511,7 @@ fn principal_user(principal: &Principal) -> User { Principal::Operator(operator) => User::new(&operator.identity.subject, UserTypeId::Admin), Principal::Sandbox(sandbox) => User::other(&sandbox.sandbox_id, "Sandbox"), Principal::Peer(peer) => User::other(&peer.replica_id, "Gateway Peer"), + Principal::SupervisorHost(host) => User::other(&host.identity.subject, "Supervisor Host"), Principal::Anonymous => User::unidentified(), } } diff --git a/crates/openshell-server/src/auth/authenticator.rs b/crates/openshell-server/src/auth/authenticator.rs index db789213fa..624f20253b 100644 --- a/crates/openshell-server/src/auth/authenticator.rs +++ b/crates/openshell-server/src/auth/authenticator.rs @@ -118,6 +118,8 @@ pub enum AuthMechanism { GatewayPeer, /// The `CreateSshSession` token an SSH relay presents. SshSessionToken, + /// The shared supervisor host's gateway-scoped client certificate. + SupervisorHost, } impl AuthMechanism { @@ -131,6 +133,7 @@ impl AuthMechanism { Self::ComputeDriver => "compute_driver", Self::GatewayPeer => "k8s_service_account", Self::SshSessionToken => "ssh_session_token", + Self::SupervisorHost => "supervisor_host_mtls", } } } diff --git a/crates/openshell-server/src/auth/descriptor_authz.rs b/crates/openshell-server/src/auth/descriptor_authz.rs index cbdadbdb29..57599a65ff 100644 --- a/crates/openshell-server/src/auth/descriptor_authz.rs +++ b/crates/openshell-server/src/auth/descriptor_authz.rs @@ -17,7 +17,7 @@ use super::method_authz::{AuthMode, Role}; const AUTHORIZATION_EXTENSION: &str = "openshell.options.v1.authorization"; /// Gateway-served protobuf packages. -const GATEWAY_PACKAGES: &[&str] = &["openshell.v1"]; +const GATEWAY_PACKAGES: &[&str] = &["openshell.v1", "openshell.supervisor.host.v1"]; /// Bearer-authenticated methods that deliberately require no role or scope. /// @@ -125,6 +125,7 @@ impl DescriptorAuthTable { "dual" => AuthMode::Dual, "peer" => AuthMode::Peer, "operator" => AuthMode::Operator, + "supervisor_host" => AuthMode::SupervisorHost, other => { return Err(format!("method {path}: unknown auth_mode '{other}'")); } diff --git a/crates/openshell-server/src/auth/guard.rs b/crates/openshell-server/src/auth/guard.rs index 2218932942..cdaa84ffd1 100644 --- a/crates/openshell-server/src/auth/guard.rs +++ b/crates/openshell-server/src/auth/guard.rs @@ -30,10 +30,10 @@ use tonic::Status; pub fn ensure_sandbox_scope(principal: &Principal, claimed_sandbox_id: &str) -> Result<(), Status> { match principal { Principal::User(_) | Principal::Operator(_) => Ok(()), - Principal::Peer(_) => Err(deny( + Principal::Peer(_) | Principal::SupervisorHost(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, - "gateway peer principals may not call sandbox-scoped methods", + "gateway peer and supervisor host principals may not call sandbox-scoped methods", )), Principal::Sandbox(p) => { if p.sandbox_id == claimed_sandbox_id { @@ -88,7 +88,10 @@ pub fn ensure_sandbox_principal_scope( ensure_sandbox_scope(principal, claimed_sandbox_id)?; Ok(p.clone()) } - Principal::User(_) | Principal::Operator(_) | Principal::Peer(_) => Err(deny( + Principal::User(_) + | Principal::Operator(_) + | Principal::Peer(_) + | Principal::SupervisorHost(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, "supervisor RPCs require a sandbox principal", diff --git a/crates/openshell-server/src/auth/method_authz.rs b/crates/openshell-server/src/auth/method_authz.rs index 1efefb3564..6e5f4958c6 100644 --- a/crates/openshell-server/src/auth/method_authz.rs +++ b/crates/openshell-server/src/auth/method_authz.rs @@ -26,6 +26,9 @@ pub enum AuthMode { Peer, /// Only a verified mTLS operator may call this method. Operator, + /// Only the gateway's shared supervisor host, authenticated by its + /// gateway-scoped host certificate, may call this method. + SupervisorHost, } /// Coarse role mapping. Maps to the configured `admin_role` / @@ -91,7 +94,11 @@ pub fn is_sandbox_callable(method: &str) -> bool { pub fn is_user_callable(method: &str) -> bool { match lookup(method).map(|m| m.auth_mode) { Some( - AuthMode::Sandbox | AuthMode::Unauthenticated | AuthMode::Peer | AuthMode::Operator, + AuthMode::Sandbox + | AuthMode::Unauthenticated + | AuthMode::Peer + | AuthMode::Operator + | AuthMode::SupervisorHost, ) => false, Some(AuthMode::Bearer | AuthMode::Dual) | None => true, } @@ -112,6 +119,16 @@ pub fn is_peer_callable(method: &str) -> bool { matches!(lookup(method).map(|m| m.auth_mode), Some(AuthMode::Peer)) } +/// `true` if the method is callable by the shared supervisor host, and by +/// nothing else. +#[must_use] +pub fn is_supervisor_host_callable(method: &str) -> bool { + matches!( + lookup(method).map(|m| m.auth_mode), + Some(AuthMode::SupervisorHost) + ) +} + #[cfg(test)] mod tests { use super::*; @@ -173,6 +190,16 @@ mod tests { assert!(!is_user_callable(path)); assert!(is_peer_callable(path)); } + // The supervisor host stream is reachable only with a host identity. + let host_attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; + assert!(is_supervisor_host_callable(host_attach)); + assert!(!is_user_callable(host_attach)); + assert!(!is_operator_callable(host_attach)); + assert!(!is_sandbox_callable(host_attach)); + assert!(!is_peer_callable(host_attach)); + assert!(!is_supervisor_host_callable( + "/openshell.v1.OpenShell/ConnectSupervisor" + )); // Unauthenticated methods are not "user callable" — they're // intercepted before principal evaluation. assert!(!is_user_callable("/openshell.v1.OpenShell/Health")); diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs index e9809133d1..9f73e9ef8c 100644 --- a/crates/openshell-server/src/auth/principal.rs +++ b/crates/openshell-server/src/auth/principal.rs @@ -32,6 +32,10 @@ pub enum Principal { Sandbox(#[allow(dead_code)] SandboxPrincipal), /// Gateway replica authenticated for internal peer RPCs. Peer(PeerPrincipal), + /// The gateway's shared supervisor host, authenticated at the connection + /// level by a certificate from the host CA. It may call only the + /// supervisor host stream and never acts for a user or a sandbox. + SupervisorHost(SupervisorHostPrincipal), /// Truly unauthenticated caller (health probes, reflection). Sandbox-class /// and user-class methods reject this variant. #[allow(dead_code)] @@ -81,6 +85,13 @@ pub struct SandboxPrincipal { pub trust_domain: Option, } +/// Shared supervisor host caller. +#[derive(Debug, Clone)] +pub struct SupervisorHostPrincipal { + /// Verified host certificate identity. + pub identity: openshell_core::supervisor_host::HostIdentity, +} + /// Gateway peer caller. #[derive(Debug, Clone)] pub struct PeerPrincipal { @@ -119,6 +130,7 @@ impl Principal { Self::Operator(operator) => format!("operator:{}", operator.identity.subject), Self::Sandbox(sandbox) => format!("sandbox:{}", sandbox.sandbox_id), Self::Peer(peer) => format!("peer:{}", peer.replica_id), + Self::SupervisorHost(host) => format!("supervisor-host:{}", host.identity.subject), Self::Anonymous => "anonymous".to_string(), } } diff --git a/crates/openshell-server/src/auth/workspace_authz.rs b/crates/openshell-server/src/auth/workspace_authz.rs index d7c746ada6..e8739d9cd0 100644 --- a/crates/openshell-server/src/auth/workspace_authz.rs +++ b/crates/openshell-server/src/auth/workspace_authz.rs @@ -207,10 +207,10 @@ pub async fn authorize_workspace( workspace, grant: AuthGrant::Sandbox, }), - Principal::Peer(_) => Err(deny( + Principal::Peer(_) | Principal::SupervisorHost(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, - "gateway peer principals cannot perform workspace operations", + "gateway peer and supervisor host principals cannot perform workspace operations", )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } @@ -244,7 +244,9 @@ pub fn require_platform_admin(admin_role: &str, principal: &Principal) -> Result } Principal::User(_) => "platform admin role required for cross-workspace operations", Principal::Sandbox(_) => "sandbox principals cannot perform cross-workspace operations", - Principal::Peer(_) => "gateway peer principals cannot perform cross-workspace operations", + Principal::Peer(_) | Principal::SupervisorHost(_) => { + "gateway peer and supervisor host principals cannot perform cross-workspace operations" + } Principal::Anonymous => return Err(Status::unauthenticated("authentication required")), }; Err(deny(principal, AuthzDenial::PlatformAdminRequired, message)) diff --git a/crates/openshell-server/src/certgen.rs b/crates/openshell-server/src/certgen.rs index 537b15aef1..10c6fb3318 100644 --- a/crates/openshell-server/src/certgen.rs +++ b/crates/openshell-server/src/certgen.rs @@ -12,9 +12,10 @@ //! - **Kubernetes JWT-only mode** (`--jwt-only`): create only the //! sandbox-JWT signing Secret. Used when another controller, such as //! cert-manager, owns the TLS Secrets. -//! - **Kubernetes TLS-only mode** (`--tls-only`): create only the server and -//! client TLS Secrets of a private PKI with no sandbox-JWT signing key, -//! such as the shared supervisor's session-control PKI. +//! - **Kubernetes client-only mode** (`--client-only`): create only the client +//! TLS Secret of a fresh private PKI (certificate, key, and CA), with no +//! server certificate and no sandbox-JWT signing key. Issues the shared +//! supervisor's host identity; the gateway mounts the Secret's `ca.crt`. //! - **Local mode** (`--output-dir `): write PEMs to the local package //! filesystem layout. Used by systemd units' `ExecStartPre`. Also copies //! client materials to @@ -31,7 +32,9 @@ use k8s_openapi::api::core::v1::{ConfigMap, Secret}; use kube::Client; use kube::api::{Api, ObjectMeta, PostParams}; use miette::{IntoDiagnostic, Result, WrapErr}; -use openshell_bootstrap::pki::{DEFAULT_SERVER_SANS, PkiBundle, generate_pki}; +use openshell_bootstrap::pki::{ + DEFAULT_SERVER_SANS, PkiBundle, generate_pki, generate_supervisor_host_pki, +}; use openshell_core::paths::{create_dir_restricted, set_file_owner_only}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; @@ -55,7 +58,7 @@ pub struct CertgenArgs { namespace: Option, /// Name of the server TLS Secret (`kubernetes.io/tls`) to create. - #[arg(long, required_unless_present_any = ["output_dir", "jwt_only"])] + #[arg(long, required_unless_present_any = ["output_dir", "jwt_only", "client_only"])] server_secret_name: Option, /// Name of the client TLS Secret (`kubernetes.io/tls`) to create. @@ -65,7 +68,7 @@ pub struct CertgenArgs { /// Name of the sandbox-JWT signing-key Secret (`Opaque`) to create. /// Holds `signing.pem`, `public.pem`, and `kid` keys. Mounted on the /// gateway pod (only) so it can mint and validate per-sandbox JWTs. - #[arg(long, required_unless_present_any = ["output_dir", "tls_only"])] + #[arg(long, required_unless_present_any = ["output_dir", "client_only"])] jwt_secret_name: Option, /// Create only the sandbox-JWT signing-key Secret in Kubernetes mode. @@ -73,13 +76,13 @@ pub struct CertgenArgs { #[arg(long, conflicts_with = "output_dir")] jwt_only: bool, - /// Create only the server and client TLS Secrets in Kubernetes mode, for - /// a private PKI that issues no sandbox-JWT signing key. + /// Create only the client TLS Secret of a fresh private PKI in Kubernetes + /// mode: the shared supervisor host's certificate, key, and issuing CA. #[arg( long, - conflicts_with_all = ["output_dir", "jwt_only", "jwt_secret_name", "backend_ca_configmap_name"] + conflicts_with_all = ["output_dir", "jwt_only", "jwt_secret_name", "server_secret_name", "server_sans", "backend_ca_configmap_name"] )] - tls_only: bool, + client_only: bool, /// Extra Subject Alternative Name for the server certificate. Repeatable. /// Auto-detected as an IP address or DNS name. @@ -135,7 +138,11 @@ pub async fn run(args: CertgenArgs) -> Result<()> { if let Some(dir) = args.output_dir.as_deref() { run_local(dir, &args.server_sans) } else { - let bundle = generate_pki(&args.server_sans)?; + let bundle = if args.client_only { + generate_supervisor_host_pki()? + } else { + generate_pki(&args.server_sans)? + }; run_kubernetes(&args, &bundle).await?; if let Some(ref cm_name) = args.backend_ca_configmap_name { @@ -215,14 +222,47 @@ async fn run_kubernetes(args: &CertgenArgs, bundle: &PkiBundle) -> Result<()> { return Ok(()); } - let server_name = args - .server_secret_name - .as_deref() - .ok_or_else(|| miette::miette!("--server-secret-name is required"))?; let client_name = args .client_secret_name .as_deref() .ok_or_else(|| miette::miette!("--client-secret-name is required"))?; + if args.client_only { + if api + .get_opt(client_name) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to read secret {client_name}"))? + .is_some() + { + info!( + namespace = %namespace, + client = %client_name, + "client TLS secret already exists, skipping." + ); + return Ok(()); + } + let client_secret = tls_secret( + client_name, + &bundle.client_cert_pem, + &bundle.client_key_pem, + &bundle.ca_cert_pem, + ); + api.create(&PostParams::default(), &client_secret) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to create secret {client_name}"))?; + info!( + namespace = %namespace, + client = %client_name, + "client TLS secret created." + ); + return Ok(()); + } + + let server_name = args + .server_secret_name + .as_deref() + .ok_or_else(|| miette::miette!("--server-secret-name is required"))?; let server_exists = api .get_opt(server_name) .await @@ -236,34 +276,6 @@ async fn run_kubernetes(args: &CertgenArgs, bundle: &PkiBundle) -> Result<()> { .wrap_err_with(|| format!("failed to read secret {client_name}"))? .is_some(); - if args.tls_only { - match (server_exists, client_exists) { - (true, true) => info!( - namespace = %namespace, - server = %server_name, - client = %client_name, - "TLS secrets already exist, skipping." - ), - (false, false) => { - create_tls_secrets(&api, server_name, client_name, bundle).await?; - info!( - namespace = %namespace, - server = %server_name, - client = %client_name, - "TLS secrets created." - ); - } - _ => { - return Err(miette::miette!( - "partial PKI state in namespace {namespace}: only one of \ - {server_name} / {client_name} exists. Recover with: \ - kubectl delete secret -n {namespace} {server_name} {client_name}", - )); - } - } - return Ok(()); - } - let jwt_name = args .jwt_secret_name .as_deref() @@ -1033,31 +1045,27 @@ mod tests { } #[test] - fn tls_only_mode_needs_no_jwt_secret_and_refuses_one() { + fn client_only_mode_needs_only_the_client_secret_and_refuses_the_rest() { use clap::Parser as _; - let tls_only = [ + let client_only = [ "generate-certs", "--namespace=gateway", - "--tls-only", - "--server-secret-name=server", + "--client-only", "--client-secret-name=client", ]; - assert!(Cli::try_parse_from(tls_only).unwrap().args.tls_only); + assert!(Cli::try_parse_from(client_only).unwrap().args.client_only); for conflicting in [ "--jwt-secret-name=jwt", "--jwt-only", + "--server-secret-name=server", + "--server-san=example.test", "--output-dir=/tmp/pki", ] { - let mut args = tls_only.to_vec(); + let mut args = client_only.to_vec(); args.push(conflicting); assert!(Cli::try_parse_from(args).is_err(), "{conflicting}"); } - let missing_client = [ - "generate-certs", - "--namespace=gateway", - "--tls-only", - "--server-secret-name=server", - ]; + let missing_client = ["generate-certs", "--namespace=gateway", "--client-only"]; assert!(Cli::try_parse_from(missing_client).is_err()); } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a100aa11d1..d69db21753 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -763,6 +763,9 @@ pub struct ComputeRuntime { restart_notify: Arc, ssh_identities: Arc>, supervisor_ocsf_headers: Arc>, + /// Desired assignments of the shared supervisor host, when the in-process + /// driver installed one. + supervisor_host: Option, } impl fmt::Debug for ComputeRuntime { @@ -871,9 +874,27 @@ impl ComputeRuntime { restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), + supervisor_host: None, }) } + /// Serve the shared supervisor host that the in-process driver installed. + pub(crate) fn with_supervisor_host( + mut self, + supervisor_host: Option, + ) -> Self { + self.supervisor_host = supervisor_host; + self + } + + /// Gateway-owned assignments of the shared supervisor host, if any. + #[must_use] + pub fn supervisor_host( + &self, + ) -> Option<&openshell_core::supervisor_host::SupervisorHostController> { + self.supervisor_host.as_ref() + } + pub(crate) fn configure_ssh_identities( &self, credentials: crate::credentials::CredentialRuntime, @@ -7850,6 +7871,7 @@ pub fn new_test_runtime_with_driver( restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), + supervisor_host: None, } } @@ -8970,6 +8992,7 @@ mod tests { restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), + supervisor_host: None, } } diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index 35f16c0441..0368025348 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -10,6 +10,7 @@ pub mod provider; mod provider_credentials; pub mod provider_readiness; mod sandbox; +pub mod supervisor_host; pub use sandbox::mint_persisted_authentication; mod service; mod validation; diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 0e3dc96a24..4c61f4a0bc 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -2788,8 +2788,8 @@ async fn resolve_sandbox_by_name_for_principal( Principal::User(_) | Principal::Operator(_) => { sandbox.ok_or_else(|| Status::not_found("sandbox not found")) } - Principal::Peer(_) => Err(Status::permission_denied( - "gateway peer principals may not resolve sandbox configuration", + Principal::Peer(_) | Principal::SupervisorHost(_) => Err(Status::permission_denied( + "gateway peer and supervisor host principals may not resolve sandbox configuration", )), Principal::Anonymous => Err(Status::unauthenticated( "sandbox-scoped methods require an authenticated caller", diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 8abcbe037e..59125bd8b7 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -4293,7 +4293,8 @@ async fn exchange_provider_subject_token_inner( Principal::Sandbox(_) | Principal::User(_) | Principal::Operator(_) - | Principal::Peer(_), + | Principal::Peer(_) + | Principal::SupervisorHost(_), ) => Some(tonic::Code::PermissionDenied), Some(Principal::Anonymous) | None => Some(tonic::Code::Unauthenticated), }; diff --git a/crates/openshell-server/src/grpc/provider_credentials_tests.rs b/crates/openshell-server/src/grpc/provider_credentials_tests.rs index c602e28461..0b78a9e2e5 100644 --- a/crates/openshell-server/src/grpc/provider_credentials_tests.rs +++ b/crates/openshell-server/src/grpc/provider_credentials_tests.rs @@ -761,6 +761,7 @@ async fn real_tls_operator_boundary_and_admin_capabilities() { identity, fingerprint, crate::auth::audit::ConnectionSource::Peer(peer), + None, ) .await; } diff --git a/crates/openshell-server/src/grpc/supervisor_host.rs b/crates/openshell-server/src/grpc/supervisor_host.rs new file mode 100644 index 0000000000..09e948373e --- /dev/null +++ b/crates/openshell-server/src/grpc/supervisor_host.rs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The private `openshell.supervisor.host.v1` service: the gateway end of the +//! shared supervisor's assignment stream. +//! +//! The router admits only connections whose client certificate verified +//! against the supervisor host CA, so the handler sees a +//! [`Principal::SupervisorHost`] and hands the stream to the controller the +//! in-process compute driver installed. + +use std::pin::Pin; +use std::sync::Arc; + +use openshell_core::proto::supervisor_host::v1::supervisor_host_server::SupervisorHost; +use openshell_core::proto::supervisor_host::v1::{HostGatewayMessage, HostMessage}; +use tonic::{Request, Response, Status, Streaming}; + +use crate::ServerState; +use crate::auth::principal::Principal; + +/// gRPC path prefix of the supervisor host service. +pub const SUPERVISOR_HOST_PATH_PREFIX: &str = "/openshell.supervisor.host.v1.SupervisorHost/"; + +#[derive(Clone)] +pub struct SupervisorHostService { + state: Arc, +} + +impl SupervisorHostService { + #[must_use] + pub fn new(state: Arc) -> Self { + Self { state } + } +} + +#[tonic::async_trait] +impl SupervisorHost for SupervisorHostService { + type AttachStream = Pin< + Box> + Send + 'static>, + >; + + async fn attach( + &self, + request: Request>, + ) -> Result, Status> { + let identity = match request.extensions().get::() { + Some(Principal::SupervisorHost(host)) => host.identity.clone(), + _ => { + return Err(Status::permission_denied( + "the supervisor host stream requires a supervisor host certificate", + )); + } + }; + let controller = self + .state + .compute + .supervisor_host() + .ok_or_else(|| { + Status::failed_precondition("this gateway does not host a shared supervisor") + })? + .clone(); + let outbound = controller.attach(identity, request.into_inner()).await?; + Ok(Response::new(Box::pin(outbound))) + } +} diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index d2063e988c..1c37204181 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -68,8 +68,8 @@ fn membership_filter_subject<'a>( } } Principal::Sandbox(_) | Principal::Operator(_) => Ok(None), - Principal::Peer(_) => Err(Status::permission_denied( - "gateway peer principals cannot list workspaces", + Principal::Peer(_) | Principal::SupervisorHost(_) => Err(Status::permission_denied( + "gateway peer and supervisor host principals cannot list workspaces", )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index c0a83f4e33..241d4787a5 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -73,7 +73,7 @@ use std::path::{Path, PathBuf}; #[cfg(test)] use std::sync::LazyLock; use std::sync::{ - Arc, Mutex, + Arc, Mutex, OnceLock, atomic::{AtomicBool, Ordering}, }; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -246,7 +246,7 @@ pub use multiplex::{MultiplexService, MultiplexedService}; pub use persistence::Store; use sandbox_index::SandboxIndex; use sandbox_watch::SandboxWatchBus; -pub use tls::TlsAcceptor; +pub use tls::{PeerRealm, TlsAcceptor, TlsHandshake}; use tracing_bus::TracingLogBus; pub(crate) struct ServerStartupConfig { @@ -933,6 +933,7 @@ pub(crate) async fn run_server( } // Build TLS acceptor when TLS is configured; otherwise serve plaintext. + let supervisor_host = state.compute.supervisor_host(); let tls_acceptor = if let Some(tls) = &config.tls { let acceptor = TlsAcceptor::from_files( &tls.cert_path, @@ -943,6 +944,19 @@ pub(crate) async fn run_server( tls.external_key_path.as_deref(), tls.external_server_names.clone(), )?; + // The shared supervisor host authenticates with a client certificate + // from its own CA; the handshake accepts it and the connection is + // classified as a host, never as a user. + let acceptor = match supervisor_host { + Some(host) => { + info!( + trust_anchor = %host.trust_anchor_path().display(), + "shared supervisor host assignment stream enabled" + ); + acceptor.with_supervisor_host_ca(host.trust_anchor_path())? + } + None => acceptor, + }; // Spawn file-watcher-based TLS certificate reload worker. // Watches parent directories of cert/key/CA files and atomically @@ -950,6 +964,10 @@ pub(crate) async fn run_server( acceptor.spawn_reload_worker(shutdown_rx.clone()); Some(acceptor) + } else if supervisor_host.is_some() { + return Err(Error::config( + "the shared supervisor host requires gateway TLS; enable TLS or disable supervisor_topology", + )); } else { info!("TLS disabled — accepting plaintext connections"); None @@ -1190,12 +1208,30 @@ fn spawn_gateway_connection( ); } Ok(ConnectionProtocol::Tls | ConnectionProtocol::Unknown) => { - // acceptor.acceptor() snapshots the current TLS config; - // the returned acceptor owns an Arc that stays alive for - // the full duration of the handshake. - match acceptor.acceptor().accept(stream).await { + // acceptor.handshake() snapshots the current TLS config + // and verifiers together; the snapshot stays alive for + // the handshake and classifies its result. + let handshake = acceptor.handshake(); + match handshake.accept(stream).await { Ok(tls_stream) => { - let peer_identity = multiplex::extract_peer_identity(&tls_stream); + // A presented certificate must belong to exactly + // one realm; anything else closes the connection. + let (peer_identity, supervisor_host) = match handshake + .classify(&tls_stream) + { + Ok(PeerRealm::Anonymous) => (None, None), + Ok(PeerRealm::Client) => { + (multiplex::extract_peer_identity(&tls_stream), None) + } + Ok(PeerRealm::SupervisorHost(identity)) => ( + None, + Some(auth::principal::SupervisorHostPrincipal { identity }), + ), + Err(e) => { + warn!(error = %e, client = %addr, "Rejected client certificate"); + return; + } + }; let certificate_sha256 = match multiplex::extract_peer_certificate_sha256(&tls_stream) { Ok(fingerprint) => fingerprint, @@ -1210,6 +1246,7 @@ fn spawn_gateway_connection( peer_identity, certificate_sha256, ConnectionSource::Peer(addr), + supervisor_host, ) .await { @@ -1617,9 +1654,23 @@ impl ComputeDriverConfigContext<'_> { pub struct ComputeDriverBuildContext<'a> { config: ComputeDriverConfigContext<'a>, shutdown_rx: watch::Receiver, + supervisor_host: &'a OnceLock, } impl ComputeDriverBuildContext<'_> { + /// Let the gateway serve the shared supervisor host whose desired + /// assignments this in-process driver owns. The gateway adds the + /// controller's trust anchor to its TLS client roots and routes + /// authenticated host streams to it. + pub fn install_supervisor_host( + &self, + controller: openshell_core::supervisor_host::SupervisorHostController, + ) -> Result<()> { + self.supervisor_host.set(controller).map_err(|_| { + Error::config("a compute driver may install only one supervisor host controller") + }) + } + #[must_use] pub fn config_context(&self) -> ComputeDriverConfigContext<'_> { self.config @@ -1737,6 +1788,7 @@ async fn build_compute_runtime( info!(driver = %driver.name(), "Using compute driver"); let runtime = match driver { ConfiguredComputeDriver::Registered(registration) => { + let supervisor_host = OnceLock::new(); let build_context = ComputeDriverBuildContext { config: ComputeDriverConfigContext { driver_name: ®istration.name, @@ -1746,8 +1798,10 @@ async fn build_compute_runtime( driver_startup, }, shutdown_rx, + supervisor_host: &supervisor_host, }; let instance = registration.factory.build(build_context).await?; + let supervisor_host = supervisor_host.into_inner(); match instance { ComputeDriverInstance::InProcess(driver) => ComputeRuntime::from_driver( registration.name, @@ -1762,7 +1816,13 @@ async fn build_compute_runtime( .await .map_err(|error| { Error::execution(format!("failed to create compute runtime: {error}")) - })?, + })? + .with_supervisor_host(supervisor_host), + ComputeDriverInstance::ManagedRemote(_) if supervisor_host.is_some() => { + return Err(Error::config( + "only an in-process compute driver can install a supervisor host controller", + )); + } ComputeDriverInstance::ManagedRemote(mut endpoint) => { endpoint.name = registration.name; ComputeRuntime::new_remote_driver( diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index cc08e2ebc6..0f3ce32c33 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -18,6 +18,7 @@ use hyper_util::{ }; use metrics::{counter, histogram}; use openshell_core::proto::open_shell_server::OpenShellServer; +use openshell_core::proto::supervisor_host::v1::supervisor_host_server::SupervisorHostServer; use openshell_core::{ Config, proto::{Provider, UpdateProviderRequest}, @@ -48,7 +49,7 @@ use crate::{ auth::authz::AuthzPolicy, auth::identity::Identity, auth::oidc::{self, OidcAuthenticator}, - auth::principal::{OperatorPrincipal, Principal, UserPrincipal}, + auth::principal::{OperatorPrincipal, Principal, SupervisorHostPrincipal, UserPrincipal}, auth::workspace_authz::{MinWorkspaceRole, authorize_workspace}, http_router, service_http_router, }; @@ -257,22 +258,31 @@ impl MultiplexService { where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - self.serve_with_verified_certificate(stream, peer_identity, None, source) + self.serve_with_verified_certificate(stream, peer_identity, None, source, None) .await } - /// TLS listener only: fingerprint and identity must originate from its verified handshake. + /// TLS listener only: fingerprint, identity, and supervisor host must + /// originate from its verified handshake. A supervisor host connection + /// carries no user or operator identity, whatever its certificate says. pub async fn serve_with_verified_certificate( &self, stream: S, peer_identity: Option, certificate_sha256: Option, source: ConnectionSource, + supervisor_host: Option, ) -> Result<(), Box> where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - let operator = if self.state.config.mtls_auth.operator_enabled { + let peer_identity = if supervisor_host.is_some() { + None + } else { + peer_identity + }; + let operator = if self.state.config.mtls_auth.operator_enabled && supervisor_host.is_none() + { peer_identity .as_ref() .filter(|identity| { @@ -294,6 +304,15 @@ impl MultiplexService { self.state.gateway_interceptors.clone(), Some(self.state.clone()), ); + // The private host stream carries launch secrets, so it bypasses the + // configurable interceptors. + let openshell = SupervisorHostRoute::new( + SupervisorHostServer::new(crate::grpc::supervisor_host::SupervisorHostService::new( + self.state.clone(), + )) + .max_decoding_message_size(MAX_GRPC_DECODE_SIZE), + openshell, + ); let reflection = self.state.reflection_service.clone(); let authz_policy = self.state.config.oidc.as_ref().map(|oidc| AuthzPolicy { admin_role: oidc.admin_role.clone(), @@ -316,6 +335,7 @@ impl MultiplexService { ) .with_peer_certificate(certificate_sha256) .with_operator(operator) + .with_supervisor_host(supervisor_host) .with_connection_source(source) .with_logons(self.state.logons.clone()); let grpc_service = @@ -371,6 +391,57 @@ impl MultiplexService { } } +/// Routes the private supervisor host service past the gateway interceptors +/// and every other gRPC path to the `OpenShell` service. +#[derive(Clone)] +struct SupervisorHostRoute { + host: SupervisorHostServer, + gateway: N, +} + +impl SupervisorHostRoute { + fn new( + host: SupervisorHostServer, + gateway: N, + ) -> Self { + Self { host, gateway } + } +} + +impl tower::Service> for SupervisorHostRoute +where + N: tower::Service, Response = Response, Error = Infallible> + + Clone + + Send + + 'static, + N::Future: Send + 'static, +{ + type Response = Response; + type Error = Infallible; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.gateway.poll_ready(cx) + } + + fn call(&mut self, req: Request) -> Self::Future { + if req + .uri() + .path() + .starts_with(crate::grpc::supervisor_host::SUPERVISOR_HOST_PATH_PREFIX) + { + let mut host = self.host.clone(); + Box::pin(async move { + as tower::Service>>::call(&mut host, req) + .await + }) + } else { + let mut gateway = self.gateway.clone(); + Box::pin(async move { gateway.ready().await?.call(req).await }) + } + } +} + /// `OpenShell` gRPC wrapper that applies configured gateway interceptors before /// tonic dispatches to a specific RPC handler. #[derive(Clone)] @@ -754,6 +825,14 @@ fn gateway_principal_fields(principal: &Principal) -> BTreeMap { fields.insert("replica_id".to_string(), peer.replica_id.clone()); fields.insert("pod_uid".to_string(), peer.pod_uid.clone()); } + Principal::SupervisorHost(host) => { + fields.insert("kind".to_string(), "supervisor_host".to_string()); + fields.insert("subject".to_string(), host.identity.subject.clone()); + fields.insert( + "certificate_sha256".to_string(), + host.identity.certificate_sha256.clone(), + ); + } Principal::Anonymous => { fields.insert("kind".to_string(), "anonymous".to_string()); } @@ -1046,6 +1125,8 @@ pub struct AuthGrpcRouter { mtls_auth_enabled: bool, allow_unauthenticated_users: bool, operator: Option, + /// The connection presented the shared supervisor host certificate. + supervisor_host: Option, connection_source: Option, logons: audit::Logons, } @@ -1077,6 +1158,7 @@ impl AuthGrpcRouter { mtls_auth_enabled, allow_unauthenticated_users, operator: None, + supervisor_host: None, connection_source: None, logons: audit::Logons::default(), } @@ -1087,6 +1169,12 @@ impl AuthGrpcRouter { self } + /// The verified supervisor host identity of this connection, if any. + fn with_supervisor_host(mut self, supervisor_host: Option) -> Self { + self.supervisor_host = supervisor_host; + self + } + fn with_connection_source(mut self, source: ConnectionSource) -> Self { self.connection_source = Some(source); self @@ -1144,6 +1232,7 @@ where let mtls_auth_enabled = self.mtls_auth_enabled; let allow_unauthenticated_users = self.allow_unauthenticated_users; let operator = self.operator.clone(); + let supervisor_host = self.supervisor_host.clone(); let connection_source = self.connection_source; let logons = self.logons.clone(); let peer_certificate_sha256 = self.peer_certificate_sha256.clone(); @@ -1163,7 +1252,20 @@ where return inner.ready().await?.call(req).await; } - let principal = if let Some(operator) = operator { + let principal = if let Some(host) = supervisor_host { + if req.headers().contains_key(http::header::AUTHORIZATION) { + audit::emit_authentication_failure( + &path, + &origin(req.headers()), + Some(AuthMechanism::SupervisorHost), + AuthFailureReason::ConflictingCredentials, + ); + return Ok(status_response(tonic::Status::unauthenticated( + "supervisor host certificates must not be combined with authorization headers", + ))); + } + Principal::SupervisorHost(host) + } else if let Some(operator) = operator { if req.headers().contains_key(http::header::AUTHORIZATION) { audit::emit_authentication_failure( &path, @@ -1270,12 +1372,20 @@ where Principal::Peer(_) if !crate::auth::method_authz::is_peer_callable(&path) => { method_denied("gateway peer principals may not call this method") } + Principal::SupervisorHost(_) + if !crate::auth::method_authz::is_supervisor_host_callable(&path) => + { + method_denied("supervisor hosts may call only the supervisor host stream") + } Principal::Anonymous => { return Ok(status_response(tonic::Status::unauthenticated( "anonymous callers may not call authenticated methods", ))); } - Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) => None, + Principal::Operator(_) + | Principal::Sandbox(_) + | Principal::Peer(_) + | Principal::SupervisorHost(_) => None, }; if let Some((denial, status)) = denial { audit::emit_authorization_denial_for(&path, &origin, &principal, denial); @@ -3481,6 +3591,102 @@ mod tests { } } + #[tokio::test] + async fn supervisor_host_connections_reach_only_the_host_stream() { + let attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; + let host = SupervisorHostPrincipal { + identity: openshell_core::supervisor_host::HostIdentity { + subject: "openshell-supervisor-host".to_string(), + certificate_sha256: "verified-leaf".to_string(), + }, + }; + // The host realm is set even when mTLS user promotion is on: the + // connection carries no peer identity, so it can never be a user. + let host_router = |recorder| { + AuthGrpcRouter::with_peer_identity(recorder, None, None, None, true, false) + .with_supervisor_host(Some(host.clone())) + }; + for (method, permitted) in [ + (attach, true), + ("/openshell.v1.OpenShell/ListSandboxes", false), + ("/openshell.v1.OpenShell/ConnectSupervisor", false), + ("/openshell.v1.OpenShell/GetProviderCredentials", false), + ("/openshell.v1.OpenShell/Unknown", false), + ] { + let (recorder, seen) = PrincipalRecorder::new(); + let response = host_router(recorder) + .call(empty_request(method)) + .await + .unwrap(); + if permitted { + assert!(grpc_status(&response).is_none(), "{method}"); + assert!(matches!( + *seen.lock().unwrap(), + Some(Principal::SupervisorHost(_)) + )); + } else { + assert_eq!(grpc_status(&response).as_deref(), Some("7"), "{method}"); + } + } + + // A host connection may not also present a bearer identity. + let (recorder, seen) = PrincipalRecorder::new(); + let mut request = empty_request(attach); + request.headers_mut().insert( + http::header::AUTHORIZATION, + HeaderValue::from_static("Bearer user-token"), + ); + let response = host_router(recorder).call(request).await.unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("16")); + assert!(seen.lock().unwrap().is_none()); + + // No other principal reaches the host stream: an mTLS user, an + // operator, a sandbox, a peer, or an unauthenticated dev user. + let (recorder, _) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::with_peer_identity( + recorder, + None, + None, + Some(mtls_identity("openshell-client")), + true, + false, + ) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + let (recorder, _) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::new(recorder, None, None) + .with_operator(Some(operator_principal())) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + for principal in [ + sandbox("sandbox-a"), + Principal::Peer(crate::auth::principal::PeerPrincipal { + replica_id: "replica".into(), + pod_uid: "pod".into(), + }), + ] { + let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning( + Ok(Authentication::Accepted(principal)), + ))]); + let (recorder, _) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::new(recorder, Some(chain), None) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + } + let (recorder, _) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::new(recorder, None, None) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + } + #[tokio::test] async fn user_bearer_claims_and_local_dev_cannot_export_credentials() { let mut user = mtls_identity("ordinary-user"); diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 808c47cab3..3b902df1f9 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -535,6 +535,7 @@ mod tests { let mut compiled_method_count = 0; let mut isolation_methods = BTreeSet::new(); let mut sandbox_protocol_methods = BTreeSet::new(); + let mut supervisor_host_methods = BTreeSet::new(); for file in &public.file { let package = file.package.as_deref().unwrap_or_default(); for service in &file.service { @@ -558,6 +559,14 @@ mod tests { .expect("sandbox protocol method name") })); } + // Private gateway-to-shared-supervisor protocol: served by the + // gateway but never part of the public SDK surface. + if package == "openshell.supervisor.host.v1" { + assert_eq!(service_name, "SupervisorHost"); + supervisor_host_methods.extend(service.method.iter().map(|method| { + method.name.as_deref().expect("supervisor host method name") + })); + } if !matches!( (package, service_name), ("openshell.v1", "OpenShell") | ("openshell.inference.v1", "Inference") @@ -594,6 +603,7 @@ mod tests { sandbox_protocol_methods, BTreeSet::from(["Exchange", "Mediate"]) ); + assert_eq!(supervisor_host_methods, BTreeSet::from(["Attach"])); for signature in PROVIDER_READINESS_RPC_SIGNATURES { assert!( methods.iter().any(|method| method == signature), @@ -618,7 +628,8 @@ mod tests { + PEER_OWNER_RPC_SIGNATURES.len() + PEER_CONFIG_RPC_SIGNATURES.len() + isolation_methods.len() - + sandbox_protocol_methods.len(), + + sandbox_protocol_methods.len() + + supervisor_host_methods.len(), "classify every compiled RPC" ); assert_eq!( @@ -638,6 +649,11 @@ mod tests { + PEER_CONFIG_RPC_SIGNATURES.len() ); assert!(methods.iter().all(|method| !method.contains(".storage."))); + assert!( + methods + .iter() + .all(|method| !method.contains(".supervisor.host.")) + ); let public_closure = schema_closure(&index, public_roots); let durable_closure = diff --git a/crates/openshell-server/src/tls.rs b/crates/openshell-server/src/tls.rs index 26bcb713ce..e479adcdb7 100644 --- a/crates/openshell-server/src/tls.rs +++ b/crates/openshell-server/src/tls.rs @@ -13,10 +13,12 @@ use std::time::Duration; use arc_swap::ArcSwap; use notify::event::EventKind; use notify::{Event, RecursiveMode, Watcher}; +use openshell_core::supervisor_host::HostIdentity; use openshell_core::{Error, Result}; use openshell_ocsf::{ConfigStateChangeBuilder, EventContext, SeverityId, StateId, StatusId}; use rustls::ServerConfig; -use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, UnixTime}; +use rustls::server::danger::ClientCertVerifier; use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier}; use rustls::sign::CertifiedKey; use tokio::sync::{mpsc, watch}; @@ -31,7 +33,7 @@ use tracing::{debug, info, warn}; /// re-read from the same files without the caller tracking them separately. #[derive(Clone)] pub struct TlsAcceptor { - config: Arc>, + state: Arc>, cert_path: PathBuf, key_path: PathBuf, client_ca_path: Option, @@ -39,9 +41,103 @@ pub struct TlsAcceptor { external_cert_path: Option, external_key_path: Option, external_server_names: Vec, + /// CA that issues the shared supervisor host's client certificate. Its + /// certificates are accepted by the handshake and classified into the + /// supervisor host realm; they never become user identities. + supervisor_host_ca_path: Option, reload_spawned: Arc, } +/// One generation of TLS configuration: the handshake config and the +/// per-realm verifiers built from the same CA files, swapped together so a +/// handshake is always classified against the roots that accepted it. +struct AcceptorState { + config: Arc, + /// Verifies chains against the ordinary client CA alone. + client_verifier: Option>, + /// Verifies chains against the supervisor host CA alone. + supervisor_host_verifier: Option>, +} + +/// Who a finished handshake proves the peer to be. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PeerRealm { + /// No client certificate was presented. + Anonymous, + /// The chain verifies against the ordinary client CA. + Client, + /// The chain verifies against the supervisor host CA. + SupervisorHost(HostIdentity), +} + +/// A snapshot of the acceptor for one connection: the handshake and the +/// classification of its result use the same configuration generation. +#[derive(Clone)] +pub struct TlsHandshake { + state: Arc, +} + +impl TlsHandshake { + /// Complete the server side of a TLS handshake. + pub async fn accept(&self, stream: S) -> std::io::Result> + where + S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, + { + tokio_rustls::TlsAcceptor::from(self.state.config.clone()) + .accept(stream) + .await + } + + /// Classify a finished handshake. Fails closed: a presented chain that + /// verifies against neither configured CA, or against both, is refused + /// rather than treated as an ordinary client. + /// + /// # Errors + /// + /// Returns an error when the chain cannot be attributed to exactly one + /// realm or the host identity cannot be derived from it. + pub fn classify(&self, stream: &tokio_rustls::server::TlsStream) -> Result { + let (_, connection) = stream.get_ref(); + let Some((end_entity, intermediates)) = connection + .peer_certificates() + .and_then(|certificates| certificates.split_first()) + else { + return Ok(PeerRealm::Anonymous); + }; + let now = UnixTime::now(); + let verifies = |verifier: &Option>| { + verifier.as_ref().is_some_and(|verifier| { + verifier + .verify_client_cert(end_entity, intermediates, now) + .is_ok() + }) + }; + match ( + verifies(&self.state.supervisor_host_verifier), + verifies(&self.state.client_verifier), + ) { + (true, true) => Err(Error::tls( + "client certificate verifies against both the client CA and the supervisor host CA; refusing the ambiguous identity", + )), + (true, false) => { + let subject = certificate_common_name(end_entity).ok_or_else(|| { + Error::tls("supervisor host certificate has no subject common name") + })?; + let digest = openshell_crypto::sha256(end_entity.as_ref()) + .map_err(|e| Error::tls(format!("fingerprint host certificate: {e}")))?; + Ok(PeerRealm::SupervisorHost(HostIdentity { + subject, + certificate_sha256: hex::encode(digest), + })) + } + (false, true) => Ok(PeerRealm::Client), + (false, false) => Err(Error::tls( + "client certificate was accepted by the handshake but verifies against no configured client CA", + )), + } + } +} + impl TlsAcceptor { /// Create a new TLS acceptor from certificate and key files. /// @@ -69,7 +165,7 @@ impl TlsAcceptor { external_key_path: Option<&Path>, external_server_names: Vec, ) -> Result { - let config = build_server_config( + let state = build_acceptor_state( cert_path, key_path, client_ca_path, @@ -77,9 +173,10 @@ impl TlsAcceptor { external_cert_path, external_key_path, &external_server_names, + None, )?; Ok(Self { - config: Arc::new(ArcSwap::from(config)), + state: Arc::new(ArcSwap::from_pointee(state)), cert_path: cert_path.to_path_buf(), key_path: key_path.to_path_buf(), client_ca_path: client_ca_path.map(Path::to_path_buf), @@ -87,17 +184,31 @@ impl TlsAcceptor { external_cert_path: external_cert_path.map(Path::to_path_buf), external_key_path: external_key_path.map(Path::to_path_buf), external_server_names, + supervisor_host_ca_path: None, reload_spawned: Arc::new(AtomicBool::new(false)), }) } + /// Also accept client certificates issued by the supervisor host CA at + /// `ca_path`; [`TlsHandshake::classify`] places them in + /// [`PeerRealm::SupervisorHost`]. + /// + /// # Errors + /// + /// Returns an error if the CA file cannot be read or parsed. + pub fn with_supervisor_host_ca(mut self, ca_path: &Path) -> Result { + self.supervisor_host_ca_path = Some(ca_path.to_path_buf()); + self.reload()?; + Ok(self) + } + /// Re-read certificates from the same paths used at construction and /// atomically swap the active config. /// /// Returns `Ok(())` when the new config was built and swapped successfully. /// Returns `Err(...)` if cert/key loading fails — the old config is preserved. pub fn reload(&self) -> Result<()> { - let new_config = build_server_config( + let state = build_acceptor_state( &self.cert_path, &self.key_path, self.client_ca_path.as_deref(), @@ -105,8 +216,9 @@ impl TlsAcceptor { self.external_cert_path.as_deref(), self.external_key_path.as_deref(), &self.external_server_names, + self.supervisor_host_ca_path.as_deref(), )?; - self.config.store(new_config); + self.state.store(Arc::new(state)); let event = ConfigStateChangeBuilder::new(&tls_ocsf_ctx()) .severity(SeverityId::Informational) @@ -124,7 +236,16 @@ impl TlsAcceptor { /// alive for the duration of the TLS handshake. #[must_use] pub fn acceptor(&self) -> tokio_rustls::TlsAcceptor { - tokio_rustls::TlsAcceptor::from(self.config.load_full()) + tokio_rustls::TlsAcceptor::from(self.state.load().config.clone()) + } + + /// Snapshot the current configuration for one connection, so its + /// handshake and the classification of its peer agree. + #[must_use] + pub fn handshake(&self) -> TlsHandshake { + TlsHandshake { + state: self.state.load_full(), + } } /// Spawn a background worker that watches the parent directories of the @@ -174,6 +295,12 @@ impl TlsAcceptor { dirs.push(ext_dir.to_path_buf()); } } + if let Some(ref host_ca) = self.supervisor_host_ca_path { + let host_dir = host_ca.parent().unwrap_or_else(|| Path::new(".")); + if !dirs.contains(&host_dir.to_path_buf()) { + dirs.push(host_dir.to_path_buf()); + } + } let debounce = Duration::from_secs(1); @@ -361,7 +488,80 @@ fn build_cert_resolver( }))) } +fn root_store(roots: &[CertificateDer<'static>]) -> Result { + let mut store = rustls::RootCertStore::empty(); + for cert in roots { + store + .add(cert.clone()) + .map_err(|e| Error::tls(format!("failed to add CA certificate: {e}")))?; + } + Ok(store) +} + +/// Strict verifier for one realm: chains must be issued by one of `roots`. +fn build_realm_verifier(roots: &[CertificateDer<'static>]) -> Result> { + WebPkiClientVerifier::builder_with_provider( + Arc::new(root_store(roots)?), + openshell_crypto::tls::configuration_provider(), + ) + .build() + .map_err(|e| Error::tls(format!("failed to build client verifier: {e}"))) +} + +/// Build the handshake config and the per-realm verifiers from one read of +/// each CA file, so the handshake and the classification agree on the roots. +#[allow(clippy::too_many_arguments)] +fn build_acceptor_state( + cert_path: &Path, + key_path: &Path, + client_ca_path: Option<&Path>, + require_client_auth: bool, + external_cert_path: Option<&Path>, + external_key_path: Option<&Path>, + external_server_names: &[String], + supervisor_host_ca_path: Option<&Path>, +) -> Result { + let client_roots = client_ca_path.map(load_certs).transpose()?; + let host_roots = supervisor_host_ca_path.map(load_certs).transpose()?; + let config = build_server_config_with_roots( + cert_path, + key_path, + client_roots.as_deref(), + require_client_auth, + external_cert_path, + external_key_path, + external_server_names, + host_roots.as_deref(), + )?; + Ok(AcceptorState { + config, + client_verifier: client_roots + .as_deref() + .map(build_realm_verifier) + .transpose()?, + supervisor_host_verifier: host_roots + .as_deref() + .map(build_realm_verifier) + .transpose()?, + }) +} + +/// Subject common name of a DER certificate. +fn certificate_common_name(certificate: &CertificateDer<'_>) -> Option { + use x509_parser::prelude::*; + + let (_, parsed) = X509Certificate::from_der(certificate.as_ref()).ok()?; + parsed + .subject() + .iter_common_name() + .next() + .and_then(|attribute| attribute.as_str().ok()) + .map(str::to_string) +} + /// Build a `ServerConfig` from certificate, key, and optional client CA files. +#[cfg(test)] +#[allow(clippy::too_many_arguments)] fn build_server_config( cert_path: &Path, key_path: &Path, @@ -370,6 +570,35 @@ fn build_server_config( external_cert_path: Option<&Path>, external_key_path: Option<&Path>, external_server_names: &[String], + supervisor_host_ca_path: Option<&Path>, +) -> Result> { + build_server_config_with_roots( + cert_path, + key_path, + client_ca_path.map(load_certs).transpose()?.as_deref(), + require_client_auth, + external_cert_path, + external_key_path, + external_server_names, + supervisor_host_ca_path + .map(load_certs) + .transpose()? + .as_deref(), + ) +} + +/// Build a `ServerConfig` from certificate and key files and already loaded +/// client roots: the ordinary client CA and the supervisor host CA. +#[allow(clippy::too_many_arguments)] +fn build_server_config_with_roots( + cert_path: &Path, + key_path: &Path, + client_roots: Option<&[CertificateDer<'static>]>, + require_client_auth: bool, + external_cert_path: Option<&Path>, + external_key_path: Option<&Path>, + external_server_names: &[String], + supervisor_host_roots: Option<&[CertificateDer<'static>]>, ) -> Result> { let certs = load_certs(cert_path)?; let key = load_key(key_path)?; @@ -387,17 +616,24 @@ fn build_server_config( external_server_names, )?; - let mut config = if let Some(ca_path) = client_ca_path { - let ca_certs = load_certs(ca_path)?; - let mut root_store = rustls::RootCertStore::empty(); - for cert in ca_certs { - root_store - .add(cert) - .map_err(|e| Error::tls(format!("failed to add CA certificate: {e}")))?; + let client_roots = client_roots + .into_iter() + .chain(supervisor_host_roots) + .flatten() + .cloned() + .collect::>(); + let mut config = if client_roots.is_empty() { + let builder = openshell_crypto::tls::server_builder().with_no_client_auth(); + if let Some(resolver) = resolver { + builder.with_cert_resolver(resolver) + } else { + builder + .with_single_cert(certs, key) + .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? } - + } else { let verifier_builder = WebPkiClientVerifier::builder_with_provider( - Arc::new(root_store), + Arc::new(root_store(&client_roots)?), openshell_crypto::tls::configuration_provider(), ); let verifier = if require_client_auth { @@ -416,15 +652,6 @@ fn build_server_config( .with_single_cert(certs, key) .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? } - } else { - let builder = openshell_crypto::tls::server_builder().with_no_client_auth(); - if let Some(resolver) = resolver { - builder.with_cert_resolver(resolver) - } else { - builder - .with_single_cert(certs, key) - .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? - } }; config @@ -543,6 +770,7 @@ mod tests { None, None, &[], + None, ) .expect("failed to build server config"); @@ -889,6 +1117,179 @@ mod tests { let _ = tokio::time::timeout(Duration::from_secs(2), handle).await; } + /// Issue a leaf certificate for `common_name` from `ca`. + fn issue_leaf( + common_name: &str, + ca: &openshell_crypto::pki::Certificate, + ca_key: &KeyPair, + ) -> (Vec>, PrivateKeyDer<'static>) { + let mut params = CertificateParams::new(Vec::::new()).expect("leaf params"); + params + .distinguished_name + .push(rcgen::DnType::CommonName, common_name); + let key = openshell_crypto::pki::generate_keypair().expect("leaf key"); + let cert = openshell_crypto::pki::signed_by(params, &key, ca, ca_key).expect("sign leaf"); + let chain = rustls_pemfile::certs(&mut cert.pem().as_bytes()) + .collect::, _>>() + .expect("leaf chain"); + let key = rustls_pemfile::private_key(&mut key.serialize_pem().unwrap().as_bytes()) + .expect("leaf key pem") + .expect("leaf key present"); + (chain, key) + } + + /// Handshake with `acceptor`, presenting `client` when given, and + /// classify the server side of the connection. + async fn classify_with( + acceptor: &TlsAcceptor, + server_ca: &Path, + client: Option<(Vec>, PrivateKeyDer<'static>)>, + ) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("address"); + let handshake = acceptor.handshake(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let tls = handshake.accept(stream).await?; + Ok::<_, std::io::Error>(handshake.classify(&tls)) + }); + let mut roots = rustls::RootCertStore::empty(); + for cert in load_certs(server_ca).expect("server CA") { + roots.add(cert).expect("root"); + } + let builder = openshell_crypto::tls::client_builder().with_root_certificates(roots); + let config = match client { + Some((chain, key)) => builder + .with_client_auth_cert(chain, key) + .expect("client identity"), + None => builder.with_no_client_auth(), + }; + let tcp = TcpStream::connect(address).await.expect("connect"); + let tls = tokio_rustls::TlsConnector::from(Arc::new(config)) + .connect( + rustls::pki_types::ServerName::try_from("localhost").unwrap(), + tcp, + ) + .await + .expect("client handshake"); + drop(tls); + server + .await + .expect("server task") + .expect("server handshake") + } + + /// The supervisor host CA and the ordinary client CA are separate realms: + /// a host certificate never becomes a client identity, a client + /// certificate never becomes a host, and overlapping roots are refused. + #[tokio::test] + async fn handshake_keeps_supervisor_host_and_client_realms_apart() { + let dir = tempfile::tempdir().expect("tempdir"); + let (client_ca, client_ca_key) = generate_test_certs_with_ca(dir.path()); + let mut host_ca_params = CertificateParams::new(Vec::::new()).unwrap(); + host_ca_params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + host_ca_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + host_ca_params + .distinguished_name + .push(rcgen::DnType::CommonName, "supervisor-host-ca"); + let host_ca_key = openshell_crypto::pki::generate_keypair().unwrap(); + let host_ca = openshell_crypto::pki::self_signed(host_ca_params, &host_ca_key).unwrap(); + write_test_file(dir.path(), "host-ca.pem", host_ca.pem().as_bytes()); + let server_ca = dir.path().join("ca.pem"); + let acceptor = || { + TlsAcceptor::from_files( + &dir.path().join("server-cert.pem"), + &dir.path().join("server-key.pem"), + Some(&server_ca), + false, + None, + None, + Vec::new(), + ) + .expect("acceptor") + }; + let two_realms = acceptor() + .with_supervisor_host_ca(&dir.path().join("host-ca.pem")) + .expect("host CA"); + + assert_eq!( + classify_with(&two_realms, &server_ca, None).await.unwrap(), + PeerRealm::Anonymous + ); + assert_eq!( + classify_with( + &two_realms, + &server_ca, + Some(issue_leaf("openshell-client", &client_ca, &client_ca_key)), + ) + .await + .unwrap(), + PeerRealm::Client + ); + match classify_with( + &two_realms, + &server_ca, + Some(issue_leaf( + "openshell-supervisor-host", + &host_ca, + &host_ca_key, + )), + ) + .await + .unwrap() + { + PeerRealm::SupervisorHost(identity) => { + assert_eq!(identity.subject, "openshell-supervisor-host"); + assert_eq!(identity.certificate_sha256.len(), 64); + } + other => panic!("host certificate classified as {other:?}"), + } + + // Without a host CA, a host certificate is simply an unknown client. + let client_only = acceptor(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let handshake = client_only.handshake(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + handshake.accept(stream).await.map(|_| ()) + }); + let mut roots = rustls::RootCertStore::empty(); + for cert in load_certs(&server_ca).unwrap() { + roots.add(cert).unwrap(); + } + let (chain, key) = issue_leaf("openshell-supervisor-host", &host_ca, &host_ca_key); + let config = openshell_crypto::tls::client_builder() + .with_root_certificates(roots) + .with_client_auth_cert(chain, key) + .unwrap(); + let tcp = TcpStream::connect(address).await.unwrap(); + let client = tokio_rustls::TlsConnector::from(Arc::new(config)) + .connect( + rustls::pki_types::ServerName::try_from("localhost").unwrap(), + tcp, + ) + .await; + assert!( + client.is_err() || server.await.unwrap().is_err(), + "a host certificate must not pass a handshake that trusts only the client CA" + ); + + // The same CA for both realms makes every certificate ambiguous. + let ambiguous = acceptor() + .with_supervisor_host_ca(&server_ca) + .expect("overlapping CA"); + assert!( + classify_with( + &ambiguous, + &server_ca, + Some(issue_leaf("openshell-client", &client_ca, &client_ca_key)), + ) + .await + .is_err() + ); + } + #[tokio::test] async fn test_reload_mtls_ca_rotation() { let dir = tempfile::tempdir().expect("failed to create tempdir"); diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index f5ad4d3909..b5f08b699b 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -28,13 +28,8 @@ openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-mid openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } openshell-supervisor-process = { path = "../openshell-supervisor-process" } -bytes = { workspace = true } clap = { workspace = true } futures = { workspace = true } -http = { workspace = true } -http-body-util = { workspace = true } -hyper = { workspace = true } -hyper-util = { workspace = true } miette = { workspace = true } nix = { workspace = true } opentelemetry = { workspace = true } @@ -49,6 +44,7 @@ tempfile = "3" thiserror = { workspace = true } tokio = { workspace = true } tokio-rustls = { workspace = true } +tokio-stream = { workspace = true } tonic = { workspace = true, features = ["channel", "tls-native-roots"] } tracing = { workspace = true } tracing-appender = { workspace = true } @@ -64,6 +60,11 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } +bytes = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } tonic-prost = { workspace = true } tokio = { workspace = true, features = ["test-util"] } temp-env = "0.3" diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 9fe2429af0..a39cd85afd 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -123,38 +123,53 @@ backend or enable a Windows workload runtime. ## Shared supervisor host -`openshell-supervisor --mode=shared --gateway-endpoint --address ` hosts the sandbox instances of many sandboxes in -one process for the Kubernetes gateway supervisor topology. It is Unix-only. - -`--gateway-endpoint` selects the gateway each sandbox instance dials. The current -admission RPCs still arrive from the gateway at `--address`; moving admission -onto a supervisor-initiated stream is a separate implementation step. - -- **Admission.** A mutually authenticated TLS 1.3 listener - (`--address`, credentials in `--session-control-tls-dir`) - serves `PUT`, `GET`, and `DELETE /v1/sessions/{route}` and `GET /v1/sessions` - as JSON; these established session-control paths and fields address sandbox - instances. Only a client holding the admission certificate, which Helm gives - only the gateway, can register instances. A registration carries one - generation's runtime descriptor and gateway-issued credentials; the host - never acts with any other identity and has no Kubernetes identity. An - identical registration is accepted again; a different one for a registered - route returns `409`. -- **Sandbox instances.** `shared_host::SharedHost` validates each - registration and admits it into the `SandboxInstanceHost` registry, keyed by - route with the registration digest as its fingerprint. Its private files - live under `--session-root` (proxy CA, trust directory, SSH socket). The - registry owns removal, so it completes when the caller disconnects, and the - route stays reserved until the instance has ended and its files are - deleted. +`openshell-supervisor --mode=shared --gateway-endpoint --host-tls-dir --address ` +hosts the sandbox instances of many sandboxes in one process for the +Kubernetes gateway supervisor topology. It is Unix-only. The host opens no +control listener: it dials the gateway and receives its assignments from it. + +- **Gateway attachment.** The host dials `--gateway-endpoint` (which must be + `https`) with the client certificate in `--host-tls-dir` (`tls.crt`, + `tls.key`), verifying the gateway with `OPENSHELL_TLS_CA` and + `OPENSHELL_GATEWAY_TLS_SERVER_NAME` like every sandbox instance does, and + opens the private `openshell.supervisor.host.v1/SupervisorHost.Attach` + stream (`proto/supervisor_host.proto`). The certificate comes from a CA the + gateway trusts only for this stream; the host identity authorizes nothing + else, and the host never acts with any other identity and has no Kubernetes + identity. The stream starts with a hello carrying a per-process instance id + and a connection epoch; the gateway supersedes an older stream of the same + process and refuses a delayed older connection. +- **Assignments.** The gateway owns the desired set and sends it as complete + `AssignmentSnapshot`s with session-scoped revisions. `shared_host::SharedHost::reconcile` + validates the snapshot as a whole (at most 256 instances, no route twice), + decodes each assignment's launch material (the JSON + `SandboxInstanceRegistration` of `openshell-sandbox-backend`), admits it into + the `SandboxInstanceHost` registry keyed by route with the registration + digest as its fingerprint, and removes every hosted route the snapshot + omits. An identical assignment is a no-op; one that cannot be admitted + (invalid material, or a different fingerprint for a live route) is reported + as a failed instance and never touches the running one. Snapshots are + reconciled one at a time on a separate task, so heartbeats and status keep + flowing while instances stop; only the newest unapplied snapshot is kept, + and a connection finishes the reconciliation it started before the next + connection begins. Private files live under `--session-root` (proxy CA, + trust directory, SSH socket); the registry owns removal and keeps the route + reserved until the instance has ended and its files are deleted. +- **Status.** After each reconciliation and whenever an instance is admitted, + changes phase or readiness, or is removed, the host sends a complete + `HostStatus` naming the revision it has reconciled to; readiness is set only + once the gateway has accepted the instance's own supervisor session. +- **Disconnects.** Losing the gateway stream never stops an instance. The + host reconnects with backoff, reports what it runs, and applies the first + snapshot of the new session; only a snapshot that omits an instance, or + process shutdown, stops it. - **Runtime connections.** One routed listener (`--runtime-listen`) accepts every instance's runtime-initiated Sandbox Protocol connections; see `openshell-sandbox-backend` for the routing preface. -- **Health.** The chart's kubelet `tcpSocket` probes connect to the - session-control port (5501) and close without completing TLS, so they report - only that the process is listening; every request still requires mTLS. - Instance readiness is reported per instance over session control. SIGTERM - and SIGINT stop every instance, which ends their workloads' supervision. +- **Health.** `--address` is a plain TCP listener that accepts and closes + kubelet `tcpSocket` probe connections; it carries no control traffic and + reports only that the process runs. SIGTERM and SIGINT stop every instance, + which ends their workloads' supervision. Sandbox instances get no upstream proxy, extra CA bundle, or provider workload identity: the Kubernetes driver refuses the topology when an upstream proxy or provider diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index a13b0d5af5..11bdfd0885 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -38,7 +38,7 @@ use std::time::Duration; use tracing::{Instrument as _, debug, info, warn}; use openshell_core::PolicyValidationFailureMode; -use openshell_sandbox_backend::session_control::RegistrationTraceContext; +use openshell_sandbox_backend::sandbox_instance::RegistrationTraceContext; use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 7cee04d542..43cf982c95 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -64,13 +64,15 @@ struct Args { #[arg(long, required_if_eq("mode", "shared"))] gateway_endpoint: Option, - /// Shared supervisor listen address, also used for TCP health probes. + /// Shared supervisor TCP health listener: it accepts and closes probe + /// connections and carries no control traffic. #[arg(long, required_if_eq("mode", "shared"))] address: Option, - /// Directory with shared-host session-control TLS credentials. + /// Directory with the shared supervisor's host client certificate + /// (`tls.crt`, `tls.key`) for its outbound gateway connection. #[arg(long, required_if_eq("mode", "shared"))] - session_control_tls_dir: Option, + host_tls_dir: Option, /// Listener for routed runtime-initiated Sandbox Protocol connections. #[arg(long, required_if_eq("mode", "shared"))] @@ -405,7 +407,7 @@ fn main() -> Result<()> { } if args.gateway_endpoint.is_some() || args.address.is_some() - || args.session_control_tls_dir.is_some() + || args.host_tls_dir.is_some() || args.runtime_listen.is_some() || args.session_root.is_some() { diff --git a/crates/openshell-supervisor/src/sandbox_instance_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs index 508c32eba2..8e06713f35 100644 --- a/crates/openshell-supervisor/src/sandbox_instance_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -28,7 +28,7 @@ use std::time::Duration; use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use openshell_core::grpc_client::GatewayCredentials; use openshell_ocsf::OcsfJsonSettings; -use openshell_sandbox_backend::session_control::{ +use openshell_sandbox_backend::sandbox_instance::{ SandboxInstancePhaseReport, SandboxInstanceStatusReport, }; use openshell_supervisor_network::policy_local::{DenialLogSource, SandboxInstanceDenialLog}; @@ -187,6 +187,16 @@ impl HostedSandboxInstance { type Instances = Arc>>; +/// Wakes observers of the registry whenever any instance's status changes. +#[derive(Clone)] +struct ChangeNotifier(Arc>); + +impl ChangeNotifier { + fn notify(&self) { + self.0.send_modify(|changes| *changes += 1); + } +} + fn lock(instances: &Instances) -> MutexGuard<'_, HashMap> { instances .lock() @@ -205,6 +215,7 @@ pub struct SandboxInstanceHost { instances: Instances, next_serial: AtomicU64, root: PathBuf, + changes: ChangeNotifier, } impl SandboxInstanceHost { @@ -219,9 +230,17 @@ impl SandboxInstanceHost { instances: Arc::default(), next_serial: AtomicU64::new(0), root, + changes: ChangeNotifier(Arc::new(watch::Sender::new(0))), }) } + /// Notified whenever an instance is admitted, changes phase or readiness, + /// or is removed. The value only counts changes. + #[must_use] + pub fn changes(&self) -> watch::Receiver { + self.changes.0.subscribe() + } + /// Admit a sandbox instance and start the runtime `launch` prepares in /// the instance's new, empty private directory. /// @@ -268,7 +287,9 @@ impl SandboxInstanceHost { let started = prepare_instance_dir(&dir) .map_err(|error| E::from(SandboxInstanceHostError::Files(error.to_string()))) .and_then(|()| launch(&dir)) - .and_then(|(bindings, body)| spawn_instance(bindings, body).map_err(E::from)); + .and_then(|(bindings, body)| { + spawn_instance(bindings, body, self.changes.clone()).map_err(E::from) + }); let running = match started { Ok(running) => running, Err(error) => { @@ -290,6 +311,7 @@ impl SandboxInstanceHost { removal: None, }, ); + self.changes.notify(); Ok(SandboxInstanceAdmitted::Created) } @@ -372,9 +394,11 @@ impl SandboxInstanceHost { dir: self.root.join(key), grace, report, + changes: self.changes.clone(), } .run(), ); + self.changes.notify(); Some((removed, instance.sandbox_id.clone())) } } @@ -389,6 +413,7 @@ struct Removal { dir: PathBuf, grace: Duration, report: watch::Sender>, + changes: ChangeNotifier, } impl Removal { @@ -439,6 +464,7 @@ impl Removal { } } self.report.send_replace(Some(status)); + self.changes.notify(); } } @@ -454,6 +480,7 @@ struct RunningInstance { fn spawn_instance( bindings: SandboxInstanceBindings, body: SandboxInstanceBody, + changes: ChangeNotifier, ) -> std::result::Result { let (stop, stop_requested) = watch::channel(false); let (abort, abort_requested) = watch::channel(false); @@ -471,6 +498,7 @@ fn spawn_instance( abort_requested, phase: phase_tx, ready: ready_tx, + changes, }, ); }) @@ -492,6 +520,7 @@ struct SandboxInstanceSignals { abort_requested: watch::Receiver, phase: watch::Sender, ready: watch::Sender, + changes: ChangeNotifier, } /// Reports a hosted sandbox instance's end and revokes its credentials, @@ -501,6 +530,7 @@ struct SandboxInstanceExit { phase: watch::Sender, ready: watch::Sender, outcome: Option, + changes: ChangeNotifier, } impl Drop for SandboxInstanceExit { @@ -514,6 +544,7 @@ impl Drop for SandboxInstanceExit { error: "sandbox instance thread ended without reporting a result".to_string(), }); self.phase.send_replace(outcome); + self.changes.notify(); unbind_thread(); } } @@ -528,12 +559,14 @@ fn run_hosted_sandbox_instance( mut abort_requested, phase, ready, + changes, } = signals; let mut exit = SandboxInstanceExit { credentials: bindings.gateway_credentials.clone(), phase: phase.clone(), ready: ready.clone(), outcome: None, + changes: changes.clone(), }; bind_thread(bindings); let thread_bindings = bindings.clone(); @@ -551,12 +584,13 @@ fn run_hosted_sandbox_instance( let (readiness, mut accepted) = watch::channel(false); let mirror_phase = phase.clone(); let mirror_ready = ready.clone(); + let mirror_changes = changes.clone(); let mirror = tokio::spawn(async move { loop { let is_ready = *accepted.borrow_and_update(); - mirror_ready.send_replace(is_ready); - if is_ready { - mirror_phase.send_if_modified(|current| { + let changed = mirror_ready.send_replace(is_ready) != is_ready; + let promoted = is_ready + && mirror_phase.send_if_modified(|current| { let promote = *current == SandboxInstancePhaseReport::Starting; if promote { @@ -564,18 +598,24 @@ fn run_hosted_sandbox_instance( } promote }); + if changed || promoted { + mirror_changes.notify(); } if accepted.changed().await.is_err() { - mirror_ready.send_replace(false); + if mirror_ready.send_replace(false) { + mirror_changes.notify(); + } return; } } }); let mut stop = stop_requested.clone(); let stop_phase = phase.clone(); + let stop_changes = changes.clone(); let shutdown = Box::pin(async move { let _ = stop.wait_for(|requested| *requested).await; stop_phase.send_replace(SandboxInstancePhaseReport::Stopping); + stop_changes.notify(); }); let instance = body(SandboxInstanceControls { shutdown, diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 47182ef815..354bec7883 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -4,6 +4,8 @@ //! Supervisor host shared by the sandboxes of one gateway. use miette::Result; +#[cfg(unix)] +use std::future::Future; pub fn run(args: crate::Args) -> Result<()> { run_platform(args) @@ -22,9 +24,9 @@ fn run_platform(args: crate::Args) -> Result<()> { let gateway_endpoint = args .gateway_endpoint .ok_or_else(|| miette::miette!("--gateway-endpoint is required for --mode=shared"))?; - let session_control_tls_dir = args.session_control_tls_dir.ok_or_else(|| { - miette::miette!("--session-control-tls-dir is required for --mode=shared") - })?; + let host_tls_dir = args + .host_tls_dir + .ok_or_else(|| miette::miette!("--host-tls-dir is required for --mode=shared"))?; let runtime_listen = args .runtime_listen .ok_or_else(|| miette::miette!("--runtime-listen is required for --mode=shared"))?; @@ -32,6 +34,18 @@ fn run_platform(args: crate::Args) -> Result<()> { let instance_root = args .session_root .ok_or_else(|| miette::miette!("--session-root is required for --mode=shared"))?; + // The host verifies the gateway the way every sandbox instance does. + let gateway_ca = std::env::var_os(openshell_core::sandbox_env::TLS_CA) + .map(std::path::PathBuf::from) + .ok_or_else(|| { + miette::miette!( + "{} is required for --mode=shared", + openshell_core::sandbox_env::TLS_CA + ) + })?; + let gateway_server_name = std::env::var(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME) + .ok() + .filter(|name| !name.is_empty()); let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() @@ -61,65 +75,77 @@ fn run_platform(args: crate::Args) -> Result<()> { ) .init(); - // Install shutdown handlers before the control listener starts - // answering health probes. + // Install shutdown handlers before the health listener starts + // answering probes. let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) .into_diagnostic()?; let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()) .into_diagnostic()?; - let tls_file = |name: &str| { - let path = session_control_tls_dir.join(name); - std::fs::read(&path) - .into_diagnostic() - .wrap_err_with(|| format!("read session-control TLS file {}", path.display())) + let link = openshell_supervisor::shared_host::GatewayLink { + endpoint: gateway_endpoint.clone(), + host_tls_dir, + gateway_ca, + gateway_server_name, }; - let control_tls = openshell_sandbox_backend::session_control::server_tls_config( - &tls_file("tls.crt")?, - &tls_file("tls.key")?, - &tls_file("ca.crt")?, - ) - .map_err(|error| miette::miette!("{error}"))?; let host = openshell_supervisor::shared_host::SharedHost::start( openshell_supervisor::shared_host::SharedHostConfig { gateway_endpoint, instance_root, runtime_listen, - removal_grace: openshell_sandbox_backend::session_control::INSTANCE_REMOVAL_GRACE, + removal_grace: openshell_sandbox_backend::sandbox_instance::INSTANCE_REMOVAL_GRACE, }, )?; - let control = tokio::net::TcpListener::bind(address) + let health = tokio::net::TcpListener::bind(address) .await .into_diagnostic() - .wrap_err("bind session-control listener")?; + .wrap_err("bind health listener")?; tracing::info!( runtime = %host.runtime_addr(), - control = %address, - "Shared supervisor ready for sandbox instances" + health = %address, + "Shared supervisor ready for sandbox instance assignments" ); let (stop, stopped) = tokio::sync::watch::channel(false); - let serve = tokio::spawn(openshell_supervisor::shared_host::serve_session_control( + let wait_stopped = |mut stopped: tokio::sync::watch::Receiver| async move { + let _ = stopped.wait_for(|stop| *stop).await; + }; + let probes = tokio::spawn(serve_health(health, wait_stopped(stopped.clone()))); + let attachment = tokio::spawn(openshell_supervisor::shared_host::attach_to_gateway( host.clone(), - control, - control_tls, - async move { - let mut stopped = stopped; - let _ = stopped.wait_for(|stop| *stop).await; - }, + link, + wait_stopped(stopped), )); tokio::select! { _ = terminate.recv() => {} _ = interrupt.recv() => {} } stop.send_replace(true); - let _ = serve.await; + let _ = probes.await; + let _ = attachment.await; // Sandboxes lose supervision with this process and fail closed. host.remove_all().await; Ok(()) }) } +/// Answer kubelet `tcpSocket` probes: accept and close. Liveness means the +/// process runs; sandbox instance readiness is reported to the gateway. +#[cfg(unix)] +async fn serve_health(listener: tokio::net::TcpListener, shutdown: impl Future) { + tokio::pin!(shutdown); + loop { + tokio::select! { + () = &mut shutdown => break, + accepted = listener.accept() => { + if accepted.is_err() { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + } + } + } +} + #[cfg(not(unix))] fn run_platform(_args: crate::Args) -> Result<()> { Err(miette::miette!("--mode=shared requires Unix")) diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index cca0964014..caf1af599d 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -3,41 +3,67 @@ //! Supervisor shared by the sandboxes of one gateway. //! -//! A compute driver registers each sandbox instance, one running generation -//! of a stable Sandbox resource, over the session-control API, which accepts -//! only mutually authenticated TLS clients holding the admission certificate. -//! A registration carries exactly the gateway-issued credentials of that -//! generation; the host never acts with any other identity. This module only -//! validates registrations and turns them into launch material: the -//! [`SandboxInstanceHost`] registry admits, runs, and removes each instance, -//! whose runtime connections arrive through one routed listener. Removing a -//! registration stops only that instance. +//! The gateway owns which sandbox instances, one running generation of a +//! stable Sandbox resource each, this process hosts. The host dials the +//! gateway with its gateway-scoped host certificate, receives complete +//! assignment snapshots over the private `openshell.supervisor.host.v1` +//! stream, reconciles the [`SandboxInstanceHost`] registry to each one, and +//! reports what it observes. An assignment carries exactly the gateway-issued +//! credentials of its generation; the host never acts with any other identity +//! toward sandbox APIs. Losing the gateway stream never stops an instance; +//! only a later snapshot that omits it does. Snapshots are reconciled one at +//! a time, and a connection finishes the reconciliation it started before the +//! next connection begins, so a superseded stream can never apply after its +//! replacement. Runtime connections arrive through one routed listener. +use std::collections::BTreeMap; use std::future::Future; use std::net::SocketAddr; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::time::Duration; -use bytes::Bytes; -use http_body_util::{BodyExt as _, Full, Limited}; use miette::Result; +use openshell_core::proto::supervisor_host::v1::supervisor_host_client::SupervisorHostClient; +use openshell_core::proto::supervisor_host::v1::{ + AssignmentSnapshot, HostHeartbeat, HostHello, HostMessage, HostStatus, host_gateway_message, + host_message, +}; use openshell_sandbox_backend::runtime_routed::RuntimeRouter; -use openshell_sandbox_backend::session_control::{ - MAX_REGISTRATION_BYTES, SESSIONS_PATH, SandboxInstanceList, SandboxInstanceRegistration, - SandboxInstanceStatusReport, +use openshell_sandbox_backend::sandbox_instance::{ + SandboxInstancePhaseReport, SandboxInstanceRegistration, SandboxInstanceStatusReport, }; +use tokio::sync::watch; +use tokio_stream::StreamExt as _; use crate::sandbox_instance_host::{ SandboxInstanceAdmission, SandboxInstanceAdmitted, SandboxInstanceBody, SandboxInstanceHost, SandboxInstanceHostError, }; -/// Concurrent session-control connections; the only client is the gateway, -/// whose removals hold a connection while a workload stops. -const MAX_CONTROL_CONNECTIONS: usize = 64; -/// Bound on a TLS handshake and on reading one request. -const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(10); +/// Interval at which the host sends heartbeats on an idle stream. +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); +/// Silence after which the host abandons a stream and reconnects. +const GATEWAY_SILENCE_TIMEOUT: Duration = Duration::from_mins(1); +/// Bound on dialing the gateway, on the gateway answering the attach call, +/// and on it accepting the host. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// Bound on queueing one host-to-gateway message; a gateway that cannot +/// drain the stream this long is treated as lost. +const SEND_TIMEOUT: Duration = Duration::from_secs(30); +const RECONNECT_BACKOFF_MIN: Duration = Duration::from_secs(1); +const RECONNECT_BACKOFF_MAX: Duration = Duration::from_secs(30); +/// Host-to-gateway messages queued before the stream applies backpressure. +const OUTBOUND_QUEUE: usize = 16; +/// Most sandbox instances one snapshot may assign to a host; the gateway +/// enforces the same bound before it assigns. +pub const MAX_ASSIGNED_INSTANCES: usize = openshell_core::supervisor_host::MAX_ASSIGNED_INSTANCES; +/// Largest gateway message the host decodes: a full snapshot of +/// `MAX_ASSIGNED_INSTANCES` assignments at their largest, plus framing. +const MAX_GATEWAY_MESSAGE_BYTES: usize = MAX_ASSIGNED_INSTANCES + * (openshell_sandbox_backend::sandbox_instance::MAX_LAUNCH_MATERIAL_BYTES + 1024); +/// Largest host message: a status report is a few hundred bytes per instance. +const MAX_HOST_MESSAGE_BYTES: usize = 1024 * 1024; /// Process-level inputs of a shared supervisor. pub struct SharedHostConfig { @@ -48,25 +74,31 @@ pub struct SharedHostConfig { pub instance_root: PathBuf, /// Listener for runtime-initiated Sandbox Protocol connections. pub runtime_listen: SocketAddr, - /// Time a removed sandbox instance may take to stop its workload. + /// Time a withdrawn sandbox instance may take to stop its workload. pub removal_grace: Duration, } -/// Session-control adapter over the sandbox instance registry and the -/// routed runtime listener its instances claim routes on. +/// The sandbox instance registry and the routed runtime listener its +/// instances claim routes on, reconciled to the gateway's assignments. pub struct SharedHost { host: SandboxInstanceHost, router: Arc, gateway_endpoint: String, removal_grace: Duration, + /// Assignments of the latest snapshot that could not be admitted, reported + /// as failed instances until the gateway withdraws them. + rejected: Mutex>, + /// Routes of the latest applied snapshot, so a withdrawal whose cleanup + /// failed can be retried without the gateway resending the snapshot. + desired_routes: Mutex>>, } -/// A rejected session-control request. +/// A rejected sandbox instance assignment. #[derive(Debug, thiserror::Error)] pub enum ControlError { #[error("{0}")] Invalid(String), - #[error("route is already registered with different launch material")] + #[error("route is already hosted with different launch material")] Conflict, #[error("route is still being removed")] Removing, @@ -98,6 +130,8 @@ impl SharedHost { router, gateway_endpoint: config.gateway_endpoint, removal_grace: config.removal_grace, + rejected: Mutex::new(BTreeMap::new()), + desired_routes: Mutex::new(None), })) } @@ -125,18 +159,37 @@ impl SharedHost { .admit(admission, |dir| self.instance_launch(registration, dir)) } - /// Status of one registered sandbox instance. + /// Status of one hosted sandbox instance. #[must_use] pub fn status(&self, route: &str) -> Option { self.host.status(route) } - /// Status of every registered sandbox instance. + /// Status of every hosted sandbox instance. An assignment of the latest + /// snapshot that could not be admitted is reported as failed, even when + /// an earlier instance still runs under its route, so the gateway sees + /// the rejection rather than the instance it could not replace. #[must_use] - pub fn list(&self) -> SandboxInstanceList { - SandboxInstanceList { - sessions: self.host.list(), - } + pub fn list(&self) -> Vec { + let rejected = self + .rejected + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut reports = self + .host + .list() + .into_iter() + .filter(|hosted| !rejected.contains_key(&hosted.route)) + .collect::>(); + reports.extend(rejected.values().cloned()); + reports + } + + /// Notified whenever a hosted instance is admitted, changes phase or + /// readiness, or is removed. + #[must_use] + pub fn changes(&self) -> watch::Receiver { + self.host.changes() } /// Stop one sandbox instance, delete its private files, and forget it. @@ -149,6 +202,124 @@ impl SharedHost { self.host.remove_all(self.removal_grace).await; } + /// Make the registry match `snapshot`: admit every assignment and remove + /// every hosted instance it omits. Assignments that cannot be admitted are + /// reported as failed instances; they never affect a running instance. + /// + /// # Errors + /// + /// A malformed snapshot (too many instances, or one route assigned twice) + /// is refused as a whole before anything is admitted or removed. + pub async fn reconcile(&self, snapshot: &AssignmentSnapshot) -> Result<(), ControlError> { + if snapshot.instances.len() > MAX_ASSIGNED_INSTANCES { + return Err(ControlError::Invalid(format!( + "snapshot assigns {} instances; at most {MAX_ASSIGNED_INSTANCES} are supported", + snapshot.instances.len() + ))); + } + let mut routes = std::collections::BTreeSet::new(); + for assignment in &snapshot.instances { + if !routes.insert(assignment.route.clone()) { + return Err(ControlError::Invalid( + "snapshot assigns one route more than once".to_string(), + )); + } + } + *self + .desired_routes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(routes); + let mut rejected = BTreeMap::new(); + for assignment in &snapshot.instances { + let outcome = SandboxInstanceRegistration::from_assignment(assignment) + .map_err(|error| ControlError::Invalid(error.to_string())) + .and_then(|registration| self.register(registration)); + match outcome { + Ok(SandboxInstanceAdmitted::Created) => { + tracing::info!( + route = %assignment.route, + sandbox_id = %assignment.sandbox_id, + "admitted assigned sandbox instance" + ); + } + Ok(SandboxInstanceAdmitted::Unchanged) => {} + Err(error) => { + tracing::warn!( + route = %assignment.route, + sandbox_id = %assignment.sandbox_id, + %error, + "rejected assigned sandbox instance" + ); + rejected.insert( + assignment.route.clone(), + SandboxInstanceStatusReport { + route: assignment.route.clone(), + sandbox_id: assignment.sandbox_id.clone(), + phase: SandboxInstancePhaseReport::Failed { + error: error.to_string(), + }, + ready: false, + }, + ); + } + } + } + *self + .rejected + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = rejected; + self.remove_withdrawn().await; + Ok(()) + } + + /// Remove every hosted instance the latest applied snapshot omits. Also + /// run between snapshots: a removal whose file cleanup failed keeps its + /// key reserved and reports a failure, and the gateway does not resend an + /// unchanged snapshot, so this retries it. Nothing happens before the + /// first snapshot of a process, since nothing is known to be withdrawn. + pub async fn remove_withdrawn(&self) { + let desired = self + .desired_routes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let Some(desired) = desired else { + return; + }; + let withdrawn = self + .host + .list() + .into_iter() + .filter(|hosted| !desired.contains(&hosted.route)) + .collect::>(); + futures::future::join_all(withdrawn.iter().map(|hosted| async move { + tracing::info!( + route = %hosted.route, + sandbox_id = %hosted.sandbox_id, + "removing withdrawn sandbox instance" + ); + self.remove(&hosted.route).await; + })) + .await; + } + + /// Whether an instance the latest applied snapshot omits is still hosted, + /// such as one whose removal could not delete its private files. + #[must_use] + pub fn has_withdrawn_leftovers(&self) -> bool { + let desired = self + .desired_routes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(desired) = desired.as_ref() else { + return false; + }; + self.host + .list() + .iter() + .any(|hosted| !desired.contains(&hosted.route)) + } + /// Launch material of `registration` with its private files in `dir`. fn instance_launch( &self, @@ -258,164 +429,297 @@ fn write_private_file(_path: &Path, _contents: &[u8]) -> Result<()> { Err(miette::miette!("shared supervision requires a Unix host")) } -type ControlResponse = http::Response>; - -fn json_response(status: http::StatusCode, body: &impl serde::Serialize) -> ControlResponse { - let body = serde_json::to_vec(body).unwrap_or_default(); - http::Response::builder() - .status(status) - .header(http::header::CONTENT_TYPE, "application/json") - .body(Full::new(Bytes::from(body))) - .unwrap_or_default() +/// How the host reaches the gateway's supervisor host service. +pub struct GatewayLink { + /// Gateway gRPC endpoint; must use `https`, since the host authenticates + /// with its client certificate. + pub endpoint: String, + /// Directory holding the host client certificate `tls.crt` and `tls.key`. + pub host_tls_dir: PathBuf, + /// PEM CA bundle that issued the gateway's server certificate. + pub gateway_ca: PathBuf, + /// Server name expected in the gateway certificate, when it differs from + /// the endpoint host. + pub gateway_server_name: Option, } -fn error_response(status: http::StatusCode, message: &str) -> ControlResponse { - json_response(status, &serde_json::json!({ "error": message })) -} +impl GatewayLink { + fn channel_endpoint(&self) -> Result { + use miette::{IntoDiagnostic as _, WrapErr as _}; -async fn handle_control( - host: Arc, - request: http::Request, -) -> ControlResponse { - let path = request.uri().path().to_string(); - let route = path - .strip_prefix(SESSIONS_PATH) - .and_then(|rest| rest.strip_prefix('/')) - .filter(|route| !route.is_empty() && !route.contains('/')); - match (request.method().clone(), route) { - (http::Method::GET, None) if path == SESSIONS_PATH => { - json_response(http::StatusCode::OK, &host.list()) + if !self.endpoint.starts_with("https://") { + return Err(miette::miette!( + "the shared supervisor requires an https gateway endpoint for its host identity" + )); } - (http::Method::GET, Some(route)) => host.status(route).map_or_else( - || { - error_response( - http::StatusCode::NOT_FOUND, - "sandbox instance is not registered", - ) - }, - |status| json_response(http::StatusCode::OK, &status), - ), - (http::Method::PUT, Some(route)) => { - let route = route.to_string(); - let body = match tokio::time::timeout( - CONTROL_IO_TIMEOUT, - Limited::new(request.into_body(), MAX_REGISTRATION_BYTES).collect(), - ) - .await - { - Ok(Ok(body)) => body.to_bytes(), - _ => { - return error_response( - http::StatusCode::PAYLOAD_TOO_LARGE, - "registration body is invalid or too large", - ); - } - }; - let registration = match serde_json::from_slice::(&body) { - Ok(registration) if registration.route == route => registration, - Ok(_) => { - return error_response( - http::StatusCode::BAD_REQUEST, - "registration route does not match the request path", - ); - } - // Decoder errors can quote field values, including credentials. - Err(_) => { - return error_response( - http::StatusCode::BAD_REQUEST, - "registration body is not a valid sandbox instance registration", - ); - } - }; - match host.register(registration) { - Ok(outcome) => { - let status = if outcome == SandboxInstanceAdmitted::Created { - http::StatusCode::CREATED - } else { - http::StatusCode::OK - }; - host.status(&route).map_or_else( - || { - error_response( - http::StatusCode::CONFLICT, - "sandbox instance was removed", - ) - }, - |report| json_response(status, &report), - ) - } - Err(error @ ControlError::Invalid(_)) => { - error_response(http::StatusCode::BAD_REQUEST, &error.to_string()) - } - Err(error @ (ControlError::Conflict | ControlError::Removing)) => { - error_response(http::StatusCode::CONFLICT, &error.to_string()) - } - Err(error @ ControlError::Unavailable(_)) => { - error_response(http::StatusCode::SERVICE_UNAVAILABLE, &error.to_string()) - } - } + let read = |path: &Path| { + std::fs::read(path) + .into_diagnostic() + .wrap_err_with(|| format!("read {}", path.display())) + }; + let certificate = read(&self.host_tls_dir.join("tls.crt"))?; + let private_key = read(&self.host_tls_dir.join("tls.key"))?; + let gateway_ca = read(&self.gateway_ca)?; + let mut tls = tonic::transport::ClientTlsConfig::new() + .ca_certificate(tonic::transport::Certificate::from_pem(gateway_ca)) + .identity(tonic::transport::Identity::from_pem( + certificate, + private_key, + )); + if let Some(server_name) = self.gateway_server_name.as_deref() { + tls = tls.domain_name(server_name); } - (http::Method::DELETE, Some(route)) => host.remove(route).await.map_or_else( - || { - error_response( - http::StatusCode::NOT_FOUND, - "sandbox instance is not registered", - ) - }, - |status| json_response(http::StatusCode::OK, &status), - ), - _ => error_response(http::StatusCode::NOT_FOUND, "unknown session-control route"), + tonic::transport::Endpoint::from_shared(self.endpoint.clone()) + .into_diagnostic() + .wrap_err("invalid gateway endpoint")? + .connect_timeout(CONNECT_TIMEOUT) + .http2_keep_alive_interval(Duration::from_secs(10)) + .keep_alive_while_idle(true) + .keep_alive_timeout(Duration::from_secs(20)) + .tcp_nodelay(true) + .tls_config(tls) + .into_diagnostic() + .wrap_err("configure gateway TLS") + } +} + +/// Everything the host reports for one snapshot revision. +fn status_message(host: &SharedHost, applied_revision: u64) -> HostMessage { + HostMessage { + payload: Some(host_message::Payload::Status(HostStatus { + applied_revision, + instances: host + .list() + .iter() + .map(SandboxInstanceStatusReport::to_proto) + .collect(), + })), } } -/// Serve the session-control API until `shutdown` completes. Every -/// connection must complete mutually authenticated TLS before any request. -pub async fn serve_session_control( +/// Why one gateway stream ended. +enum StreamEnd { + /// The gateway rejected or closed the stream; reconnect after a backoff. + /// `accepted` records whether the gateway had accepted the host. + Lost { accepted: bool }, +} + +/// Keep one assignment stream open to the gateway until `shutdown` +/// completes, reconnecting with backoff. Hosted instances run on across +/// disconnects. +pub async fn attach_to_gateway( host: Arc, - listener: tokio::net::TcpListener, - tls: Arc, + link: GatewayLink, shutdown: impl Future, ) { - let acceptor = tokio_rustls::TlsAcceptor::from(tls); - let slots = Arc::new(tokio::sync::Semaphore::new(MAX_CONTROL_CONNECTIONS)); - let mut connections = tokio::task::JoinSet::new(); + let instance_id = uuid::Uuid::new_v4().to_string(); + let mut connection_epoch = 0_u64; + let mut backoff = RECONNECT_BACKOFF_MIN; tokio::pin!(shutdown); loop { + connection_epoch += 1; + let hello = HostHello { + instance_id: instance_id.clone(), + connection_epoch, + }; + let StreamEnd::Lost { accepted } = tokio::select! { + end = run_stream(&host, &link, hello) => end, + () = &mut shutdown => return, + }; + if accepted { + backoff = RECONNECT_BACKOFF_MIN; + } + tracing::info!( + retry_in_secs = backoff.as_secs(), + "gateway assignment stream lost; hosted sandbox instances keep running" + ); tokio::select! { - () = &mut shutdown => break, - accepted = listener.accept() => { - let Ok((stream, _)) = accepted else { - tokio::time::sleep(Duration::from_millis(100)).await; - continue; - }; - let Ok(slot) = slots.clone().try_acquire_owned() else { - continue; + () = tokio::time::sleep(backoff) => {} + () = &mut shutdown => return, + } + backoff = (backoff * 2).min(RECONNECT_BACKOFF_MAX); + } +} + +/// One connection and stream to the gateway. +/// +/// Snapshots are reconciled on a separate task so heartbeats and status +/// reports keep flowing while instances stop; at most one reconciliation runs +/// at a time, only the newest unapplied snapshot is kept, and the stream does +/// not end until a running reconciliation has finished. +async fn run_stream(host: &Arc, link: &GatewayLink, hello: HostHello) -> StreamEnd { + let lost = |accepted| StreamEnd::Lost { accepted }; + let endpoint = match link.channel_endpoint() { + Ok(endpoint) => endpoint, + Err(error) => { + tracing::error!(%error, "invalid gateway link configuration"); + return lost(false); + } + }; + let channel = match endpoint.connect().await { + Ok(channel) => channel, + Err(error) => { + tracing::warn!(%error, "could not connect to the gateway"); + return lost(false); + } + }; + let mut client = SupervisorHostClient::new(channel) + .max_decoding_message_size(MAX_GATEWAY_MESSAGE_BYTES) + .max_encoding_message_size(MAX_HOST_MESSAGE_BYTES); + let (tx, rx) = tokio::sync::mpsc::channel::(OUTBOUND_QUEUE); + let send = |message: HostMessage| { + let tx = tx.clone(); + async move { tx.send_timeout(message, SEND_TIMEOUT).await.is_ok() } + }; + if !send(HostMessage { + payload: Some(host_message::Payload::Hello(hello)), + }) + .await + { + return lost(false); + } + let attach = client.attach(tokio_stream::wrappers::ReceiverStream::new(rx)); + let mut inbound = match tokio::time::timeout(CONNECT_TIMEOUT, attach).await { + Ok(Ok(response)) => response.into_inner(), + Ok(Err(status)) => { + tracing::warn!(%status, "gateway refused the supervisor host stream"); + return lost(false); + } + Err(_) => { + tracing::warn!("gateway did not answer the supervisor host stream in time"); + return lost(false); + } + }; + let mut changes = host.changes(); + changes.mark_unchanged(); + let mut applied_revision = 0_u64; + // Highest snapshot revision received on this stream, applied or not, so a + // snapshot that arrives behind a newer in-flight one is never applied. + let mut received_revision = 0_u64; + let mut accepted = false; + let accept_deadline = tokio::time::Instant::now() + CONNECT_TIMEOUT; + let mut last_heard = tokio::time::Instant::now(); + let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // The reconciliation in flight and the newest snapshot waiting for it. + // A retry of withdrawals whose cleanup failed runs through the same slot, + // so at most one reconciliation touches the registry at a time. + let mut applying: Option, ControlError>>> = None; + let mut queued: Option = None; + let start = |snapshot: AssignmentSnapshot| { + let host = host.clone(); + tokio::spawn(async move { + host.reconcile(&snapshot).await?; + Ok(Some(snapshot.revision)) + }) + }; + let retry_withdrawn = || { + let host = host.clone(); + tokio::spawn(async move { + host.remove_withdrawn().await; + Ok(None) + }) + }; + let end = loop { + tokio::select! { + message = inbound.next() => { + let payload = match message { + Some(Ok(message)) => message.payload, + Some(Err(status)) => { + tracing::warn!(%status, "gateway assignment stream failed"); + break lost(accepted); + } + None => break lost(accepted), }; - openshell_core::net::set_tcp_nodelay_best_effort(&stream); - let acceptor = acceptor.clone(); - let host = host.clone(); - connections.spawn(async move { - let _slot = slot; - let Ok(Ok(stream)) = - tokio::time::timeout(CONTROL_IO_TIMEOUT, acceptor.accept(stream)).await - else { - tracing::debug!("rejected session-control connection without admission TLS"); - return; - }; - let service = hyper::service::service_fn(move |request| { - let host = host.clone(); - async move { - Ok::<_, std::convert::Infallible>(handle_control(host, request).await) + last_heard = tokio::time::Instant::now(); + match payload { + Some(host_gateway_message::Payload::Accepted(accepted_message)) => { + accepted = true; + tracing::info!(session = %accepted_message.session_id, "gateway accepted the supervisor host"); + // Report what runs before any snapshot arrives. + if !send(status_message(host, applied_revision)).await { + break lost(accepted); + } + } + Some(host_gateway_message::Payload::Rejected(rejected)) => { + tracing::error!(reason = %rejected.reason, "gateway rejected the supervisor host"); + break lost(accepted); + } + Some(host_gateway_message::Payload::Assignments(snapshot)) => { + if !accepted { + tracing::warn!("gateway sent assignments before accepting the host"); + break lost(accepted); + } + if snapshot.revision <= received_revision { + tracing::debug!(revision = snapshot.revision, received_revision, "ignoring stale assignment snapshot"); + continue; + } + received_revision = snapshot.revision; + if applying.is_some() { + queued = Some(snapshot); + } else { + applying = Some(start(snapshot)); + } + } + Some(host_gateway_message::Payload::Heartbeat(_)) | None => {} + } + } + applied = async { applying.as_mut().expect("polled only while applying").await }, if applying.is_some() => { + applying = None; + match applied { + Ok(Ok(revision)) => { + if let Some(revision) = revision { + applied_revision = revision; + } + changes.mark_unchanged(); + if !send(status_message(host, applied_revision)).await { + break lost(accepted); } - }); - let _ = hyper::server::conn::http1::Builder::new() - .timer(hyper_util::rt::TokioTimer::new()) - .header_read_timeout(CONTROL_IO_TIMEOUT) - .serve_connection(hyper_util::rt::TokioIo::new(stream), service) - .await; - }); + } + Ok(Err(error)) => { + tracing::warn!(%error, "refused a malformed assignment snapshot"); + } + Err(error) => { + tracing::error!(%error, "assignment reconciliation task failed"); + } + } + if let Some(snapshot) = queued.take() { + applying = Some(start(snapshot)); + } + } + notified = changes.changed() => { + if notified.is_err() { + break lost(accepted); + } + if accepted && !send(status_message(host, applied_revision)).await { + break lost(accepted); + } + } + _ = heartbeat.tick() => { + if !accepted && tokio::time::Instant::now() > accept_deadline { + tracing::warn!("gateway did not accept the supervisor host in time"); + break lost(accepted); + } + if last_heard.elapsed() > GATEWAY_SILENCE_TIMEOUT { + tracing::warn!("gateway assignment stream went silent; reconnecting"); + break lost(accepted); + } + let beat = HostMessage { payload: Some(host_message::Payload::Heartbeat(HostHeartbeat {})) }; + if !send(beat).await { + break lost(accepted); + } + // Withdrawn instances whose cleanup failed are retried at the + // heartbeat cadence, never concurrently with a snapshot. + if applying.is_none() && queued.is_none() && host.has_withdrawn_leftovers() { + applying = Some(retry_withdrawn()); + } } - Some(_) = connections.join_next(), if !connections.is_empty() => {} } + }; + // Finish the reconciliation this stream started before another stream + // can begin one; the queued snapshot belongs to a session that is gone. + if let Some(applying) = applying { + let _ = applying.await; } + end } diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index c1c2d511d8..8b375a08b5 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -197,8 +197,8 @@ spec: readOnly: true {{- end }} {{- if include "openshell.supervisorEnabled" . }} - - name: supervisor-control - mountPath: /etc/openshell-tls/supervisor-control + - name: supervisor-host-ca + mountPath: /etc/openshell-tls/supervisor-host readOnly: true {{- end }} {{- with .Values.server.extraVolumeMounts }} @@ -296,10 +296,14 @@ spec: path: ca.crt {{- end }} {{- if include "openshell.supervisorEnabled" . }} - # Admission credential for the shared supervisor's session-control API. - - name: supervisor-control + # CA of the shared supervisor's host certificate. Only the CA is projected; + # the host's private key stays with the supervisor. + - name: supervisor-host-ca secret: - secretName: {{ include "openshell.supervisorName" . }}-control-client + secretName: {{ include "openshell.supervisorName" . }}-host + items: + - key: ca.crt + path: ca.crt {{- end }} {{- if .Values.upstreamProxy.caBundle.configMapName }} - name: upstream-proxy-ca diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index f122b35c5a..070e64dbd5 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -363,7 +363,7 @@ raw gateway configuration so openshell.effectiveKubernetesConfig can use it. {{- end }} {{/* -Name of the shared supervisor Deployment, Service, and session-control Secrets: +Name of the shared supervisor Deployment, Service, and host identity Secret: the fullname with a fixed suffix, within the 63-character Service name limit. A fullname without room for the suffix keeps a prefix and a hash of the whole fullname, so the name never collapses into another release resource's name. @@ -617,9 +617,9 @@ like user-supplied schema-v2 fields during compatibility resolution. */}} {{- if not (hasKey $kubernetes "supervisor_image") -}}{{- $_ := set $kubernetes "supervisor_image" (include "openshell.supervisorImage" .) -}}{{- end -}} {{- if not (hasKey $runtimeConfig "boundary_port") -}}{{- $_ := set $runtimeConfig "boundary_port" 5500 -}}{{- end -}} {{- $_ := set $kubernetes "sandbox_runtime" $runtimeConfig -}} -{{/* The chart owns the shared supervisor and its session-control credential. */}} +{{/* The chart owns the shared supervisor and the CA of its host identity. */}} {{- if include "openshell.supervisorEnabled" . -}} -{{- $_ := set $kubernetes "shared_supervisor" (dict "service_name" (include "openshell.supervisorName" .) "service_namespace" .Release.Namespace "runtime_port" 5500 "control_port" 5501 "control_tls_dir" "/etc/openshell-tls/supervisor-control") -}} +{{- $_ := set $kubernetes "shared_supervisor" (dict "service_name" (include "openshell.supervisorName" .) "service_namespace" .Release.Namespace "runtime_port" 5500 "host_ca_file" "/etc/openshell-tls/supervisor-host/ca.crt") -}} {{- end -}} {{- toYaml $kubernetes -}} {{- end }} @@ -644,6 +644,9 @@ Validate chart values that Helm would otherwise accept silently. {{- if or (get $kubernetes "https_proxy") (get $kubernetes "proxy_ca_bundle") (get $kubernetes "proxy_auth_secret_name") (get $kubernetes "provider_spiffe_workload_api_socket_path") -}} {{- fail "supervisor_topology=gateway does not support upstream proxy settings or provider SPIFFE identity" -}} {{- end -}} +{{- if .Values.server.disableTls -}} +{{- fail "supervisor_topology=gateway requires gateway TLS; the shared supervisor authenticates with a client certificate" -}} +{{- end -}} {{- if and (eq (get $kubernetes "workspace_mode" | default "shared") "operator") (not (regexMatch "^[A-Za-z0-9][-A-Za-z0-9_./]*==?[-A-Za-z0-9_.]*$" (get $kubernetes "operator_namespace_label" | default ""))) -}} {{- fail "supervisor_topology=gateway in operator workspace mode requires operator_namespace_label of the form key=value" -}} {{- end -}} diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index d4b5a7aa81..0ebc2bf7e7 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -124,8 +124,9 @@ spec: {{- end }} {{- if $supervisor }} {{- $supervisorName := include "openshell.supervisorName" . }} - # Private PKI for the shared supervisor's session-control API. Only - # the gateway receives its client credential. + # Private PKI for the shared supervisor's host identity: the supervisor + # presents the client certificate to the gateway, which trusts only + # the CA. - name: supervisor-certgen image: {{ include "openshell.image" . | quote }} imagePullPolicy: {{ .Values.gateway.image.pullPolicy | default .Values.global.image.pullPolicy }} @@ -139,10 +140,8 @@ spec: command: ["/usr/local/bin/openshell-gateway"] args: - generate-certs - - --tls-only - - --server-secret-name={{ $supervisorName }}-control - - --client-secret-name={{ $supervisorName }}-control-client - - --server-san={{ $supervisorName }}.{{ $ns }}.svc + - --client-only + - --client-secret-name={{ $supervisorName }}-host {{- end }} {{- if and .Values.certManager.enabled .Values.grpcRoute.backendTLSPolicy.enabled }} --- diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index eb31b93f55..d8cbb028fa 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -12,11 +12,11 @@ {{- $endpoint := get $kubernetes "grpc_endpoint" }} {{- $endpointUrl := urlParse $endpoint }} {{- $gatewayTls := and (not .Values.server.disableTls) .Values.server.tls.clientTlsSecretName }} -# One supervisor process hosting a session per sandbox generation. Sessions -# are registered only by the gateway, over a mutually authenticated API whose -# client credential only the gateway mounts. The Pod holds no Kubernetes or -# gateway identity of its own: each session acts only with the credentials -# the gateway issued for its generation. +# One supervisor process hosting a sandbox instance per sandbox generation. +# The process dials the gateway with its host certificate and receives its +# assignments from it; nothing but sandbox runtimes and kubelet probes connect +# to it. The Pod holds no Kubernetes identity of its own: each instance acts +# only with the credentials the gateway issued for its generation. apiVersion: apps/v1 kind: Deployment metadata: @@ -26,7 +26,7 @@ metadata: {{- include "openshell.labels" . | nindent 4 }} app.kubernetes.io/component: supervisor spec: - # Sessions live in this process; a second replica could not serve them. + # Instances live in this process; a second replica could not serve them. replicas: 1 strategy: type: Recreate @@ -60,9 +60,9 @@ spec: - --gateway-endpoint - {{ $endpoint | quote }} - --address - - 0.0.0.0:{{ get $shared "control_port" }} - - --session-control-tls-dir - - /etc/openshell/session-control + - 0.0.0.0:5501 + - --host-tls-dir + - /etc/openshell/supervisor-host - --runtime-listen - 0.0.0.0:{{ get $shared "runtime_port" }} - --session-root @@ -84,8 +84,8 @@ spec: - name: runtime containerPort: {{ get $shared "runtime_port" }} protocol: TCP - - name: session-control - containerPort: {{ get $shared "control_port" }} + - name: health + containerPort: 5501 protocol: TCP securityContext: allowPrivilegeEscalation: false @@ -98,22 +98,22 @@ spec: memory: 64Mi limits: memory: 1Gi - # Process health only: the probes open and close a TCP connection - # to the mTLS session-control listener. Each session reports its own - # readiness to the gateway through the session-control API. + # Process health only: the health listener accepts and closes probe + # connections. Each instance's readiness is reported to the gateway + # over the supervisor's own connection to it. readinessProbe: tcpSocket: - port: session-control + port: health livenessProbe: tcpSocket: - port: session-control + port: health volumeMounts: - name: run mountPath: /run/openshell - name: logs mountPath: /var/log - - name: session-control - mountPath: /etc/openshell/session-control + - name: supervisor-host + mountPath: /etc/openshell/supervisor-host readOnly: true {{- if $gatewayTls }} - name: gateway-ca @@ -125,9 +125,9 @@ spec: emptyDir: {} - name: logs emptyDir: {} - - name: session-control + - name: supervisor-host secret: - secretName: {{ $name }}-control + secretName: {{ $name }}-host {{- if $gatewayTls }} - name: gateway-ca secret: @@ -153,17 +153,13 @@ spec: port: {{ get $shared "runtime_port" }} targetPort: runtime protocol: TCP - - name: session-control - port: {{ get $shared "control_port" }} - targetPort: session-control - protocol: TCP {{- if .Values.networkPolicy.enabled }} --- -# Only the gateway reaches session control, and only sandbox workloads in this -# gateway's sandbox namespaces reach the routed runtime listener. Each -# sandbox's own egress allowance limits its workload to that port, and the -# routing preface plus the generation's pinned TLS identity decide which -# session it reaches. +# Only sandbox workloads in this gateway's sandbox namespaces reach the routed +# runtime listener. Each sandbox's own egress allowance limits its workload to +# that port, and the routing preface plus the generation's pinned TLS identity +# decide which instance it reaches. The supervisor dials the gateway itself, +# so no ingress from the gateway is needed. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -180,13 +176,6 @@ spec: - Ingress - Egress ingress: - - from: - - podSelector: - matchLabels: - {{- include "openshell.selectorLabels" . | nindent 14 }} - ports: - - protocol: TCP - port: {{ get $shared "control_port" }} - from: - namespaceSelector: {{- include "openshell.supervisorSandboxNamespaces" (dict "kubernetes" $kubernetes) | nindent 12 }} @@ -196,7 +185,8 @@ spec: ports: - protocol: TCP port: {{ get $shared "runtime_port" }} - # Sessions carry policy-approved sandbox egress, as dedicated supervisors do. + # Instances carry policy-approved sandbox egress, as dedicated supervisors + # do, and the host reaches the gateway. egress: - {} {{- end }} diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index e85713e2e1..db8dd4b1b7 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -13,7 +13,7 @@ tests: asserts: - hasDocuments: count: 0 - - it: deploys one session host without Kubernetes identity in the gateway namespace + - it: deploys one instance host without Kubernetes identity in the gateway namespace template: templates/supervisor.yaml set: gatewayConfig: @@ -51,8 +51,8 @@ tests: - https://openshell.gateway.svc.cluster.local:8080 - --address - 0.0.0.0:5501 - - --session-control-tls-dir - - /etc/openshell/session-control + - --host-tls-dir + - /etc/openshell/supervisor-host - --runtime-listen - 0.0.0.0:5500 - --session-root @@ -66,13 +66,13 @@ tests: path: spec.template.spec.containers[0].readinessProbe value: tcpSocket: - port: session-control + port: health documentIndex: 0 - equal: path: spec.template.spec.containers[0].livenessProbe value: tcpSocket: - port: session-control + port: health documentIndex: 0 - contains: path: spec.template.spec.containers[0].volumeMounts @@ -80,9 +80,13 @@ tests: name: run mountPath: /run/openshell documentIndex: 0 - - lengthEqual: + - equal: path: spec.ports - count: 2 + value: + - name: runtime + port: 5500 + targetPort: runtime + protocol: TCP documentIndex: 1 - equal: path: spec.template.spec.automountServiceAccountToken @@ -103,9 +107,9 @@ tests: - contains: path: spec.template.spec.volumes content: - name: session-control + name: supervisor-host secret: - secretName: openshell-supervisor-control + secretName: openshell-supervisor-host documentIndex: 0 - contains: path: spec.template.spec.volumes @@ -132,20 +136,12 @@ tests: path: kind value: NetworkPolicy documentIndex: 2 - - equal: - path: spec.ingress[0] - value: - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: openshell - app.kubernetes.io/instance: openshell - ports: - - protocol: TCP - port: 5501 + - lengthEqual: + path: spec.ingress + count: 1 documentIndex: 2 - equal: - path: spec.ingress[1] + path: spec.ingress[0] value: from: - namespaceSelector: @@ -168,7 +164,7 @@ tests: gateway_id: gw-a asserts: - equal: - path: spec.ingress[1].from[0].namespaceSelector + path: spec.ingress[0].from[0].namespaceSelector value: matchLabels: openshell.ai/managed-by: openshell @@ -184,7 +180,7 @@ tests: operator_namespace_label: example.com/openshell-workspace=true asserts: - equal: - path: spec.ingress[1].from[0].namespaceSelector + path: spec.ingress[0].from[0].namespaceSelector value: matchLabels: example.com/openshell-workspace: "true" @@ -216,7 +212,7 @@ tests: path: metadata.name value: openshell-gateway-release-with-a-very-very-long-name-0057 documentIndex: 1 - - it: omits the gateway CA when gateway TLS is disabled + - it: requires gateway TLS for the host identity template: templates/supervisor.yaml set: server: @@ -225,18 +221,8 @@ tests: openshell.drivers.kubernetes: supervisor_topology: gateway asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: OPENSHELL_TLS_CA - value: /var/run/secrets/openshell-tls/ca.crt - documentIndex: 0 - - contains: - path: spec.template.spec.containers[0].env - content: - name: OPENSHELL_ENDPOINT - value: http://openshell.gateway.svc.cluster.local:8080 - documentIndex: 0 + - failedTemplate: + errorMessage: supervisor_topology=gateway requires gateway TLS; the shared supervisor authenticates with a client certificate - it: renders the chart-owned shared supervisor connection into gateway TOML template: templates/gateway-config.yaml set: @@ -249,7 +235,7 @@ tests: pattern: 'supervisor_topology = "gateway"' - matchRegex: path: data["gateway.toml"] - pattern: 'shared_supervisor = \{ control_port = 5501, control_tls_dir = "/etc/openshell-tls/supervisor-control", runtime_port = 5500, service_name = "openshell-supervisor", service_namespace = "gateway" \}' + pattern: 'shared_supervisor = \{ host_ca_file = "/etc/openshell-tls/supervisor-host/ca.crt", runtime_port = 5500, service_name = "openshell-supervisor", service_namespace = "gateway" \}' - it: rejects a hand-written shared supervisor connection template: templates/supervisor.yaml set: diff --git a/deploy/helm/openshell/tests/supervisor_wiring_test.yaml b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml index aca63a5963..0cf1937bd3 100644 --- a/deploy/helm/openshell/tests/supervisor_wiring_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml @@ -10,7 +10,7 @@ release: name: openshell namespace: gateway tests: - - it: issues the session-control PKI with the hook + - it: issues the supervisor host identity with the hook template: templates/certgen.yaml set: gatewayConfig: @@ -21,12 +21,10 @@ tests: path: spec.template.spec.containers[1].args value: - generate-certs - - --tls-only - - --server-secret-name=openshell-supervisor-control - - --client-secret-name=openshell-supervisor-control-client - - --server-san=openshell-supervisor.gateway.svc + - --client-only + - --client-secret-name=openshell-supervisor-host documentIndex: 3 - - it: issues only the session-control PKI when the gateway PKI is external + - it: issues only the supervisor host identity when the gateway PKI is external template: templates/certgen.yaml set: pkiInitJob: @@ -62,7 +60,7 @@ tests: verbs: - get documentIndex: 0 - - it: mounts the session-control client credential only on the gateway + - it: projects only the supervisor host CA onto the gateway template: templates/statefulset.yaml set: gatewayConfig: @@ -72,12 +70,15 @@ tests: - contains: path: spec.template.spec.volumes content: - name: supervisor-control + name: supervisor-host-ca secret: - secretName: openshell-supervisor-control-client + secretName: openshell-supervisor-host + items: + - key: ca.crt + path: ca.crt - contains: path: spec.template.spec.containers[0].volumeMounts content: - name: supervisor-control - mountPath: /etc/openshell-tls/supervisor-control + name: supervisor-host-ca + mountPath: /etc/openshell-tls/supervisor-host readOnly: true diff --git a/proto/README.md b/proto/README.md index 61fc8d6456..663446e147 100644 --- a/proto/README.md +++ b/proto/README.md @@ -86,6 +86,19 @@ identifies the service being exposed. Run `mise run pre-commit`, the affected SDK checks, and relevant server tests before submitting the change. +## Supervisor host protocol + +[`supervisor_host.proto`](supervisor_host.proto) is the private stream between +a gateway and the supervisor process it shares among its sandboxes: the +supervisor attaches, the gateway sends complete assignment snapshots, and the +supervisor reports what it hosts. It is a version-locked deployment protocol +served on the gateway listener but authorized only for the supervisor host +certificate. It is not imported by `openshell.proto`, so SDK generation and +gRPC reflection exclude it; the server's schema inventory test classifies it +explicitly. `SandboxInstanceAssignment.launch_material` carries the existing +JSON sandbox instance registration of `openshell-sandbox-backend` and is +marked secret. + ## Isolation backend protocol [`isolation_backend.proto`](isolation_backend.proto) defines backend discovery, diff --git a/proto/supervisor_host.proto b/proto/supervisor_host.proto new file mode 100644 index 0000000000..4c31decaa4 --- /dev/null +++ b/proto/supervisor_host.proto @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.supervisor.host.v1; + +import "google/protobuf/duration.proto"; +import "options.proto"; + +// Private contract between a gateway and the supervisor process it shares +// among its sandboxes. It is a version-locked deployment protocol, not a +// public API: the gateway, the Kubernetes compute driver, and the shared +// supervisor are released together, and no SDK generates bindings for it. +// +// The supervisor host opens one Attach stream, identifies itself, and keeps +// the stream for its lifetime. The gateway owns the desired set of sandbox +// instances and sends it as complete snapshots; the host reconciles its +// registry to each snapshot and reports what it observes. Disconnecting +// never stops hosted instances; only a later snapshot that omits them does. +service SupervisorHost { + // Supervisor-initiated assignment stream. Only a gateway-scoped supervisor + // host certificate may call it. + rpc Attach(stream HostMessage) returns (stream HostGatewayMessage) { + option (openshell.options.v1.authorization) = { + auth_mode: "supervisor_host" + }; + } +} + +// Envelope for host-to-gateway messages. +message HostMessage { + oneof payload { + HostHello hello = 1; + HostStatus status = 2; + HostHeartbeat heartbeat = 3; + } +} + +// Envelope for gateway-to-host messages. +message HostGatewayMessage { + oneof payload { + HostAccepted accepted = 1; + HostRejected rejected = 2; + AssignmentSnapshot assignments = 3; + HostGatewayHeartbeat heartbeat = 4; + } +} + +// First message of every stream. +message HostHello { + // Identifies one host process (for example a boot id). A reconnect from the + // same process keeps it; a restarted process has a new one. + string instance_id = 1; + // Counter the host increments for every connection of one instance_id, so + // the gateway can order reconnects. + uint64 connection_epoch = 2; +} + +// The gateway accepted the host and will send assignment snapshots. +message HostAccepted { + // Scopes every snapshot revision and status report of this stream. + string session_id = 1; + google.protobuf.Duration heartbeat_interval = 2; +} + +// The gateway refuses the host; the stream ends after this message. +message HostRejected { + string reason = 1; +} + +// Complete desired set of sandbox instances. Each snapshot replaces the +// previous one; an instance absent from a snapshot must stop. +message AssignmentSnapshot { + // Monotonic within the session, starting at 1. + uint64 revision = 1; + repeated SandboxInstanceAssignment instances = 2; +} + +// One running generation of a stable Sandbox resource, assigned to the host. +message SandboxInstanceAssignment { + // Route handle the sandbox runtime names; the instance's key on the host. + string route = 1; + string sandbox_id = 2; + // Runtime generation this assignment launches. Never reused. + string generation = 3; + // Trusted launch material for the generation: the JSON-encoded sandbox + // instance registration (runtime descriptor, gateway-issued credentials, + // proxy CA, main process). Never logged or forwarded. + bytes launch_material = 4 [(openshell.options.v1.secret) = true]; +} + +// Everything the host observes, sent after reconciling each snapshot and +// whenever an instance changes. +message HostStatus { + // Highest snapshot revision of this session the host has reconciled to; + // 0 before the first one. + uint64 applied_revision = 1; + // Every hosted instance, including ones that ended but are still being + // removed. + repeated SandboxInstanceStatus instances = 2; +} + +// Lifecycle phase of a hosted sandbox instance. +enum SandboxInstancePhase { + SANDBOX_INSTANCE_PHASE_UNSPECIFIED = 0; + SANDBOX_INSTANCE_PHASE_STARTING = 1; + SANDBOX_INSTANCE_PHASE_SUPERVISING = 2; + SANDBOX_INSTANCE_PHASE_STOPPING = 3; + SANDBOX_INSTANCE_PHASE_EXITED = 4; + SANDBOX_INSTANCE_PHASE_FAILED = 5; +} + +// Observed state of one hosted sandbox instance. +message SandboxInstanceStatus { + string route = 1; + string sandbox_id = 2; + SandboxInstancePhase phase = 3; + // The gateway currently accepts this instance's supervision session. + bool ready = 4; + // Workload exit code; meaningful only for EXITED. + int32 exit_code = 5; + // Sanitized failure; meaningful only for FAILED. + string error = 6; +} + +message HostHeartbeat {} + +message HostGatewayHeartbeat {} diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 1dab79be80..c038b93020 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1280,6 +1280,7 @@ credential failures. | Gateway or `-certgen` hook pod rejected with `violates PodSecurity "restricted:latest"` | The rendered container lacks `runAsNonRoot` or `seccompProfile`: an older chart version does not set them, or an override in `securityContext` / `pkiInitJob.securityContext` unset them | `kubectl -n openshell get events --field-selector reason=FailedCreate`; upgrade the chart, then remove overrides that unset those fields | | OpenShift gateway pod fails to start with an SCC/`runAsUser` error (e.g. `unable to validate against any security context constraint`) | Chart's default `podSecurityContext`/`securityContext` hardcodes `runAsUser`/`fsGroup`, which the restricted-v2 SCC rejects; it must instead inject the namespace-assigned UID/GID range | `oc -n openshell describe pod `; deploy with `podSecurityContext: null` and clear `securityContext.runAsUser` (see `deploy/helm/openshell/ci/values-openshift-scc.yaml`) | | OpenShift `-supervisor` Deployment never creates pods after setting `supervisor_topology = "gateway"` (SCC `runAsUser`/`fsGroup` not in the namespace range) | `supervisor.podSecurityContext` defaults `runAsUser`, `runAsGroup`, and `fsGroup` to 65534, outside the namespace's assigned UID/GID ranges | `kubectl -n openshell get events --field-selector reason=FailedCreate`; compare with the `openshift.io/sa.scc.*` annotations from `oc describe ns openshell`; set `supervisor.podSecurityContext.runAsUser`, `.runAsGroup`, and `.fsGroup` to `null`, as `deploy/helm/openshell/ci/values-openshift-scc.yaml` does, and keep `runAsNonRoot` and `seccompProfile` | +| Shared sandboxes stay `Starting` or report `unavailable` readiness with `supervisor_topology = "gateway"`, gateway logs say no shared supervisor host is connected | The `-supervisor` Pod cannot reach or authenticate to the gateway: wrong `OPENSHELL_TLS_CA`, missing `-supervisor-host` Secret, or the gateway was deployed without TLS | `kubectl -n openshell logs deployment/-supervisor` for `gateway accepted the supervisor host` or a refusal; gateway logs for `supervisor host attached`; `kubectl -n openshell get secret -supervisor-host`; the gateway refuses to start the topology without TLS | | OpenShift sandbox pod fails to start (`unable to validate against any security context constraint`) | The `openshell-sandbox` service account lacks the privileged SCC it needs | `oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell`; remove with `remove-scc-from-user` when done | | OpenShift self-hosted Vault/OpenBao credential store pod never schedules (waits time out with `no matching resources found`) | The store's Helm chart pins `runAsUser`/`fsGroup`/seccomp, which restricted-v2 rejects, so the StatefulSet controller never creates the pod | Deploy the store's chart in its OpenShift mode (`--set global.openshift=true` for the OpenBao/Vault chart) so the namespace SCC assigns a compliant security context — no manual SCC grant needed | | Vault credential driver returns HTTP 403 / `Vault Kubernetes auth denied the configured role` on provider create | Vault's `auth/kubernetes` method or the gateway login role is not provisioned, or the role is not bound to the gateway service account and namespace | In Vault: `bao auth enable kubernetes` and `bao write auth/kubernetes/config kubernetes_host=... kubernetes_ca_cert=@...`; ensure the login role's `bound_service_account_names`/`bound_service_account_namespaces` match the gateway SA and namespace and its policy grants the credential paths | From d954404149a4561d60a7f6700d34354fdfeaa64b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 21 Oct 2026 23:18:01 -0700 Subject: [PATCH 23/25] refactor(supervisor): authenticate the shared supervisor stream with a gateway-minted supervisor token Replace the shared supervisor's host CA and client certificate with a supervisor token: a bearer the gateway mints from its existing launch-signing key under a profile of its own (typ openshell-supervisor+jwt, subject openshell-supervisor:, no sandbox, generation, or epoch claims). The supervisor dials the gateway's Attach stream over ordinary server-authenticated TLS and presents the token; the gateway verifies it with the same key infrastructure, grants only the assignment stream, refuses every other credential on it, and ends a stream when the token that opened it expires. The gateway renews the token at half its lifetime and publishes each renewal into the supervisor host controller; the Kubernetes driver's only part in authentication is delivery, replacing the token in the chart-created -supervisor-token Secret, which the supervisor mounts and re-reads on every connection attempt. The gateway Role may get and patch only that Secret. Remove the host-only TLS realm classification, the supervisor host CA mount and config, certgen --client-only, and the supervisor host PKI; gateway TLS returns to its pre-feature behavior and remains required for the topology. Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 25 +- crates/openshell-bootstrap/src/pki.rs | 27 +- crates/openshell-core/src/grpc_client.rs | 9 + crates/openshell-core/src/jwt.rs | 429 +++++++++++++++-- crates/openshell-core/src/lib.rs | 1 + crates/openshell-core/src/supervisor_host.rs | 91 +++- crates/openshell-driver-kubernetes/Cargo.toml | 2 +- crates/openshell-driver-kubernetes/README.md | 25 +- .../openshell-driver-kubernetes/src/config.rs | 31 +- .../openshell-driver-kubernetes/src/driver.rs | 3 + .../src/shared_supervisor.rs | 119 ++++- crates/openshell-server/src/auth/audit.rs | 9 +- .../src/auth/authenticator.rs | 8 +- .../src/auth/descriptor_authz.rs | 2 +- crates/openshell-server/src/auth/guard.rs | 6 +- .../src/auth/launch_signing.rs | 1 + .../openshell-server/src/auth/method_authz.rs | 23 +- crates/openshell-server/src/auth/mod.rs | 1 + crates/openshell-server/src/auth/principal.rs | 16 +- .../openshell-server/src/auth/sandbox_jwt.rs | 50 +- .../src/auth/supervisor_token.rs | 187 ++++++++ .../src/auth/workspace_authz.rs | 8 +- crates/openshell-server/src/certgen.rs | 99 +--- crates/openshell-server/src/grpc/policy.rs | 4 +- crates/openshell-server/src/grpc/provider.rs | 2 +- .../src/grpc/provider_credentials_tests.rs | 1 - .../src/grpc/supervisor_host.rs | 11 +- crates/openshell-server/src/grpc/workspace.rs | 4 +- crates/openshell-server/src/lib.rs | 135 ++++-- crates/openshell-server/src/multiplex.rs | 132 +++-- crates/openshell-server/src/tls.rs | 451 +----------------- crates/openshell-supervisor/README.md | 26 +- crates/openshell-supervisor/src/main.rs | 9 +- crates/openshell-supervisor/src/shared.rs | 36 +- .../openshell-supervisor/src/shared_host.rs | 107 ++--- .../openshell/templates/_gateway-workload.tpl | 15 - deploy/helm/openshell/templates/_helpers.tpl | 8 +- deploy/helm/openshell/templates/certgen.yaml | 24 +- .../helm/openshell/templates/peer-role.yaml | 11 + .../helm/openshell/templates/supervisor.yaml | 35 +- .../helm/openshell/tests/supervisor_test.yaml | 41 +- .../tests/supervisor_wiring_test.yaml | 78 +-- proto/README.md | 5 +- proto/supervisor_host.proto | 6 +- skills/debug-openshell-cluster/SKILL.md | 2 +- 45 files changed, 1312 insertions(+), 1003 deletions(-) create mode 100644 crates/openshell-server/src/auth/supervisor_token.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 9c294000ca..0b5a069eeb 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -69,18 +69,19 @@ gatewayConfig: ``` The gateway chart creates a single-replica shared supervisor Deployment, -Service, and NetworkPolicy in the release namespace, regardless of -shared/managed/operator workspace mode, and the certgen hook issues the -`-supervisor-host` Secret: a private CA and the client certificate -the supervisor presents to the gateway. The gateway mounts only that Secret's -`ca.crt` and accepts the certificate solely for the private -`openshell.supervisor.host.v1` assignment stream; the topology therefore -requires gateway TLS (`server.disableTls` is rejected). New sandbox generations -run as sandbox instances in that process. The chart starts `--mode=shared`, -sets `--gateway-endpoint` to the gateway gRPC URL, mounts the host certificate -at `--host-tls-dir`, and binds `--address` (5501) to a plain TCP health -listener for the `tcpSocket` probes; nothing dials the supervisor but sandbox -runtimes (5500) and the kubelet. Build the gateway, supervisor, and sandbox +Service, NetworkPolicy, and an empty `-supervisor-token` Secret in +the release namespace, regardless of shared/managed/operator workspace mode. +The gateway mints a supervisor token from its sandbox-JWT signing key, writes +it into that Secret (the gateway Role may `get`/`patch` only that Secret), and +renews it at half its lifetime; the supervisor mounts the Secret and presents +the token on the private `openshell.supervisor.host.v1` assignment stream, +which accepts nothing else. The topology requires gateway TLS and the signing +key (`server.disableTls` is rejected; the gateway refuses to start without +`gateway_jwt`). New sandbox generations run as sandbox instances in that +process. The chart starts `--mode=shared`, sets `--gateway-endpoint` to the +gateway gRPC URL, mounts the token at `--token-file`, and binds `--address` +(5501) to a plain TCP health listener for the `tcpSocket` probes; nothing +dials the supervisor but sandbox runtimes (5500) and the kubelet. Build the gateway, supervisor, and sandbox images from the same branch: all three must support the routed runtime connection and the assignment stream. `workspace` is reserved and rejected. Omit the field to disable it. `mise run diff --git a/crates/openshell-bootstrap/src/pki.rs b/crates/openshell-bootstrap/src/pki.rs index a2bd5ea006..4ade3465f5 100644 --- a/crates/openshell-bootstrap/src/pki.rs +++ b/crates/openshell-bootstrap/src/pki.rs @@ -44,10 +44,6 @@ pub const DEFAULT_SERVER_SANS: &[&str] = &[ "::1", ]; -/// Subject of the shared supervisor host's client certificate. The gateway -/// binds host identity to the issuing CA, not to this name. -pub const SUPERVISOR_HOST_CLIENT_NAME: &str = "openshell-supervisor-host"; - /// Generate a complete PKI bundle: CA, server cert, and client cert. /// /// `extra_sans` are additional Subject Alternative Names to add to the server @@ -57,25 +53,6 @@ pub const SUPERVISOR_HOST_CLIENT_NAME: &str = "openshell-supervisor-host"; /// never expire. This is appropriate for an internal dev-cluster PKI where certs /// are ephemeral to the cluster's lifetime. pub fn generate_pki(extra_sans: &[String]) -> Result { - generate_pki_with_client(extra_sans, "openshell-client", "openshell-user") -} - -/// Generate the private PKI of a gateway's shared supervisor host: a CA and -/// one client certificate the host presents to the gateway. The server -/// certificate is unused and never written. -pub fn generate_supervisor_host_pki() -> Result { - generate_pki_with_client( - &[], - SUPERVISOR_HOST_CLIENT_NAME, - SUPERVISOR_HOST_CLIENT_NAME, - ) -} - -fn generate_pki_with_client( - extra_sans: &[String], - client_common_name: &str, - client_organizational_unit: &str, -) -> Result { // --- CA --- let ca_key = openshell_crypto::pki::generate_keypair() .into_diagnostic() @@ -125,10 +102,10 @@ fn generate_pki_with_client( client_params.use_authority_key_identifier_extension = true; client_params .distinguished_name - .push(DnType::CommonName, client_common_name); + .push(DnType::CommonName, "openshell-client"); client_params .distinguished_name - .push(DnType::OrganizationalUnitName, client_organizational_unit); + .push(DnType::OrganizationalUnitName, "openshell-user"); let client_cert = openshell_crypto::pki::signed_by(client_params, &client_key, &ca_cert, &ca_key) diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 75672d0263..3a13ed88e5 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -564,6 +564,15 @@ pub async fn connect_channel_pub(gateway: &GatewayEndpoint) -> Result Result { + build_plain_channel(endpoint).await +} + /// Report installed provider state for the current authenticated supervisor session. /// /// The observation must carry the session ID returned by `ConnectSupervisor`. diff --git a/crates/openshell-core/src/jwt.rs b/crates/openshell-core/src/jwt.rs index c6b3d1b33a..7b1bf57ffd 100644 --- a/crates/openshell-core/src/jwt.rs +++ b/crates/openshell-core/src/jwt.rs @@ -47,6 +47,9 @@ mod session { pub const GATEWAY_SESSION_JWT_TYPE: &str = "openshell-gateway-session+jwt"; pub const SANDBOX_SESSION_JWT_TYPE: &str = "openshell-sandbox-session+jwt"; + /// JOSE `typ` of the supervisor token: the credential of the gateway's + /// shared supervisor process for the assignment stream, and nothing else. + pub const SUPERVISOR_JWT_TYPE: &str = "openshell-supervisor+jwt"; pub const SANDBOX_SESSION_AUDIENCE: &str = "openshell-sandbox"; pub const DEFAULT_SESSION_TOKEN_TTL: Duration = Duration::from_hours(1); pub const MIN_SESSION_TOKEN_TTL: Duration = Duration::from_mins(1); @@ -55,6 +58,7 @@ mod session { const GATEWAY_ISSUER_PREFIX: &str = "openshell-gateway:"; const SANDBOX_SUBJECT_PREFIX: &str = "spiffe://openshell/sandbox/"; + const SUPERVISOR_SUBJECT_PREFIX: &str = "openshell-supervisor:"; /// Canonical sandbox identity carried by both session-token profiles. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] @@ -175,6 +179,21 @@ mod session { component: SessionComponent, } + /// Claims of the supervisor token. It names no sandbox, generation, or + /// credential epoch: the shared supervisor acts for sandboxes only with the + /// per-instance credentials each assignment carries. + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + struct SupervisorClaims { + iss: String, + sub: String, + aud: String, + iat: i64, + exp: i64, + jti: String, + component: SessionComponent, + } + /// Durable identity shared by every short-lived token for one sandbox runtime. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SandboxRuntimeIdentity { @@ -668,6 +687,40 @@ mod session { }) } + /// Mint the shared supervisor's token for the gateway's assignment + /// stream with its own bounded lifetime, independent of the session + /// token lifetime (which may be non-expiring). + pub fn mint_supervisor_token( + &self, + ttl: Duration, + ) -> Result { + validate_ttl(ttl)?; + let issued_at = self.clock.now_unix_seconds(); + let expires_at = + issued_at.saturating_add(i64::try_from(ttl.as_secs()).unwrap_or(i64::MAX)); + let token_id = Uuid::new_v4(); + let gateway_id = &self.issuer[GATEWAY_ISSUER_PREFIX.len()..]; + let claims = SupervisorClaims { + iss: self.issuer.clone(), + sub: format!("{SUPERVISOR_SUBJECT_PREFIX}{gateway_id}"), + aud: self.issuer.clone(), + iat: issued_at, + exp: expires_at, + jti: token_id.to_string(), + component: SessionComponent::OpenShellSupervisor, + }; + let mut header = Header::new(Algorithm::EdDSA); + header.kid = Some(self.key_id.clone()); + header.typ = Some(SUPERVISOR_JWT_TYPE.to_string()); + let token = encode(&header, &claims, &self.encoding_key) + .map_err(|_| SessionJwtError::SigningFailed)?; + Ok(MintedSessionToken { + token: SecretJwt::parse(token)?, + expires_at, + token_id, + }) + } + fn mint( &self, profile: SessionTokenProfile, @@ -749,20 +802,8 @@ mod session { ) -> Result { install_crypto_provider(); let gateway_id = validate_gateway_id(gateway_id)?; - let mut parsed = BTreeMap::new(); - for key in keys { - let key_id = validate_key_id(key.key_id)?; - let decoding_key = DecodingKey::from_ed_pem(&key.public_key_pem) - .map_err(|_| SessionJwtError::InvalidVerificationKey)?; - if parsed.insert(key_id, decoding_key).is_some() { - return Err(SessionJwtError::DuplicateKeyId); - } - } - if parsed.is_empty() { - return Err(SessionJwtError::NoVerificationKeys); - } Ok(Self { - keys: parsed, + keys: parse_verification_keys(keys)?, issuer: format!("{GATEWAY_ISSUER_PREFIX}{gateway_id}"), profile, clock, @@ -770,27 +811,12 @@ mod session { } pub fn verify(&self, token: &str) -> Result { - install_crypto_provider(); - let header = decode_header(token).map_err(|_| SessionJwtError::InvalidToken)?; - if header.alg != Algorithm::EdDSA { - return Err(SessionJwtError::WrongAlgorithm); - } - if header.typ.as_deref() != Some(self.profile.token_type()) { - return Err(SessionJwtError::WrongTokenType); - } - let key_id = header.kid.ok_or(SessionJwtError::MissingKeyId)?; - let key = self - .keys - .get(&key_id) - .ok_or(SessionJwtError::UnknownKeyId)?; - let mut validation = Validation::new(Algorithm::EdDSA); - validation.algorithms = vec![Algorithm::EdDSA]; - validation.validate_exp = false; - validation.validate_aud = false; - validation.set_required_spec_claims(&["iss", "aud", "iat", "exp", "sub", "jti"]); - let claims = decode::(token, key, &validation) - .map_err(|_| SessionJwtError::InvalidToken)? - .claims; + let claims = decode_signed::( + &self.keys, + token, + self.profile.token_type(), + &["iss", "aud", "iat", "exp", "sub", "jti"], + )?; self.validate_claims(claims) } @@ -810,23 +836,7 @@ mod session { SandboxGenerationId::parse(claims.runtime_generation.to_string()) .map_err(|_| SessionJwtError::InvalidRuntimeIdentity)?; let token_id = Uuid::parse_str(&claims.jti).map_err(|_| SessionJwtError::InvalidJti)?; - let now = self.clock.now_unix_seconds(); - let leeway = i64::try_from(MAX_SESSION_CLOCK_LEEWAY.as_secs()).unwrap_or(30); - if claims.iat > now.saturating_add(leeway) { - return Err(SessionJwtError::IssuedInFuture); - } - if claims.exp != 0 { - if claims.exp <= claims.iat { - return Err(SessionJwtError::InvalidLifetime); - } - let lifetime = claims.exp.saturating_sub(claims.iat); - if lifetime > i64::try_from(MAX_SESSION_TOKEN_TTL.as_secs()).unwrap_or(i64::MAX) { - return Err(SessionJwtError::InvalidLifetime); - } - if claims.exp < now.saturating_sub(leeway) { - return Err(SessionJwtError::Expired); - } - } + validate_lifetime(claims.iat, claims.exp, self.clock.now_unix_seconds())?; Ok(AuthenticatedSandboxSession { sandbox_id: claims.sandbox_id, runtime_generation: claims.runtime_generation, @@ -838,6 +848,151 @@ mod session { } } + /// A verified supervisor token. + #[derive(Clone, Debug, PartialEq, Eq)] + pub struct AuthenticatedSupervisor { + /// `openshell-supervisor:`. + pub subject: String, + pub token_id: Uuid, + pub issued_at: i64, + pub expires_at: i64, + } + + /// Strict gateway-side verifier of supervisor tokens. Accepts only the + /// supervisor profile signed by this gateway for this gateway. + pub struct SupervisorJwtVerifier { + keys: BTreeMap, + issuer: String, + subject: String, + clock: Arc, + } + + impl fmt::Debug for SupervisorJwtVerifier { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("SupervisorJwtVerifier") + .field("key_ids", &self.keys.keys().collect::>()) + .field("issuer", &self.issuer) + .finish_non_exhaustive() + } + } + + impl SupervisorJwtVerifier { + pub fn new( + gateway_id: &str, + keys: impl IntoIterator, + clock: Arc, + ) -> Result { + install_crypto_provider(); + let gateway_id = validate_gateway_id(gateway_id)?; + Ok(Self { + keys: parse_verification_keys(keys)?, + issuer: format!("{GATEWAY_ISSUER_PREFIX}{gateway_id}"), + subject: format!("{SUPERVISOR_SUBJECT_PREFIX}{gateway_id}"), + clock, + }) + } + + pub fn verify(&self, token: &str) -> Result { + let claims = decode_signed::( + &self.keys, + token, + SUPERVISOR_JWT_TYPE, + &["iss", "aud", "iat", "exp", "sub", "jti"], + )?; + if claims.iss != self.issuer { + return Err(SessionJwtError::WrongIssuer); + } + if claims.aud != self.issuer { + return Err(SessionJwtError::WrongAudience); + } + if claims.sub != self.subject { + return Err(SessionJwtError::SubjectMismatch); + } + let token_id = Uuid::parse_str(&claims.jti).map_err(|_| SessionJwtError::InvalidJti)?; + // Supervisor tokens must expire; a non-expiring one is refused. + if claims.exp == 0 { + return Err(SessionJwtError::InvalidLifetime); + } + validate_lifetime(claims.iat, claims.exp, self.clock.now_unix_seconds())?; + Ok(AuthenticatedSupervisor { + subject: claims.sub, + token_id, + issued_at: claims.iat, + expires_at: claims.exp, + }) + } + } + + fn parse_verification_keys( + keys: impl IntoIterator, + ) -> Result, SessionJwtError> { + let mut parsed = BTreeMap::new(); + for key in keys { + let key_id = validate_key_id(key.key_id)?; + let decoding_key = DecodingKey::from_ed_pem(&key.public_key_pem) + .map_err(|_| SessionJwtError::InvalidVerificationKey)?; + if parsed.insert(key_id, decoding_key).is_some() { + return Err(SessionJwtError::DuplicateKeyId); + } + } + if parsed.is_empty() { + return Err(SessionJwtError::NoVerificationKeys); + } + Ok(parsed) + } + + /// Check the header (`EdDSA` only, exact `typ`, known `kid`) and the + /// signature, then decode the claims. Time and audience are validated by + /// the caller against its own clock and profile. + fn decode_signed( + keys: &BTreeMap, + token: &str, + token_type: &str, + required_claims: &[&str], + ) -> Result { + install_crypto_provider(); + let header = decode_header(token).map_err(|_| SessionJwtError::InvalidToken)?; + if header.alg != Algorithm::EdDSA { + return Err(SessionJwtError::WrongAlgorithm); + } + if header.typ.as_deref() != Some(token_type) { + return Err(SessionJwtError::WrongTokenType); + } + let key_id = header.kid.ok_or(SessionJwtError::MissingKeyId)?; + let key = keys.get(&key_id).ok_or(SessionJwtError::UnknownKeyId)?; + let mut validation = Validation::new(Algorithm::EdDSA); + validation.algorithms = vec![Algorithm::EdDSA]; + validation.validate_exp = false; + validation.validate_aud = false; + validation.set_required_spec_claims(required_claims); + Ok(decode::(token, key, &validation) + .map_err(|_| SessionJwtError::InvalidToken)? + .claims) + } + + /// Shared lifetime rules: not issued in the future, and when expiring, + /// a positive lifetime within the session bound that has not passed. + fn validate_lifetime(issued_at: i64, expires_at: i64, now: i64) -> Result<(), SessionJwtError> { + let leeway = i64::try_from(MAX_SESSION_CLOCK_LEEWAY.as_secs()).unwrap_or(30); + if issued_at > now.saturating_add(leeway) { + return Err(SessionJwtError::IssuedInFuture); + } + if expires_at != 0 { + if expires_at <= issued_at { + return Err(SessionJwtError::InvalidLifetime); + } + let lifetime = expires_at.saturating_sub(issued_at); + if lifetime > i64::try_from(MAX_SESSION_TOKEN_TTL.as_secs()).unwrap_or(i64::MAX) { + return Err(SessionJwtError::InvalidLifetime); + } + if expires_at < now.saturating_sub(leeway) { + return Err(SessionJwtError::Expired); + } + } + Ok(()) + } + #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum SessionJwtError { #[error("sandbox ID is invalid")] @@ -1073,6 +1228,138 @@ mod tests { ); } + #[test] + fn supervisor_tokens_are_a_separate_profile_bound_to_the_gateway() { + let (issuer, gateway, sandbox, _identity) = fixture(); + let minted = issuer + .mint_supervisor_token(std::time::Duration::from_mins(15)) + .expect("supervisor token"); + assert_eq!(minted.expires_at, 1_900_000_000 + 900); + + let key = openshell_crypto::pki::generate_keypair_for(&PKCS_ED25519).unwrap(); + let public_key_pem = key.public_key_pem().into_bytes(); + let clock: Arc = Arc::new(FixedClock(1_900_000_000)); + // A verifier holding a different key under the same key ID sees a + // signature failure, not a key lookup failure. + let verification_key = |pem: Vec| SessionVerificationKey { + key_id: "current".to_string(), + public_key_pem: pem, + }; + let wrong_key = SupervisorJwtVerifier::new( + "test", + [verification_key(public_key_pem)], + clock.clone(), + ) + .expect("verifier"); + assert_eq!( + wrong_key.verify(minted.token.expose_secret()), + Err(SessionJwtError::InvalidToken) + ); + + // Session verifiers refuse the supervisor profile and the supervisor + // verifier refuses session tokens, whatever key signed them. + assert_eq!( + gateway.verify(minted.token.expose_secret()), + Err(SessionJwtError::WrongTokenType) + ); + assert_eq!( + sandbox.verify(minted.token.expose_secret()), + Err(SessionJwtError::WrongTokenType) + ); + let identity = SandboxRuntimeIdentity { + sandbox_id: SandboxId::parse("sandbox-a").unwrap(), + runtime_generation: SandboxGenerationId::parse("generation-1").unwrap(), + auth_epoch: CredentialEpoch::new(1).unwrap(), + }; + let pair = issuer.mint_pair(&identity).unwrap(); + assert_eq!( + wrong_key.verify(pair.gateway.token.expose_secret()), + Err(SessionJwtError::WrongTokenType) + ); + + // The supervisor lifetime is bounded on its own, whatever the + // session lifetime is. + drop(clock); + for ttl in [ + std::time::Duration::from_secs(59), + std::time::Duration::from_secs(3_601), + ] { + assert_eq!( + issuer.mint_supervisor_token(ttl).unwrap_err(), + SessionJwtError::InvalidLifetime + ); + } + } + + #[test] + fn supervisor_tokens_verify_only_for_their_gateway_and_lifetime() { + let key = openshell_crypto::pki::generate_keypair_for(&PKCS_ED25519).unwrap(); + let public_key_pem = key.public_key_pem().into_bytes(); + let signing_pem = key.serialize_pem().unwrap(); + let verification_key = || SessionVerificationKey { + key_id: "current".to_string(), + public_key_pem: public_key_pem.clone(), + }; + let issuer = |gateway_id: &str, now: i64| { + SessionJwtIssuer::from_ed25519_pem( + signing_pem.as_bytes(), + "current", + gateway_id, + Some(std::time::Duration::from_mins(15)), + Arc::new(FixedClock(now)), + ) + .unwrap() + }; + let verifier = |gateway_id: &str, now: i64| { + SupervisorJwtVerifier::new( + gateway_id, + [verification_key()], + Arc::new(FixedClock(now)), + ) + .unwrap() + }; + + let minted = issuer("gw-a", 1_900_000_000) + .mint_supervisor_token(std::time::Duration::from_mins(15)) + .unwrap(); + let authenticated = verifier("gw-a", 1_900_000_100) + .verify(minted.token.expose_secret()) + .expect("valid supervisor token"); + assert_eq!(authenticated.subject, "openshell-supervisor:gw-a"); + assert_eq!(authenticated.token_id, minted.token_id); + assert_eq!(authenticated.expires_at, 1_900_000_900); + + // Another gateway's token is refused by issuer, audience, and subject. + assert_eq!( + verifier("gw-b", 1_900_000_100).verify(minted.token.expose_secret()), + Err(SessionJwtError::WrongIssuer) + ); + // Expired tokens are refused past the leeway. + assert_eq!( + verifier("gw-a", 1_900_001_000).verify(minted.token.expose_secret()), + Err(SessionJwtError::Expired) + ); + assert_eq!( + verifier("gw-a", 1_899_999_000).verify(minted.token.expose_secret()), + Err(SessionJwtError::IssuedInFuture) + ); + assert_eq!( + verifier("gw-a", 1_900_000_100).verify("not-a-token"), + Err(SessionJwtError::InvalidToken) + ); + } + + #[derive(Serialize)] + struct NonExpiringSupervisor<'a> { + iss: &'a str, + sub: &'a str, + aud: &'a str, + iat: i64, + exp: i64, + jti: String, + component: SessionComponent, + } + #[test] fn non_expiring_session_tokens_remain_usable() { let key = openshell_crypto::pki::generate_keypair_for(&PKCS_ED25519) @@ -1110,6 +1397,46 @@ mod tests { .verify(pair.sandbox.token.expose_secret()) .expect("non-expiring token remains valid"); + // The default (non-expiring) session issuer still mints expiring + // supervisor tokens, and a supervisor token must expire. + let supervisor = issuer + .mint_supervisor_token(std::time::Duration::from_mins(15)) + .expect("supervisor token from a non-expiring issuer"); + assert_eq!(supervisor.expires_at, 1_900_000_000 + 900); + let supervisor_verifier = SupervisorJwtVerifier::new( + "test", + [SessionVerificationKey { + key_id: "current".to_string(), + public_key_pem: key.public_key_pem().into_bytes(), + }], + Arc::new(FixedClock(1_900_000_100)), + ) + .unwrap(); + supervisor_verifier + .verify(supervisor.token.expose_secret()) + .expect("expiring supervisor token"); + let mut header = Header::new(Algorithm::EdDSA); + header.kid = Some("current".to_string()); + header.typ = Some(SUPERVISOR_JWT_TYPE.to_string()); + let forever = encode( + &header, + &NonExpiringSupervisor { + iss: "openshell-gateway:test", + sub: "openshell-supervisor:test", + aud: "openshell-gateway:test", + iat: 1_900_000_000, + exp: 0, + jti: uuid::Uuid::new_v4().to_string(), + component: SessionComponent::OpenShellSupervisor, + }, + &EncodingKey::from_ed_pem(key.serialize_pem().unwrap().as_bytes()).unwrap(), + ) + .unwrap(); + assert_eq!( + supervisor_verifier.verify(&forever), + Err(SessionJwtError::InvalidLifetime) + ); + let slot = SessionBearerTokenSlot::new( pair.sandbox.token.clone(), pair.sandbox.expires_at, diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index c9b351d71e..2791c70292 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -61,6 +61,7 @@ pub mod secrets; pub mod settings; pub mod shell; pub mod spiffe; +#[cfg(feature = "jwt")] pub mod supervisor_host; pub mod telemetry; pub mod time; diff --git a/crates/openshell-core/src/supervisor_host.rs b/crates/openshell-core/src/supervisor_host.rs index 436cf13d3e..25d032ba14 100644 --- a/crates/openshell-core/src/supervisor_host.rs +++ b/crates/openshell-core/src/supervisor_host.rs @@ -6,8 +6,10 @@ //! //! The gateway owns the desired set of sandbox instances a shared supervisor //! runs. A compute driver publishes that set here; the gateway's gRPC server -//! hands each authenticated host stream to [`SupervisorHostController::attach`], -//! which sends complete snapshots and records what the host reports. +//! hands each host stream authenticated by a supervisor token to +//! [`SupervisorHostController::attach`], which sends complete snapshots and +//! records what the host reports. The gateway also mints that token and +//! publishes each renewal here for the driver to deliver to the host. //! //! Desired state is uninitialized until the driver has derived it from its //! durable records once. Nothing is sent before that, so a restarted gateway @@ -23,9 +25,8 @@ //! delayed hello from an older connection of the current process is refused. use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::{mpsc, watch}; use tokio_stream::StreamExt as _; @@ -57,13 +58,28 @@ pub const MAX_ASSIGNED_INSTANCES: usize = 256; /// instances still being removed. pub const MAX_REPORTED_INSTANCES: usize = 4 * MAX_ASSIGNED_INSTANCES; -/// Verified identity of a connected supervisor host. +/// Verified identity of a connected supervisor host: the claims of the +/// supervisor token that authenticated its stream. #[derive(Clone, Debug, PartialEq, Eq)] pub struct HostIdentity { - /// Subject common name of the host certificate. + /// Subject of the supervisor token (`openshell-supervisor:`). pub subject: String, - /// SHA-256 fingerprint of the verified host certificate. - pub certificate_sha256: String, + /// `jti` of the supervisor token. + pub token_id: String, + /// `exp` of the supervisor token (Unix seconds). The gateway ends the + /// stream when it passes; the host reconnects with a renewed token. + pub expires_at: i64, +} + +/// The current supervisor token, as the gateway mints and renews it. +/// +/// A compute driver delivers it to the supervisor process by whatever means +/// its deployment packaging provides; the Kubernetes driver writes a Secret. +#[derive(Clone, Debug)] +pub struct SupervisorCredential { + pub token: crate::jwt::SecretJwt, + /// Unix seconds. + pub expires_at: i64, } /// What the gateway knows about one assigned route on the host. @@ -146,8 +162,9 @@ struct Observed { } struct Inner { - trust_anchor: PathBuf, desired: Mutex, + /// Latest supervisor token the gateway minted; `None` until the first. + credential: watch::Sender>, /// Current desired revision; 0 while uninitialized. revision: watch::Sender, observed: Mutex, @@ -160,11 +177,16 @@ pub struct SupervisorHostController { inner: Arc, } +impl Default for SupervisorHostController { + fn default() -> Self { + Self::new() + } +} + impl std::fmt::Debug for SupervisorHostController { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("SupervisorHostController") - .field("trust_anchor", &self.inner.trust_anchor) .finish_non_exhaustive() } } @@ -189,24 +211,27 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { } impl SupervisorHostController { - /// A controller whose hosts authenticate with certificates issued by the - /// CA at `trust_anchor`. #[must_use] - pub fn new(trust_anchor: PathBuf) -> Self { + pub fn new() -> Self { Self { inner: Arc::new(Inner { - trust_anchor, desired: Mutex::new(Desired::default()), + credential: watch::Sender::new(None), revision: watch::Sender::new(0), observed: Mutex::new(Observed::default()), }), } } - /// PEM CA certificate that issues supervisor host client certificates. + /// Publish a freshly minted supervisor token for delivery to the host. + pub fn publish_credential(&self, credential: SupervisorCredential) { + self.inner.credential.send_replace(Some(credential)); + } + + /// The current supervisor token and every renewal after it. #[must_use] - pub fn trust_anchor_path(&self) -> &Path { - &self.inner.trust_anchor + pub fn credential(&self) -> watch::Receiver> { + self.inner.credential.subscribe() } /// The driver has derived the complete desired set at least once. @@ -403,6 +428,7 @@ impl SupervisorHostController { )); } let (tx, rx) = mpsc::channel(OUTBOUND_QUEUE); + let expires_at = identity.expires_at; let (session_id, superseded) = self.begin_session(identity, &hello)?; tx.send(Ok(HostGatewayMessage { payload: Some(host_gateway_message::Payload::Accepted(HostAccepted { @@ -418,7 +444,7 @@ impl SupervisorHostController { let controller = self.clone(); tokio::spawn(async move { controller - .run_session(session_id, inbound, tx, superseded) + .run_session(session_id, expires_at, inbound, tx, superseded) .await; controller.end_session(session_id); }); @@ -533,10 +559,16 @@ impl SupervisorHostController { async fn run_session( &self, session_id: u64, + expires_at: i64, mut inbound: Streaming, tx: mpsc::Sender>, mut superseded: watch::Receiver, ) { + // A bearer is verified once per stream, so the stream lives no longer + // than the token did. Hosted instances are unaffected; the host + // reconnects with the renewed token it has been delivered. + let credential_expiry = tokio::time::sleep(credential_remaining(expires_at)); + tokio::pin!(credential_expiry); let mut revision = self.inner.revision.subscribe(); let mut sent_revision = 0; let mut last_heard = tokio::time::Instant::now(); @@ -619,11 +651,25 @@ impl SupervisorHostController { } } _ = superseded.changed() => return, + () = &mut credential_expiry => { + tracing::info!(session = session_id, "supervisor token expired; closing the stream for reconnection"); + return; + } } } } } +/// Time until `expires_at` (Unix seconds); zero once it has passed. +fn credential_remaining(expires_at: i64) -> Duration { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + Duration::from_secs(u64::try_from(expires_at.saturating_sub(now)).unwrap_or(0)) +} + #[cfg(test)] mod tests { use super::*; @@ -656,7 +702,7 @@ mod tests { #[test] fn full_resyncs_initialize_once_and_then_compare_and_replace() { - let controller = SupervisorHostController::new(PathBuf::from("/ca.crt")); + let controller = SupervisorHostController::new(); let revision = controller.inner.revision.subscribe(); // Edits before the first full list wait for it. controller @@ -738,15 +784,16 @@ mod tests { #[test] fn sessions_fence_stale_streams_epochs_and_acks() { - let controller = SupervisorHostController::new(PathBuf::from("/ca.crt")); + let controller = SupervisorHostController::new(); controller .replace_assignments(0, [assignment("rt-a", b"a")]) .unwrap(); assert_eq!(controller.observed("rt-a"), ObservedInstance::NoHost); let identity = HostIdentity { - subject: "host".to_string(), - certificate_sha256: "00".to_string(), + subject: "openshell-supervisor:test".to_string(), + token_id: "token".to_string(), + expires_at: i64::MAX, }; let hello = |instance_id: &str, connection_epoch| HostHello { instance_id: instance_id.to_string(), diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 29eb9f98ae..210abde9a5 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -16,7 +16,7 @@ path = "src/main.rs" [dependencies] openshell-crypto = { workspace = true, features = ["tls-kube"] } -openshell-core = { path = "../openshell-core", default-features = false } +openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } openshell-otel = { path = "../openshell-otel" } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index a91b57b7a2..1684bb7565 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -197,9 +197,17 @@ fails closed. `openshell_core::supervisor_host::SupervisorHostController`, installed into the gateway through the compute-driver build context, and the gateway serves it to the supervisor over the private `openshell.supervisor.host.v1` stream. - The supervisor dials the gateway with a client certificate issued by the CA - at `shared_supervisor.host_ca_file`; that identity authorizes only the - assignment stream. + The supervisor dials the gateway over TLS with a gateway-minted supervisor + token (see the `openshell-supervisor` README); that token authorizes only + the assignment stream. +- **Supervisor token delivery.** The gateway mints the supervisor token from + its `gateway_jwt` signing key and renews it at half its lifetime (the + `gateway_jwt` extension-token TTL, 15 minutes by default, bounded to one + hour). The driver's part is delivery only: it watches the controller for + each renewal and replaces the `token` key of the chart-created + `shared_supervisor.token_secret_name` Secret in `service_namespace`, which + the supervisor Pod mounts. A missing Secret or a denied patch is logged and + retried; a token published while a retry waits supersedes the retry. - **Create and start.** The driver creates a per-sandbox egress allowance `os-shared-control-` from the workload Pod to the Pods the shared supervisor's Service selects, on the runtime port, binds its UID and @@ -226,11 +234,12 @@ fails closed. its assignments before it sends anything. The rebuild is compare-and-replace against the revision read before the list, so it never undoes a lifecycle edit that raced it. -- **Isolation.** The host certificate is separate from every instance's - credentials and from the ordinary client CA; the gateway refuses it on every - other RPC and refuses every other identity on the assignment stream. - Sandboxes cannot reach the stream, and the routing preface only selects - which generation-pinned TLS identity the supervisor demands. +- **Isolation.** The supervisor token is a profile of its own, separate from + every instance's credentials; the gateway refuses it on every other RPC and + refuses every other identity on the assignment stream, and ends a stream + when the token that opened it expires. Sandboxes cannot reach the stream, + and the routing preface only selects which generation-pinned TLS identity + the supervisor demands. The topology does not support upstream proxies or provider SPIFFE identity, and the shared supervisor is a single point of failure for the sandboxes it hosts. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index d2f3b5b8c5..598e0b20ce 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -211,10 +211,10 @@ pub struct SharedSupervisorConfig { pub service_namespace: String, /// Service port for runtime-initiated Sandbox Protocol connections. pub runtime_port: u16, - /// PEM CA certificate that issued the supervisor host's client - /// certificate. The gateway accepts that certificate on its own listener - /// and classifies the connection as the supervisor host. - pub host_ca_file: String, + /// Secret in `service_namespace` the driver keeps current with the + /// gateway-minted supervisor token (key `token`); the supervisor Pod + /// mounts it and presents the token on its assignment stream. + pub token_secret_name: String, } impl SharedSupervisorConfig { @@ -227,8 +227,8 @@ impl SharedSupervisorConfig { if self.runtime_port == 0 { return Err("shared_supervisor runtime_port must be nonzero".into()); } - if !Path::new(&self.host_ca_file).is_absolute() { - return Err("shared_supervisor host_ca_file must be absolute".into()); + if !is_dns_subdomain(&self.token_secret_name) { + return Err("shared_supervisor token_secret_name must be a DNS subdomain name".into()); } Ok(()) } @@ -240,6 +240,21 @@ impl SharedSupervisorConfig { } } +/// RFC 1123 subdomain, as Kubernetes object names require. +fn is_dns_subdomain(value: &str) -> bool { + !value.is_empty() + && value.len() <= 253 + && value.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && !label.starts_with('-') + && !label.ends_with('-') + }) +} + fn is_dns_label(value: &str) -> bool { !value.is_empty() && value.len() <= 63 @@ -1104,7 +1119,7 @@ mod tests { service_name: "openshell-supervisor".to_string(), service_namespace: "openshell".to_string(), runtime_port: 8443, - host_ca_file: "/etc/openshell-tls/supervisor-host/ca.crt".to_string(), + token_secret_name: "openshell-supervisor-token".to_string(), }; let gateway = |shared: Option| KubernetesComputeConfig { supervisor_topology: Some(SupervisorTopology::Gateway), @@ -1133,7 +1148,7 @@ mod tests { ..shared() }, SharedSupervisorConfig { - host_ca_file: "relative/ca.crt".to_string(), + token_secret_name: "Not_A_Secret_Name".to_string(), ..shared() }, ] { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index d06ec35fb4..62660f48e9 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1028,6 +1028,9 @@ impl KubernetesComputeDriver { }; let shared_supervisor = config.shared_supervisor.clone().map(SharedSupervisor::new); + if let Some(shared) = &shared_supervisor { + shared.spawn_token_secret_writer(client.clone(), shutdown_rx.clone()); + } if shared_supervisor.is_some() && (config.supervisor_otlp_endpoint.is_some() || config.supervisor_ocsf_export.is_some()) diff --git a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs index b549b55ce6..b2f370b079 100644 --- a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs +++ b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs @@ -14,19 +14,28 @@ //! set is the generation's Sandbox resource plus its launch Secret, so a //! restarted gateway rebuilds it before publishing anything, and a full //! resync is compare-and-replace against the revision it started from. +//! +//! The gateway also mints the supervisor token the supervisor presents on +//! that stream and publishes each renewal into the controller; the driver's +//! only part in authentication is delivery, writing the current token into +//! the Secret the supervisor Pod mounts. -use std::path::PathBuf; +use std::collections::BTreeMap; +use std::time::Duration; +use k8s_openapi::api::core::v1::Secret; +use kube::api::{Api, ObjectMeta, Patch, PatchParams}; use kube::core::DynamicObject; use openshell_core::proto::supervisor_host::v1::{ SandboxInstanceAssignment, SandboxInstanceStatus, }; use openshell_core::supervisor_host::{ - AssignmentError, ObservedInstance, SupervisorHostController, + AssignmentError, ObservedInstance, SupervisorCredential, SupervisorHostController, }; use openshell_sandbox_backend::sandbox_instance::{ SandboxInstanceRegistration, SandboxInstanceStatusReport, }; +use tracing::{info, warn}; use crate::config::SharedSupervisorConfig; use crate::driver::KubernetesDriverError; @@ -38,6 +47,11 @@ pub const ANNOTATION_SUPERVISOR_TOPOLOGY: &str = "openshell.ai/sandbox-runtime-s /// annotation key keeps its established `-session` name. pub const ANNOTATION_SUPERVISOR_SESSION: &str = "openshell.ai/sandbox-runtime-supervisor-session"; pub const GATEWAY_TOPOLOGY: &str = "gateway"; +/// Key of the supervisor token in `shared_supervisor.token_secret_name`. +pub const TOKEN_SECRET_KEY: &str = "token"; +/// Annotation recording the token's expiry (Unix seconds) for operators. +pub const ANNOTATION_TOKEN_EXPIRES_AT: &str = "openshell.ai/supervisor-token-expires-at"; +const TOKEN_WRITE_RETRY: Duration = Duration::from_secs(5); /// Where a generation's supervisor runs, as recorded when it started. #[derive(Clone, Debug, PartialEq, Eq)] @@ -89,8 +103,74 @@ pub struct SharedSupervisor { impl SharedSupervisor { pub fn new(config: SharedSupervisorConfig) -> Self { - let controller = SupervisorHostController::new(PathBuf::from(&config.host_ca_file)); - Self { config, controller } + Self { + config, + controller: SupervisorHostController::new(), + } + } + + /// Keep the supervisor token Secret current with every token the gateway + /// publishes, until shutdown. The Secret itself is deployment packaging + /// (the chart creates it empty); the driver only replaces its token. A + /// failed write is retried until it succeeds or a newer token supersedes + /// it. + pub fn spawn_token_secret_writer( + &self, + client: kube::Client, + mut shutdown_rx: tokio::sync::watch::Receiver, + ) { + let secrets = Api::::namespaced(client, &self.config.service_namespace); + let name = self.config.token_secret_name.clone(); + let mut credential = self.controller.credential(); + tokio::spawn(async move { + loop { + let current = credential.borrow_and_update().clone(); + if let Some(current) = current { + while let Err(error) = write_token_secret(&secrets, &name, ¤t).await { + match &error { + kube::Error::Api(response) if response.code == 404 => warn!( + secret = %name, + "the supervisor token Secret does not exist; the chart creates it, \ + restore it (helm upgrade) so the shared supervisor can authenticate" + ), + kube::Error::Api(response) if response.code == 403 => warn!( + secret = %name, + "the gateway may not patch the supervisor token Secret; check the \ + chart's gateway Role" + ), + _ => warn!( + secret = %name, + %error, + "failed to write the supervisor token Secret; retrying" + ), + } + tokio::select! { + () = tokio::time::sleep(TOKEN_WRITE_RETRY) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + if credential.has_changed().unwrap_or(true) { + break; + } + } + } + tokio::select! { + changed = credential.changed() => { + if changed.is_err() { + return; + } + } + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + } + }); } pub fn config(&self) -> &SharedSupervisorConfig { @@ -162,6 +242,37 @@ pub fn reported(status: &SandboxInstanceStatus) -> SandboxInstanceStatusReport { SandboxInstanceStatusReport::from_proto(status) } +/// Replace the token in the chart-created Secret. +async fn write_token_secret( + secrets: &Api, + name: &str, + credential: &SupervisorCredential, +) -> Result<(), kube::Error> { + let secret = Secret { + metadata: ObjectMeta { + annotations: Some(BTreeMap::from([( + ANNOTATION_TOKEN_EXPIRES_AT.to_string(), + credential.expires_at.to_string(), + )])), + ..ObjectMeta::default() + }, + string_data: Some(BTreeMap::from([( + TOKEN_SECRET_KEY.to_string(), + credential.token.expose_secret().to_string(), + )])), + ..Secret::default() + }; + secrets + .patch(name, &PatchParams::default(), &Patch::Merge(&secret)) + .await?; + info!( + secret = %name, + expires_at = credential.expires_at, + "wrote the supervisor token Secret" + ); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-server/src/auth/audit.rs b/crates/openshell-server/src/auth/audit.rs index 778f8b725f..6c9136aa6f 100644 --- a/crates/openshell-server/src/auth/audit.rs +++ b/crates/openshell-server/src/auth/audit.rs @@ -219,7 +219,7 @@ fn authenticated(principal: &Principal) -> bool { Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) - | Principal::SupervisorHost(_) => true, + | Principal::Supervisor(_) => true, } } @@ -237,7 +237,7 @@ fn mechanism(principal: &Principal) -> Option { SandboxIdentitySource::ComputeDriver { .. } => AuthMechanism::ComputeDriver, }), Principal::Peer(_) => Some(AuthMechanism::GatewayPeer), - Principal::SupervisorHost(_) => Some(AuthMechanism::SupervisorHost), + Principal::Supervisor(_) => Some(AuthMechanism::SupervisorToken), Principal::Anonymous => None, } } @@ -254,7 +254,6 @@ fn credential_digest( ) -> Option<[u8; 32]> { let credential = match principal { Principal::Operator(operator) => operator.certificate_sha256.as_str(), - Principal::SupervisorHost(host) => host.identity.certificate_sha256.as_str(), Principal::User(user) if user.identity.provider == IdentityProvider::Mtls => { peer_certificate_sha256.unwrap_or_default() } @@ -511,7 +510,9 @@ fn principal_user(principal: &Principal) -> User { Principal::Operator(operator) => User::new(&operator.identity.subject, UserTypeId::Admin), Principal::Sandbox(sandbox) => User::other(&sandbox.sandbox_id, "Sandbox"), Principal::Peer(peer) => User::other(&peer.replica_id, "Gateway Peer"), - Principal::SupervisorHost(host) => User::other(&host.identity.subject, "Supervisor Host"), + Principal::Supervisor(supervisor) => { + User::other(&supervisor.identity.subject, "Supervisor") + } Principal::Anonymous => User::unidentified(), } } diff --git a/crates/openshell-server/src/auth/authenticator.rs b/crates/openshell-server/src/auth/authenticator.rs index 624f20253b..c9fc367a3f 100644 --- a/crates/openshell-server/src/auth/authenticator.rs +++ b/crates/openshell-server/src/auth/authenticator.rs @@ -19,6 +19,8 @@ //! gateway session JWTs //! - [`super::compute_driver::ComputeDriverAuthenticator`] — driver-native //! sandbox bootstrap credentials (path-scoped to `IssueSandboxToken`) +//! - [`super::supervisor_token::SupervisorTokenAuthenticator`] — the shared +//! supervisor's gateway-minted token for the assignment stream //! - [`super::oidc::OidcAuthenticator`] — user OIDC Bearer tokens use super::principal::Principal; use async_trait::async_trait; @@ -118,8 +120,8 @@ pub enum AuthMechanism { GatewayPeer, /// The `CreateSshSession` token an SSH relay presents. SshSessionToken, - /// The shared supervisor host's gateway-scoped client certificate. - SupervisorHost, + /// The shared supervisor's gateway-minted supervisor token. + SupervisorToken, } impl AuthMechanism { @@ -133,7 +135,7 @@ impl AuthMechanism { Self::ComputeDriver => "compute_driver", Self::GatewayPeer => "k8s_service_account", Self::SshSessionToken => "ssh_session_token", - Self::SupervisorHost => "supervisor_host_mtls", + Self::SupervisorToken => "supervisor_token", } } } diff --git a/crates/openshell-server/src/auth/descriptor_authz.rs b/crates/openshell-server/src/auth/descriptor_authz.rs index 57599a65ff..13e669a27b 100644 --- a/crates/openshell-server/src/auth/descriptor_authz.rs +++ b/crates/openshell-server/src/auth/descriptor_authz.rs @@ -125,7 +125,7 @@ impl DescriptorAuthTable { "dual" => AuthMode::Dual, "peer" => AuthMode::Peer, "operator" => AuthMode::Operator, - "supervisor_host" => AuthMode::SupervisorHost, + "supervisor" => AuthMode::Supervisor, other => { return Err(format!("method {path}: unknown auth_mode '{other}'")); } diff --git a/crates/openshell-server/src/auth/guard.rs b/crates/openshell-server/src/auth/guard.rs index cdaa84ffd1..2578ec44be 100644 --- a/crates/openshell-server/src/auth/guard.rs +++ b/crates/openshell-server/src/auth/guard.rs @@ -30,10 +30,10 @@ use tonic::Status; pub fn ensure_sandbox_scope(principal: &Principal, claimed_sandbox_id: &str) -> Result<(), Status> { match principal { Principal::User(_) | Principal::Operator(_) => Ok(()), - Principal::Peer(_) | Principal::SupervisorHost(_) => Err(deny( + Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, - "gateway peer and supervisor host principals may not call sandbox-scoped methods", + "gateway peer and supervisor principals may not call sandbox-scoped methods", )), Principal::Sandbox(p) => { if p.sandbox_id == claimed_sandbox_id { @@ -91,7 +91,7 @@ pub fn ensure_sandbox_principal_scope( Principal::User(_) | Principal::Operator(_) | Principal::Peer(_) - | Principal::SupervisorHost(_) => Err(deny( + | Principal::Supervisor(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, "supervisor RPCs require a sandbox principal", diff --git a/crates/openshell-server/src/auth/launch_signing.rs b/crates/openshell-server/src/auth/launch_signing.rs index a1b510bb24..c6eebcdd69 100644 --- a/crates/openshell-server/src/auth/launch_signing.rs +++ b/crates/openshell-server/src/auth/launch_signing.rs @@ -11,6 +11,7 @@ use super::sandbox_jwt::{ExtensionJwtIssuer, SandboxSessionJwtAuthority}; pub struct LaunchSigningAuthorities { pub extension: Arc, + /// Also mints and verifies the shared supervisor's token with the same key. pub sandbox_session: Arc, } diff --git a/crates/openshell-server/src/auth/method_authz.rs b/crates/openshell-server/src/auth/method_authz.rs index 6e5f4958c6..e00357b8ac 100644 --- a/crates/openshell-server/src/auth/method_authz.rs +++ b/crates/openshell-server/src/auth/method_authz.rs @@ -26,9 +26,9 @@ pub enum AuthMode { Peer, /// Only a verified mTLS operator may call this method. Operator, - /// Only the gateway's shared supervisor host, authenticated by its - /// gateway-scoped host certificate, may call this method. - SupervisorHost, + /// Only the gateway's shared supervisor, authenticated by its + /// gateway-minted supervisor token, may call this method. + Supervisor, } /// Coarse role mapping. Maps to the configured `admin_role` / @@ -98,7 +98,7 @@ pub fn is_user_callable(method: &str) -> bool { | AuthMode::Unauthenticated | AuthMode::Peer | AuthMode::Operator - | AuthMode::SupervisorHost, + | AuthMode::Supervisor, ) => false, Some(AuthMode::Bearer | AuthMode::Dual) | None => true, } @@ -119,13 +119,13 @@ pub fn is_peer_callable(method: &str) -> bool { matches!(lookup(method).map(|m| m.auth_mode), Some(AuthMode::Peer)) } -/// `true` if the method is callable by the shared supervisor host, and by -/// nothing else. +/// `true` if the method is callable by the shared supervisor, and by nothing +/// else. #[must_use] -pub fn is_supervisor_host_callable(method: &str) -> bool { +pub fn is_supervisor_callable(method: &str) -> bool { matches!( lookup(method).map(|m| m.auth_mode), - Some(AuthMode::SupervisorHost) + Some(AuthMode::Supervisor) ) } @@ -190,14 +190,15 @@ mod tests { assert!(!is_user_callable(path)); assert!(is_peer_callable(path)); } - // The supervisor host stream is reachable only with a host identity. + // The supervisor assignment stream is reachable only with a + // supervisor token. let host_attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; - assert!(is_supervisor_host_callable(host_attach)); + assert!(is_supervisor_callable(host_attach)); assert!(!is_user_callable(host_attach)); assert!(!is_operator_callable(host_attach)); assert!(!is_sandbox_callable(host_attach)); assert!(!is_peer_callable(host_attach)); - assert!(!is_supervisor_host_callable( + assert!(!is_supervisor_callable( "/openshell.v1.OpenShell/ConnectSupervisor" )); // Unauthenticated methods are not "user callable" — they're diff --git a/crates/openshell-server/src/auth/mod.rs b/crates/openshell-server/src/auth/mod.rs index 5b0209cae4..f1800d5044 100644 --- a/crates/openshell-server/src/auth/mod.rs +++ b/crates/openshell-server/src/auth/mod.rs @@ -25,6 +25,7 @@ pub mod principal; pub mod sandbox_jwt; pub mod sandbox_methods; pub mod sandbox_session; +pub mod supervisor_token; pub mod workspace_authz; pub use http::router; diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs index 9f73e9ef8c..825cd8a0e4 100644 --- a/crates/openshell-server/src/auth/principal.rs +++ b/crates/openshell-server/src/auth/principal.rs @@ -32,10 +32,10 @@ pub enum Principal { Sandbox(#[allow(dead_code)] SandboxPrincipal), /// Gateway replica authenticated for internal peer RPCs. Peer(PeerPrincipal), - /// The gateway's shared supervisor host, authenticated at the connection - /// level by a certificate from the host CA. It may call only the - /// supervisor host stream and never acts for a user or a sandbox. - SupervisorHost(SupervisorHostPrincipal), + /// The gateway's shared supervisor, authenticated by a gateway-minted + /// supervisor token. It may call only the supervisor assignment stream + /// and never acts for a user or a sandbox. + Supervisor(SupervisorPrincipal), /// Truly unauthenticated caller (health probes, reflection). Sandbox-class /// and user-class methods reject this variant. #[allow(dead_code)] @@ -85,10 +85,10 @@ pub struct SandboxPrincipal { pub trust_domain: Option, } -/// Shared supervisor host caller. +/// Shared supervisor caller. #[derive(Debug, Clone)] -pub struct SupervisorHostPrincipal { - /// Verified host certificate identity. +pub struct SupervisorPrincipal { + /// Claims of the verified supervisor token. pub identity: openshell_core::supervisor_host::HostIdentity, } @@ -130,7 +130,7 @@ impl Principal { Self::Operator(operator) => format!("operator:{}", operator.identity.subject), Self::Sandbox(sandbox) => format!("sandbox:{}", sandbox.sandbox_id), Self::Peer(peer) => format!("peer:{}", peer.replica_id), - Self::SupervisorHost(host) => format!("supervisor-host:{}", host.identity.subject), + Self::Supervisor(supervisor) => format!("supervisor:{}", supervisor.identity.subject), Self::Anonymous => "anonymous".to_string(), } } diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 57c64385f5..774b9f5f47 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -4,9 +4,11 @@ //! Gateway-minted session and extension JWTs. //! //! [`SandboxSessionJwtAuthority`] mints and verifies generation-bound sandbox -//! session tokens. [`ExtensionJwtIssuer`] mints explicitly typed credentials -//! for supervisor middleware and gateway interceptors and publishes the -//! corresponding JWKS metadata. +//! session tokens, and with the same key the shared supervisor's token for +//! the private assignment stream (its own `typ` and claims, see +//! [`super::supervisor_token`]). [`ExtensionJwtIssuer`] mints explicitly +//! typed credentials for supervisor middleware and gateway interceptors and +//! publishes the corresponding JWKS metadata. //! //! Algorithm: `EdDSA` (Ed25519). Pinned via `Validation::algorithms` to //! prevent algorithm-confusion attacks. @@ -35,9 +37,10 @@ use x509_parser::{oid_registry::OID_SIG_ED25519, prelude::FromDer, x509::Subject use openshell_core::SandboxSessionId; use openshell_core::jwt::{ - AuthenticatedSandboxSession, CredentialEpoch, GATEWAY_SESSION_JWT_TYPE, SandboxId, - SandboxLaunchAuthentication, SandboxRuntimeIdentity, SessionJwtError, SessionJwtIssuer, - SessionJwtVerifier, SessionTokenProfile, SessionVerificationKey, SupervisorAuthBundle, + AuthenticatedSandboxSession, AuthenticatedSupervisor, CredentialEpoch, + GATEWAY_SESSION_JWT_TYPE, MintedSessionToken, SandboxId, SandboxLaunchAuthentication, + SandboxRuntimeIdentity, SessionJwtError, SessionJwtIssuer, SessionJwtVerifier, + SessionTokenProfile, SessionVerificationKey, SupervisorAuthBundle, SupervisorJwtVerifier, SystemJwtClock, }; use openshell_core::sandbox_generation::SandboxGenerationId; @@ -102,6 +105,7 @@ pub struct MintedToken { pub struct SandboxSessionJwtAuthority { issuer: SessionJwtIssuer, gateway_verifier: SessionJwtVerifier, + supervisor_verifier: SupervisorJwtVerifier, gateway_id: String, verification_keys: Vec, } @@ -148,12 +152,16 @@ impl SandboxSessionJwtAuthority { gateway_id, SessionTokenProfile::Gateway, verification_keys.clone(), - clock, + clock.clone(), ) .map_err(|error| error.to_string())?; + let supervisor_verifier = + SupervisorJwtVerifier::new(gateway_id, verification_keys.clone(), clock) + .map_err(|error| error.to_string())?; Ok(Self { issuer, gateway_verifier, + supervisor_verifier, gateway_id: gateway_id.to_string(), verification_keys, }) @@ -276,6 +284,34 @@ impl SandboxSessionJwtAuthority { ) }) } + + /// Mint the shared supervisor's token for the assignment stream. Its + /// lifetime is bounded on its own; the session lifetime may be + /// non-expiring. + pub fn mint_supervisor_token( + &self, + ttl: Duration, + ) -> Result { + self.issuer.mint_supervisor_token(ttl) + } + + /// Verify a presented supervisor token. + pub fn verify_supervisor_token( + &self, + token: &str, + ) -> Result { + self.supervisor_verifier.verify(token).map_err(|error| { + let reason = match error { + SessionJwtError::SubjectMismatch => AuthFailureReason::AudienceOrIssuerMismatch, + ref other => session_failure_reason(other), + }; + AuthRejection::new( + AuthMechanism::SupervisorToken, + reason, + Status::unauthenticated(format!("invalid supervisor token: {error}")), + ) + }) + } } /// Authenticates launch-scoped supervisor tokens and checks their identity diff --git a/crates/openshell-server/src/auth/supervisor_token.rs b/crates/openshell-server/src/auth/supervisor_token.rs new file mode 100644 index 0000000000..c3274693d4 --- /dev/null +++ b/crates/openshell-server/src/auth/supervisor_token.rs @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The supervisor token: the credential of the gateway's shared supervisor +//! for the private assignment stream. +//! +//! The token is minted and verified by the launch-signing +//! [`super::sandbox_jwt::SandboxSessionJwtAuthority`] with the same Ed25519 +//! key, key ID, and gateway identity as the sandbox session tokens, under a +//! distinct profile (`typ`, subject, claims), so no sandbox, extension, user, +//! or peer credential is accepted on the stream and the supervisor token is +//! accepted nowhere else. [`SupervisorTokenAuthenticator`] slots the verifier +//! into the gateway's authenticator chain. + +use std::sync::Arc; + +use async_trait::async_trait; +use jsonwebtoken::decode_header; +use openshell_core::jwt::SUPERVISOR_JWT_TYPE; +use openshell_core::supervisor_host::HostIdentity; + +use super::authenticator::{AuthRejection, Authentication, Authenticator, bearer_credential}; +use super::principal::{Principal, SupervisorPrincipal}; +use super::sandbox_jwt::SandboxSessionJwtAuthority; + +/// Authenticates bearers typed as supervisor tokens. Any other bearer is not +/// this authenticator's business; a supervisor-typed bearer that fails +/// verification ends the chain. +pub struct SupervisorTokenAuthenticator { + authority: Arc, +} + +impl SupervisorTokenAuthenticator { + pub fn new(authority: Arc) -> Self { + Self { authority } + } +} + +impl std::fmt::Debug for SupervisorTokenAuthenticator { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SupervisorTokenAuthenticator") + .finish_non_exhaustive() + } +} + +#[async_trait] +impl Authenticator for SupervisorTokenAuthenticator { + async fn authenticate( + &self, + headers: &http::HeaderMap, + _path: &str, + ) -> Result { + let Some(token) = bearer_credential(headers) else { + return Ok(Authentication::NotApplicable); + }; + let Ok(header) = decode_header(token) else { + return Ok(Authentication::NotApplicable); + }; + if header.typ.as_deref() != Some(SUPERVISOR_JWT_TYPE) { + return Ok(Authentication::NotApplicable); + } + let authenticated = self.authority.verify_supervisor_token(token)?; + Ok(Authentication::Accepted(Principal::Supervisor( + SupervisorPrincipal { + identity: HostIdentity { + subject: authenticated.subject, + token_id: authenticated.token_id.to_string(), + expires_at: authenticated.expires_at, + }, + }, + ))) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::auth::authenticator::{AuthFailureReason, AuthMechanism}; + use openshell_bootstrap::jwt::generate_jwt_key; + + fn authority(gateway_id: &str) -> Arc { + let material = generate_jwt_key().expect("jwt key"); + Arc::new( + SandboxSessionJwtAuthority::from_pem( + material.signing_key_pem.as_bytes(), + material.public_key_pem.as_bytes(), + material.kid, + gateway_id, + None, + ) + .expect("authority"), + ) + } + + fn headers(token: &str) -> http::HeaderMap { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::AUTHORIZATION, + format!("Bearer {token}").parse().unwrap(), + ); + headers + } + + #[tokio::test] + async fn supervisor_tokens_authenticate_only_as_the_supervisor() { + let attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; + let gateway = authority("test-gateway"); + let minted = gateway + .mint_supervisor_token(Duration::from_mins(15)) + .expect("mint"); + let authenticator = SupervisorTokenAuthenticator::new(gateway.clone()); + + let accepted = authenticator + .authenticate(&headers(minted.token.expose_secret()), attach) + .await + .expect("valid token"); + match accepted { + Authentication::Accepted(Principal::Supervisor(supervisor)) => { + assert_eq!( + supervisor.identity.subject, + "openshell-supervisor:test-gateway" + ); + assert_eq!(supervisor.identity.token_id, minted.token_id.to_string()); + assert_eq!(supervisor.identity.expires_at, minted.expires_at); + } + other => panic!("unexpected outcome {other:?}"), + } + + // Session tokens of the same authority, opaque bearers, and non-JWT + // bearers fall through to the other authenticators. + let identity = crate::auth::sandbox_session::PersistedSandboxIdentity::new().unwrap(); + let launch = gateway + .mint_persisted_launch("sandbox-a", &identity) + .unwrap(); + for bearer in [ + launch.supervisor.gateway_token.expose_secret(), + "opaque-user-token", + "a.b.c", + ] { + assert!(matches!( + authenticator + .authenticate(&headers(bearer), attach) + .await + .unwrap(), + Authentication::NotApplicable + )); + } + // And the session verifier refuses the supervisor token. + assert!( + gateway + .verify_gateway_token(minted.token.expose_secret()) + .is_err() + ); + + // A supervisor-typed token signed by another gateway's key fails + // closed, as does one for another gateway identity. + let other_key = authority("test-gateway"); + let foreign = other_key + .mint_supervisor_token(Duration::from_mins(15)) + .unwrap(); + let rejection = authenticator + .authenticate(&headers(foreign.token.expose_secret()), attach) + .await + .expect_err("foreign key must be rejected"); + assert_eq!(rejection.mechanism, AuthMechanism::SupervisorToken); + assert_eq!(rejection.reason, AuthFailureReason::InvalidCredential); + assert_eq!(rejection.status.code(), tonic::Code::Unauthenticated); + let other_gateway = authority("other-gateway"); + let rejection = SupervisorTokenAuthenticator::new(other_gateway.clone()) + .authenticate( + &headers( + other_gateway + .mint_supervisor_token(Duration::from_mins(15)) + .unwrap() + .token + .expose_secret(), + ), + attach, + ) + .await + .expect("own token verifies"); + assert!(matches!(rejection, Authentication::Accepted(_))); + } +} diff --git a/crates/openshell-server/src/auth/workspace_authz.rs b/crates/openshell-server/src/auth/workspace_authz.rs index e8739d9cd0..bc5cac8778 100644 --- a/crates/openshell-server/src/auth/workspace_authz.rs +++ b/crates/openshell-server/src/auth/workspace_authz.rs @@ -207,10 +207,10 @@ pub async fn authorize_workspace( workspace, grant: AuthGrant::Sandbox, }), - Principal::Peer(_) | Principal::SupervisorHost(_) => Err(deny( + Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, - "gateway peer and supervisor host principals cannot perform workspace operations", + "gateway peer and supervisor principals cannot perform workspace operations", )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } @@ -244,8 +244,8 @@ pub fn require_platform_admin(admin_role: &str, principal: &Principal) -> Result } Principal::User(_) => "platform admin role required for cross-workspace operations", Principal::Sandbox(_) => "sandbox principals cannot perform cross-workspace operations", - Principal::Peer(_) | Principal::SupervisorHost(_) => { - "gateway peer and supervisor host principals cannot perform cross-workspace operations" + Principal::Peer(_) | Principal::Supervisor(_) => { + "gateway peer and supervisor principals cannot perform cross-workspace operations" } Principal::Anonymous => return Err(Status::unauthenticated("authentication required")), }; diff --git a/crates/openshell-server/src/certgen.rs b/crates/openshell-server/src/certgen.rs index 10c6fb3318..764ceba98c 100644 --- a/crates/openshell-server/src/certgen.rs +++ b/crates/openshell-server/src/certgen.rs @@ -12,10 +12,6 @@ //! - **Kubernetes JWT-only mode** (`--jwt-only`): create only the //! sandbox-JWT signing Secret. Used when another controller, such as //! cert-manager, owns the TLS Secrets. -//! - **Kubernetes client-only mode** (`--client-only`): create only the client -//! TLS Secret of a fresh private PKI (certificate, key, and CA), with no -//! server certificate and no sandbox-JWT signing key. Issues the shared -//! supervisor's host identity; the gateway mounts the Secret's `ca.crt`. //! - **Local mode** (`--output-dir `): write PEMs to the local package //! filesystem layout. Used by systemd units' `ExecStartPre`. Also copies //! client materials to @@ -32,9 +28,7 @@ use k8s_openapi::api::core::v1::{ConfigMap, Secret}; use kube::Client; use kube::api::{Api, ObjectMeta, PostParams}; use miette::{IntoDiagnostic, Result, WrapErr}; -use openshell_bootstrap::pki::{ - DEFAULT_SERVER_SANS, PkiBundle, generate_pki, generate_supervisor_host_pki, -}; +use openshell_bootstrap::pki::{DEFAULT_SERVER_SANS, PkiBundle, generate_pki}; use openshell_core::paths::{create_dir_restricted, set_file_owner_only}; use std::collections::{BTreeMap, BTreeSet}; use std::fmt; @@ -44,7 +38,6 @@ use tracing::{info, warn}; use tracing_subscriber::EnvFilter; #[derive(Args, Debug)] -#[allow(clippy::struct_excessive_bools)] pub struct CertgenArgs { /// Write PEMs to a filesystem directory instead of Kubernetes Secrets. /// When set, the kube-related flags are not required. @@ -58,7 +51,7 @@ pub struct CertgenArgs { namespace: Option, /// Name of the server TLS Secret (`kubernetes.io/tls`) to create. - #[arg(long, required_unless_present_any = ["output_dir", "jwt_only", "client_only"])] + #[arg(long, required_unless_present_any = ["output_dir", "jwt_only"])] server_secret_name: Option, /// Name of the client TLS Secret (`kubernetes.io/tls`) to create. @@ -68,7 +61,7 @@ pub struct CertgenArgs { /// Name of the sandbox-JWT signing-key Secret (`Opaque`) to create. /// Holds `signing.pem`, `public.pem`, and `kid` keys. Mounted on the /// gateway pod (only) so it can mint and validate per-sandbox JWTs. - #[arg(long, required_unless_present_any = ["output_dir", "client_only"])] + #[arg(long, required_unless_present = "output_dir")] jwt_secret_name: Option, /// Create only the sandbox-JWT signing-key Secret in Kubernetes mode. @@ -76,14 +69,6 @@ pub struct CertgenArgs { #[arg(long, conflicts_with = "output_dir")] jwt_only: bool, - /// Create only the client TLS Secret of a fresh private PKI in Kubernetes - /// mode: the shared supervisor host's certificate, key, and issuing CA. - #[arg( - long, - conflicts_with_all = ["output_dir", "jwt_only", "jwt_secret_name", "server_secret_name", "server_sans", "backend_ca_configmap_name"] - )] - client_only: bool, - /// Extra Subject Alternative Name for the server certificate. Repeatable. /// Auto-detected as an IP address or DNS name. #[arg(long = "server-san", value_name = "SAN")] @@ -138,11 +123,7 @@ pub async fn run(args: CertgenArgs) -> Result<()> { if let Some(dir) = args.output_dir.as_deref() { run_local(dir, &args.server_sans) } else { - let bundle = if args.client_only { - generate_supervisor_host_pki()? - } else { - generate_pki(&args.server_sans)? - }; + let bundle = generate_pki(&args.server_sans)?; run_kubernetes(&args, &bundle).await?; if let Some(ref cm_name) = args.backend_ca_configmap_name { @@ -222,47 +203,14 @@ async fn run_kubernetes(args: &CertgenArgs, bundle: &PkiBundle) -> Result<()> { return Ok(()); } - let client_name = args - .client_secret_name - .as_deref() - .ok_or_else(|| miette::miette!("--client-secret-name is required"))?; - if args.client_only { - if api - .get_opt(client_name) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to read secret {client_name}"))? - .is_some() - { - info!( - namespace = %namespace, - client = %client_name, - "client TLS secret already exists, skipping." - ); - return Ok(()); - } - let client_secret = tls_secret( - client_name, - &bundle.client_cert_pem, - &bundle.client_key_pem, - &bundle.ca_cert_pem, - ); - api.create(&PostParams::default(), &client_secret) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to create secret {client_name}"))?; - info!( - namespace = %namespace, - client = %client_name, - "client TLS secret created." - ); - return Ok(()); - } - let server_name = args .server_secret_name .as_deref() .ok_or_else(|| miette::miette!("--server-secret-name is required"))?; + let client_name = args + .client_secret_name + .as_deref() + .ok_or_else(|| miette::miette!("--client-secret-name is required"))?; let server_exists = api .get_opt(server_name) .await @@ -1038,37 +986,6 @@ mod tests { use openshell_bootstrap::pki::generate_pki; use std::path::Path; - #[derive(clap::Parser)] - struct Cli { - #[command(flatten)] - args: super::CertgenArgs, - } - - #[test] - fn client_only_mode_needs_only_the_client_secret_and_refuses_the_rest() { - use clap::Parser as _; - let client_only = [ - "generate-certs", - "--namespace=gateway", - "--client-only", - "--client-secret-name=client", - ]; - assert!(Cli::try_parse_from(client_only).unwrap().args.client_only); - for conflicting in [ - "--jwt-secret-name=jwt", - "--jwt-only", - "--server-secret-name=server", - "--server-san=example.test", - "--output-dir=/tmp/pki", - ] { - let mut args = client_only.to_vec(); - args.push(conflicting); - assert!(Cli::try_parse_from(args).is_err(), "{conflicting}"); - } - let missing_client = ["generate-certs", "--namespace=gateway", "--client-only"]; - assert!(Cli::try_parse_from(missing_client).is_err()); - } - // ── Kubernetes-mode decision ── #[test] diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 4c61f4a0bc..374a3b5759 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -2788,8 +2788,8 @@ async fn resolve_sandbox_by_name_for_principal( Principal::User(_) | Principal::Operator(_) => { sandbox.ok_or_else(|| Status::not_found("sandbox not found")) } - Principal::Peer(_) | Principal::SupervisorHost(_) => Err(Status::permission_denied( - "gateway peer and supervisor host principals may not resolve sandbox configuration", + Principal::Peer(_) | Principal::Supervisor(_) => Err(Status::permission_denied( + "gateway peer and supervisor principals may not resolve sandbox configuration", )), Principal::Anonymous => Err(Status::unauthenticated( "sandbox-scoped methods require an authenticated caller", diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 59125bd8b7..b77b9ed45b 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -4294,7 +4294,7 @@ async fn exchange_provider_subject_token_inner( | Principal::User(_) | Principal::Operator(_) | Principal::Peer(_) - | Principal::SupervisorHost(_), + | Principal::Supervisor(_), ) => Some(tonic::Code::PermissionDenied), Some(Principal::Anonymous) | None => Some(tonic::Code::Unauthenticated), }; diff --git a/crates/openshell-server/src/grpc/provider_credentials_tests.rs b/crates/openshell-server/src/grpc/provider_credentials_tests.rs index 0b78a9e2e5..c602e28461 100644 --- a/crates/openshell-server/src/grpc/provider_credentials_tests.rs +++ b/crates/openshell-server/src/grpc/provider_credentials_tests.rs @@ -761,7 +761,6 @@ async fn real_tls_operator_boundary_and_admin_capabilities() { identity, fingerprint, crate::auth::audit::ConnectionSource::Peer(peer), - None, ) .await; } diff --git a/crates/openshell-server/src/grpc/supervisor_host.rs b/crates/openshell-server/src/grpc/supervisor_host.rs index 09e948373e..e51ca87097 100644 --- a/crates/openshell-server/src/grpc/supervisor_host.rs +++ b/crates/openshell-server/src/grpc/supervisor_host.rs @@ -4,10 +4,9 @@ //! The private `openshell.supervisor.host.v1` service: the gateway end of the //! shared supervisor's assignment stream. //! -//! The router admits only connections whose client certificate verified -//! against the supervisor host CA, so the handler sees a -//! [`Principal::SupervisorHost`] and hands the stream to the controller the -//! in-process compute driver installed. +//! The router admits only requests bearing a verified supervisor token, so +//! the handler sees a [`Principal::Supervisor`] and hands the stream to the +//! controller the in-process compute driver installed. use std::pin::Pin; use std::sync::Arc; @@ -45,10 +44,10 @@ impl SupervisorHost for SupervisorHostService { request: Request>, ) -> Result, Status> { let identity = match request.extensions().get::() { - Some(Principal::SupervisorHost(host)) => host.identity.clone(), + Some(Principal::Supervisor(supervisor)) => supervisor.identity.clone(), _ => { return Err(Status::permission_denied( - "the supervisor host stream requires a supervisor host certificate", + "the supervisor assignment stream requires a supervisor token", )); } }; diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index 1c37204181..56156f7d3f 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -68,8 +68,8 @@ fn membership_filter_subject<'a>( } } Principal::Sandbox(_) | Principal::Operator(_) => Ok(None), - Principal::Peer(_) | Principal::SupervisorHost(_) => Err(Status::permission_denied( - "gateway peer and supervisor host principals cannot list workspaces", + Principal::Peer(_) | Principal::Supervisor(_) => Err(Status::permission_denied( + "gateway peer and supervisor principals cannot list workspaces", )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 241d4787a5..417312fc25 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -241,12 +241,59 @@ fn spawn_gateway_extension_token_refresh( } }); } + +/// Mint the shared supervisor's token and renew it at half its lifetime so +/// the delivered copy is never close to expiry when the supervisor restarts +/// or reconnects. Minting failures are logged and retried shortly. +fn spawn_supervisor_token_refresh( + authority: Arc, + ttl: Duration, + controller: openshell_core::supervisor_host::SupervisorHostController, + mut shutdown_rx: watch::Receiver, +) { + tokio::spawn(async move { + loop { + let delay = match authority.mint_supervisor_token(ttl) { + Ok(minted) => { + let remaining = minted.expires_at.saturating_sub( + i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()), + ) + .unwrap_or(i64::MAX), + ); + controller.publish_credential( + openshell_core::supervisor_host::SupervisorCredential { + token: minted.token, + expires_at: minted.expires_at, + }, + ); + Duration::from_secs(u64::try_from(remaining).unwrap_or(0) / 2) + .max(Duration::from_secs(1)) + } + Err(error) => { + warn!(%error, "failed to mint the supervisor token; retrying"); + Duration::from_secs(10) + } + }; + tokio::select! { + () = tokio::time::sleep(delay) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + } + }); +} pub use auth::audit::ConnectionSource; pub use multiplex::{MultiplexService, MultiplexedService}; pub use persistence::Store; use sandbox_index::SandboxIndex; use sandbox_watch::SandboxWatchBus; -pub use tls::{PeerRealm, TlsAcceptor, TlsHandshake}; +pub use tls::TlsAcceptor; use tracing_bus::TracingLogBus; pub(crate) struct ServerStartupConfig { @@ -744,6 +791,37 @@ pub(crate) async fn run_server( if let Some(issuer) = extension_jwt_issuer { spawn_gateway_extension_token_refresh(issuer, gateway_extension_credentials); } + // The shared supervisor authenticates with a token this gateway mints + // from its launch-signing key and carries only over TLS, so the topology + // needs both. + if let Some(controller) = state.compute.supervisor_host() { + let (authority, jwt) = state + .sandbox_session_jwt_authority + .clone() + .zip(config.gateway_jwt.as_ref()) + .ok_or_else(|| { + Error::config( + "the shared supervisor requires gateway_jwt signing material to mint \ + supervisor tokens; configure [openshell.gateway.gateway_jwt] or disable \ + supervisor_topology", + ) + })?; + if config.tls.is_none() { + return Err(Error::config( + "the shared supervisor requires gateway TLS; its token travels only over TLS. \ + Enable TLS or disable supervisor_topology", + )); + } + let ttl = jwt.token_ttl().clamp( + openshell_core::jwt::MIN_SESSION_TOKEN_TTL, + openshell_core::jwt::MAX_SESSION_TOKEN_TTL, + ); + info!( + ttl_secs = ttl.as_secs(), + "shared supervisor assignment stream enabled; minting supervisor tokens" + ); + spawn_supervisor_token_refresh(authority, ttl, controller.clone(), shutdown_rx.clone()); + } if state.sandbox_session_jwt_authority.is_some() && state.compute.supports_sandbox_authentication() @@ -933,7 +1011,6 @@ pub(crate) async fn run_server( } // Build TLS acceptor when TLS is configured; otherwise serve plaintext. - let supervisor_host = state.compute.supervisor_host(); let tls_acceptor = if let Some(tls) = &config.tls { let acceptor = TlsAcceptor::from_files( &tls.cert_path, @@ -944,19 +1021,6 @@ pub(crate) async fn run_server( tls.external_key_path.as_deref(), tls.external_server_names.clone(), )?; - // The shared supervisor host authenticates with a client certificate - // from its own CA; the handshake accepts it and the connection is - // classified as a host, never as a user. - let acceptor = match supervisor_host { - Some(host) => { - info!( - trust_anchor = %host.trust_anchor_path().display(), - "shared supervisor host assignment stream enabled" - ); - acceptor.with_supervisor_host_ca(host.trust_anchor_path())? - } - None => acceptor, - }; // Spawn file-watcher-based TLS certificate reload worker. // Watches parent directories of cert/key/CA files and atomically @@ -964,10 +1028,6 @@ pub(crate) async fn run_server( acceptor.spawn_reload_worker(shutdown_rx.clone()); Some(acceptor) - } else if supervisor_host.is_some() { - return Err(Error::config( - "the shared supervisor host requires gateway TLS; enable TLS or disable supervisor_topology", - )); } else { info!("TLS disabled — accepting plaintext connections"); None @@ -1208,30 +1268,12 @@ fn spawn_gateway_connection( ); } Ok(ConnectionProtocol::Tls | ConnectionProtocol::Unknown) => { - // acceptor.handshake() snapshots the current TLS config - // and verifiers together; the snapshot stays alive for - // the handshake and classifies its result. - let handshake = acceptor.handshake(); - match handshake.accept(stream).await { + // acceptor.acceptor() snapshots the current TLS config; + // the returned acceptor owns an Arc that stays alive for + // the full duration of the handshake. + match acceptor.acceptor().accept(stream).await { Ok(tls_stream) => { - // A presented certificate must belong to exactly - // one realm; anything else closes the connection. - let (peer_identity, supervisor_host) = match handshake - .classify(&tls_stream) - { - Ok(PeerRealm::Anonymous) => (None, None), - Ok(PeerRealm::Client) => { - (multiplex::extract_peer_identity(&tls_stream), None) - } - Ok(PeerRealm::SupervisorHost(identity)) => ( - None, - Some(auth::principal::SupervisorHostPrincipal { identity }), - ), - Err(e) => { - warn!(error = %e, client = %addr, "Rejected client certificate"); - return; - } - }; + let peer_identity = multiplex::extract_peer_identity(&tls_stream); let certificate_sha256 = match multiplex::extract_peer_certificate_sha256(&tls_stream) { Ok(fingerprint) => fingerprint, @@ -1246,7 +1288,6 @@ fn spawn_gateway_connection( peer_identity, certificate_sha256, ConnectionSource::Peer(addr), - supervisor_host, ) .await { @@ -1658,10 +1699,10 @@ pub struct ComputeDriverBuildContext<'a> { } impl ComputeDriverBuildContext<'_> { - /// Let the gateway serve the shared supervisor host whose desired - /// assignments this in-process driver owns. The gateway adds the - /// controller's trust anchor to its TLS client roots and routes - /// authenticated host streams to it. + /// Let the gateway serve the shared supervisor whose desired assignments + /// this in-process driver owns. The gateway mints the supervisor token, + /// publishes each renewal into the controller for the driver to deliver, + /// and routes streams authenticated by that token to it. pub fn install_supervisor_host( &self, controller: openshell_core::supervisor_host::SupervisorHostController, diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 0f3ce32c33..ff1ca10da2 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -49,7 +49,7 @@ use crate::{ auth::authz::AuthzPolicy, auth::identity::Identity, auth::oidc::{self, OidcAuthenticator}, - auth::principal::{OperatorPrincipal, Principal, SupervisorHostPrincipal, UserPrincipal}, + auth::principal::{OperatorPrincipal, Principal, UserPrincipal}, auth::workspace_authz::{MinWorkspaceRole, authorize_workspace}, http_router, service_http_router, }; @@ -258,31 +258,22 @@ impl MultiplexService { where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - self.serve_with_verified_certificate(stream, peer_identity, None, source, None) + self.serve_with_verified_certificate(stream, peer_identity, None, source) .await } - /// TLS listener only: fingerprint, identity, and supervisor host must - /// originate from its verified handshake. A supervisor host connection - /// carries no user or operator identity, whatever its certificate says. + /// TLS listener only: fingerprint and identity must originate from its verified handshake. pub async fn serve_with_verified_certificate( &self, stream: S, peer_identity: Option, certificate_sha256: Option, source: ConnectionSource, - supervisor_host: Option, ) -> Result<(), Box> where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - let peer_identity = if supervisor_host.is_some() { - None - } else { - peer_identity - }; - let operator = if self.state.config.mtls_auth.operator_enabled && supervisor_host.is_none() - { + let operator = if self.state.config.mtls_auth.operator_enabled { peer_identity .as_ref() .filter(|identity| { @@ -335,7 +326,6 @@ impl MultiplexService { ) .with_peer_certificate(certificate_sha256) .with_operator(operator) - .with_supervisor_host(supervisor_host) .with_connection_source(source) .with_logons(self.state.logons.clone()); let grpc_service = @@ -825,13 +815,10 @@ fn gateway_principal_fields(principal: &Principal) -> BTreeMap { fields.insert("replica_id".to_string(), peer.replica_id.clone()); fields.insert("pod_uid".to_string(), peer.pod_uid.clone()); } - Principal::SupervisorHost(host) => { - fields.insert("kind".to_string(), "supervisor_host".to_string()); - fields.insert("subject".to_string(), host.identity.subject.clone()); - fields.insert( - "certificate_sha256".to_string(), - host.identity.certificate_sha256.clone(), - ); + Principal::Supervisor(supervisor) => { + fields.insert("kind".to_string(), "supervisor".to_string()); + fields.insert("subject".to_string(), supervisor.identity.subject.clone()); + fields.insert("token_id".to_string(), supervisor.identity.token_id.clone()); } Principal::Anonymous => { fields.insert("kind".to_string(), "anonymous".to_string()); @@ -1059,10 +1046,12 @@ where /// 2. `ComputeDriverAuthenticator` (path-scoped to `IssueSandboxToken`) /// — delegates a driver-native credential and receives a sandbox identity /// so the handler can mint a gateway JWT. No-op on every other path. -/// 3. `SandboxSessionJwtAuthenticator` — validates generation-bound gateway +/// 3. `SupervisorTokenAuthenticator` — validates the shared supervisor's +/// gateway-minted token (its own `typ`); applies to no other bearer. +/// 4. `SandboxSessionJwtAuthenticator` — validates generation-bound gateway /// JWTs against the durable sandbox identity. Untyped legacy sandbox JWTs /// are not admitted. -/// 4. `OidcAuthenticator` — validates user Bearer tokens against the +/// 5. `OidcAuthenticator` — validates user Bearer tokens against the /// configured OIDC issuer. Returns `Unauthenticated` for missing /// Bearer headers so non-OIDC clients can't sneak through. /// @@ -1085,6 +1074,9 @@ fn build_authenticator_chain(state: &ServerState) -> Option authenticators.push(driver); } if let Some(authority) = state.sandbox_session_jwt_authority.clone() { + authenticators.push(Arc::new( + crate::auth::supervisor_token::SupervisorTokenAuthenticator::new(authority.clone()), + )); authenticators.push(Arc::new( crate::auth::sandbox_jwt::SandboxSessionJwtAuthenticator::new( authority, @@ -1125,8 +1117,6 @@ pub struct AuthGrpcRouter { mtls_auth_enabled: bool, allow_unauthenticated_users: bool, operator: Option, - /// The connection presented the shared supervisor host certificate. - supervisor_host: Option, connection_source: Option, logons: audit::Logons, } @@ -1158,7 +1148,6 @@ impl AuthGrpcRouter { mtls_auth_enabled, allow_unauthenticated_users, operator: None, - supervisor_host: None, connection_source: None, logons: audit::Logons::default(), } @@ -1169,12 +1158,6 @@ impl AuthGrpcRouter { self } - /// The verified supervisor host identity of this connection, if any. - fn with_supervisor_host(mut self, supervisor_host: Option) -> Self { - self.supervisor_host = supervisor_host; - self - } - fn with_connection_source(mut self, source: ConnectionSource) -> Self { self.connection_source = Some(source); self @@ -1232,7 +1215,6 @@ where let mtls_auth_enabled = self.mtls_auth_enabled; let allow_unauthenticated_users = self.allow_unauthenticated_users; let operator = self.operator.clone(); - let supervisor_host = self.supervisor_host.clone(); let connection_source = self.connection_source; let logons = self.logons.clone(); let peer_certificate_sha256 = self.peer_certificate_sha256.clone(); @@ -1252,20 +1234,7 @@ where return inner.ready().await?.call(req).await; } - let principal = if let Some(host) = supervisor_host { - if req.headers().contains_key(http::header::AUTHORIZATION) { - audit::emit_authentication_failure( - &path, - &origin(req.headers()), - Some(AuthMechanism::SupervisorHost), - AuthFailureReason::ConflictingCredentials, - ); - return Ok(status_response(tonic::Status::unauthenticated( - "supervisor host certificates must not be combined with authorization headers", - ))); - } - Principal::SupervisorHost(host) - } else if let Some(operator) = operator { + let principal = if let Some(operator) = operator { if req.headers().contains_key(http::header::AUTHORIZATION) { audit::emit_authentication_failure( &path, @@ -1372,10 +1341,12 @@ where Principal::Peer(_) if !crate::auth::method_authz::is_peer_callable(&path) => { method_denied("gateway peer principals may not call this method") } - Principal::SupervisorHost(_) - if !crate::auth::method_authz::is_supervisor_host_callable(&path) => + Principal::Supervisor(_) + if !crate::auth::method_authz::is_supervisor_callable(&path) => { - method_denied("supervisor hosts may call only the supervisor host stream") + method_denied( + "supervisor tokens may call only the supervisor assignment stream", + ) } Principal::Anonymous => { return Ok(status_response(tonic::Status::unauthenticated( @@ -1385,7 +1356,7 @@ where Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) - | Principal::SupervisorHost(_) => None, + | Principal::Supervisor(_) => None, }; if let Some((denial, status)) = denial { audit::emit_authorization_denial_for(&path, &origin, &principal, denial); @@ -3592,29 +3563,32 @@ mod tests { } #[tokio::test] - async fn supervisor_host_connections_reach_only_the_host_stream() { + async fn supervisor_tokens_reach_only_the_assignment_stream() { let attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; - let host = SupervisorHostPrincipal { + let supervisor = Principal::Supervisor(crate::auth::principal::SupervisorPrincipal { identity: openshell_core::supervisor_host::HostIdentity { - subject: "openshell-supervisor-host".to_string(), - certificate_sha256: "verified-leaf".to_string(), + subject: "openshell-supervisor:test".to_string(), + token_id: "token".to_string(), + expires_at: i64::MAX, }, - }; - // The host realm is set even when mTLS user promotion is on: the - // connection carries no peer identity, so it can never be a user. - let host_router = |recorder| { - AuthGrpcRouter::with_peer_identity(recorder, None, None, None, true, false) - .with_supervisor_host(Some(host.clone())) + }); + let supervisor_chain = || { + AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok( + Authentication::Accepted(supervisor.clone()), + )))]) }; for (method, permitted) in [ (attach, true), ("/openshell.v1.OpenShell/ListSandboxes", false), ("/openshell.v1.OpenShell/ConnectSupervisor", false), + ("/openshell.v1.OpenShell/GetSandboxConfig", false), ("/openshell.v1.OpenShell/GetProviderCredentials", false), + ("/openshell.v1.OpenShell/RefreshSandboxToken", false), + ("/openshell.v1.OpenShell/PeerRelay", false), ("/openshell.v1.OpenShell/Unknown", false), ] { let (recorder, seen) = PrincipalRecorder::new(); - let response = host_router(recorder) + let response = AuthGrpcRouter::new(recorder, Some(supervisor_chain()), None) .call(empty_request(method)) .await .unwrap(); @@ -3622,26 +3596,16 @@ mod tests { assert!(grpc_status(&response).is_none(), "{method}"); assert!(matches!( *seen.lock().unwrap(), - Some(Principal::SupervisorHost(_)) + Some(Principal::Supervisor(_)) )); } else { assert_eq!(grpc_status(&response).as_deref(), Some("7"), "{method}"); } } - // A host connection may not also present a bearer identity. - let (recorder, seen) = PrincipalRecorder::new(); - let mut request = empty_request(attach); - request.headers_mut().insert( - http::header::AUTHORIZATION, - HeaderValue::from_static("Bearer user-token"), - ); - let response = host_router(recorder).call(request).await.unwrap(); - assert_eq!(grpc_status(&response).as_deref(), Some("16")); - assert!(seen.lock().unwrap().is_none()); - - // No other principal reaches the host stream: an mTLS user, an - // operator, a sandbox, a peer, or an unauthenticated dev user. + // No other principal reaches the stream: an mTLS user, an + // operator, a sandbox, a peer, or an unauthenticated dev user, + // even when local unauthenticated users are allowed. let (recorder, _) = PrincipalRecorder::new(); let response = AuthGrpcRouter::with_peer_identity( recorder, @@ -3655,6 +3619,22 @@ mod tests { .await .unwrap(); assert_eq!(grpc_status(&response).as_deref(), Some("7")); + let (recorder, seen) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::with_peer_identity( + recorder, + Some(AuthenticatorChain::new(vec![Arc::new( + MockAuthenticator::returning(Ok(Authentication::NotApplicable)), + )])), + None, + None, + false, + true, + ) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + assert!(seen.lock().unwrap().is_none()); let (recorder, _) = PrincipalRecorder::new(); let response = AuthGrpcRouter::new(recorder, None, None) .with_operator(Some(operator_principal())) diff --git a/crates/openshell-server/src/tls.rs b/crates/openshell-server/src/tls.rs index e479adcdb7..26bcb713ce 100644 --- a/crates/openshell-server/src/tls.rs +++ b/crates/openshell-server/src/tls.rs @@ -13,12 +13,10 @@ use std::time::Duration; use arc_swap::ArcSwap; use notify::event::EventKind; use notify::{Event, RecursiveMode, Watcher}; -use openshell_core::supervisor_host::HostIdentity; use openshell_core::{Error, Result}; use openshell_ocsf::{ConfigStateChangeBuilder, EventContext, SeverityId, StateId, StatusId}; use rustls::ServerConfig; -use rustls::pki_types::{CertificateDer, PrivateKeyDer, UnixTime}; -use rustls::server::danger::ClientCertVerifier; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier}; use rustls::sign::CertifiedKey; use tokio::sync::{mpsc, watch}; @@ -33,7 +31,7 @@ use tracing::{debug, info, warn}; /// re-read from the same files without the caller tracking them separately. #[derive(Clone)] pub struct TlsAcceptor { - state: Arc>, + config: Arc>, cert_path: PathBuf, key_path: PathBuf, client_ca_path: Option, @@ -41,103 +39,9 @@ pub struct TlsAcceptor { external_cert_path: Option, external_key_path: Option, external_server_names: Vec, - /// CA that issues the shared supervisor host's client certificate. Its - /// certificates are accepted by the handshake and classified into the - /// supervisor host realm; they never become user identities. - supervisor_host_ca_path: Option, reload_spawned: Arc, } -/// One generation of TLS configuration: the handshake config and the -/// per-realm verifiers built from the same CA files, swapped together so a -/// handshake is always classified against the roots that accepted it. -struct AcceptorState { - config: Arc, - /// Verifies chains against the ordinary client CA alone. - client_verifier: Option>, - /// Verifies chains against the supervisor host CA alone. - supervisor_host_verifier: Option>, -} - -/// Who a finished handshake proves the peer to be. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum PeerRealm { - /// No client certificate was presented. - Anonymous, - /// The chain verifies against the ordinary client CA. - Client, - /// The chain verifies against the supervisor host CA. - SupervisorHost(HostIdentity), -} - -/// A snapshot of the acceptor for one connection: the handshake and the -/// classification of its result use the same configuration generation. -#[derive(Clone)] -pub struct TlsHandshake { - state: Arc, -} - -impl TlsHandshake { - /// Complete the server side of a TLS handshake. - pub async fn accept(&self, stream: S) -> std::io::Result> - where - S: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin, - { - tokio_rustls::TlsAcceptor::from(self.state.config.clone()) - .accept(stream) - .await - } - - /// Classify a finished handshake. Fails closed: a presented chain that - /// verifies against neither configured CA, or against both, is refused - /// rather than treated as an ordinary client. - /// - /// # Errors - /// - /// Returns an error when the chain cannot be attributed to exactly one - /// realm or the host identity cannot be derived from it. - pub fn classify(&self, stream: &tokio_rustls::server::TlsStream) -> Result { - let (_, connection) = stream.get_ref(); - let Some((end_entity, intermediates)) = connection - .peer_certificates() - .and_then(|certificates| certificates.split_first()) - else { - return Ok(PeerRealm::Anonymous); - }; - let now = UnixTime::now(); - let verifies = |verifier: &Option>| { - verifier.as_ref().is_some_and(|verifier| { - verifier - .verify_client_cert(end_entity, intermediates, now) - .is_ok() - }) - }; - match ( - verifies(&self.state.supervisor_host_verifier), - verifies(&self.state.client_verifier), - ) { - (true, true) => Err(Error::tls( - "client certificate verifies against both the client CA and the supervisor host CA; refusing the ambiguous identity", - )), - (true, false) => { - let subject = certificate_common_name(end_entity).ok_or_else(|| { - Error::tls("supervisor host certificate has no subject common name") - })?; - let digest = openshell_crypto::sha256(end_entity.as_ref()) - .map_err(|e| Error::tls(format!("fingerprint host certificate: {e}")))?; - Ok(PeerRealm::SupervisorHost(HostIdentity { - subject, - certificate_sha256: hex::encode(digest), - })) - } - (false, true) => Ok(PeerRealm::Client), - (false, false) => Err(Error::tls( - "client certificate was accepted by the handshake but verifies against no configured client CA", - )), - } - } -} - impl TlsAcceptor { /// Create a new TLS acceptor from certificate and key files. /// @@ -165,7 +69,7 @@ impl TlsAcceptor { external_key_path: Option<&Path>, external_server_names: Vec, ) -> Result { - let state = build_acceptor_state( + let config = build_server_config( cert_path, key_path, client_ca_path, @@ -173,10 +77,9 @@ impl TlsAcceptor { external_cert_path, external_key_path, &external_server_names, - None, )?; Ok(Self { - state: Arc::new(ArcSwap::from_pointee(state)), + config: Arc::new(ArcSwap::from(config)), cert_path: cert_path.to_path_buf(), key_path: key_path.to_path_buf(), client_ca_path: client_ca_path.map(Path::to_path_buf), @@ -184,31 +87,17 @@ impl TlsAcceptor { external_cert_path: external_cert_path.map(Path::to_path_buf), external_key_path: external_key_path.map(Path::to_path_buf), external_server_names, - supervisor_host_ca_path: None, reload_spawned: Arc::new(AtomicBool::new(false)), }) } - /// Also accept client certificates issued by the supervisor host CA at - /// `ca_path`; [`TlsHandshake::classify`] places them in - /// [`PeerRealm::SupervisorHost`]. - /// - /// # Errors - /// - /// Returns an error if the CA file cannot be read or parsed. - pub fn with_supervisor_host_ca(mut self, ca_path: &Path) -> Result { - self.supervisor_host_ca_path = Some(ca_path.to_path_buf()); - self.reload()?; - Ok(self) - } - /// Re-read certificates from the same paths used at construction and /// atomically swap the active config. /// /// Returns `Ok(())` when the new config was built and swapped successfully. /// Returns `Err(...)` if cert/key loading fails — the old config is preserved. pub fn reload(&self) -> Result<()> { - let state = build_acceptor_state( + let new_config = build_server_config( &self.cert_path, &self.key_path, self.client_ca_path.as_deref(), @@ -216,9 +105,8 @@ impl TlsAcceptor { self.external_cert_path.as_deref(), self.external_key_path.as_deref(), &self.external_server_names, - self.supervisor_host_ca_path.as_deref(), )?; - self.state.store(Arc::new(state)); + self.config.store(new_config); let event = ConfigStateChangeBuilder::new(&tls_ocsf_ctx()) .severity(SeverityId::Informational) @@ -236,16 +124,7 @@ impl TlsAcceptor { /// alive for the duration of the TLS handshake. #[must_use] pub fn acceptor(&self) -> tokio_rustls::TlsAcceptor { - tokio_rustls::TlsAcceptor::from(self.state.load().config.clone()) - } - - /// Snapshot the current configuration for one connection, so its - /// handshake and the classification of its peer agree. - #[must_use] - pub fn handshake(&self) -> TlsHandshake { - TlsHandshake { - state: self.state.load_full(), - } + tokio_rustls::TlsAcceptor::from(self.config.load_full()) } /// Spawn a background worker that watches the parent directories of the @@ -295,12 +174,6 @@ impl TlsAcceptor { dirs.push(ext_dir.to_path_buf()); } } - if let Some(ref host_ca) = self.supervisor_host_ca_path { - let host_dir = host_ca.parent().unwrap_or_else(|| Path::new(".")); - if !dirs.contains(&host_dir.to_path_buf()) { - dirs.push(host_dir.to_path_buf()); - } - } let debounce = Duration::from_secs(1); @@ -488,80 +361,7 @@ fn build_cert_resolver( }))) } -fn root_store(roots: &[CertificateDer<'static>]) -> Result { - let mut store = rustls::RootCertStore::empty(); - for cert in roots { - store - .add(cert.clone()) - .map_err(|e| Error::tls(format!("failed to add CA certificate: {e}")))?; - } - Ok(store) -} - -/// Strict verifier for one realm: chains must be issued by one of `roots`. -fn build_realm_verifier(roots: &[CertificateDer<'static>]) -> Result> { - WebPkiClientVerifier::builder_with_provider( - Arc::new(root_store(roots)?), - openshell_crypto::tls::configuration_provider(), - ) - .build() - .map_err(|e| Error::tls(format!("failed to build client verifier: {e}"))) -} - -/// Build the handshake config and the per-realm verifiers from one read of -/// each CA file, so the handshake and the classification agree on the roots. -#[allow(clippy::too_many_arguments)] -fn build_acceptor_state( - cert_path: &Path, - key_path: &Path, - client_ca_path: Option<&Path>, - require_client_auth: bool, - external_cert_path: Option<&Path>, - external_key_path: Option<&Path>, - external_server_names: &[String], - supervisor_host_ca_path: Option<&Path>, -) -> Result { - let client_roots = client_ca_path.map(load_certs).transpose()?; - let host_roots = supervisor_host_ca_path.map(load_certs).transpose()?; - let config = build_server_config_with_roots( - cert_path, - key_path, - client_roots.as_deref(), - require_client_auth, - external_cert_path, - external_key_path, - external_server_names, - host_roots.as_deref(), - )?; - Ok(AcceptorState { - config, - client_verifier: client_roots - .as_deref() - .map(build_realm_verifier) - .transpose()?, - supervisor_host_verifier: host_roots - .as_deref() - .map(build_realm_verifier) - .transpose()?, - }) -} - -/// Subject common name of a DER certificate. -fn certificate_common_name(certificate: &CertificateDer<'_>) -> Option { - use x509_parser::prelude::*; - - let (_, parsed) = X509Certificate::from_der(certificate.as_ref()).ok()?; - parsed - .subject() - .iter_common_name() - .next() - .and_then(|attribute| attribute.as_str().ok()) - .map(str::to_string) -} - /// Build a `ServerConfig` from certificate, key, and optional client CA files. -#[cfg(test)] -#[allow(clippy::too_many_arguments)] fn build_server_config( cert_path: &Path, key_path: &Path, @@ -570,35 +370,6 @@ fn build_server_config( external_cert_path: Option<&Path>, external_key_path: Option<&Path>, external_server_names: &[String], - supervisor_host_ca_path: Option<&Path>, -) -> Result> { - build_server_config_with_roots( - cert_path, - key_path, - client_ca_path.map(load_certs).transpose()?.as_deref(), - require_client_auth, - external_cert_path, - external_key_path, - external_server_names, - supervisor_host_ca_path - .map(load_certs) - .transpose()? - .as_deref(), - ) -} - -/// Build a `ServerConfig` from certificate and key files and already loaded -/// client roots: the ordinary client CA and the supervisor host CA. -#[allow(clippy::too_many_arguments)] -fn build_server_config_with_roots( - cert_path: &Path, - key_path: &Path, - client_roots: Option<&[CertificateDer<'static>]>, - require_client_auth: bool, - external_cert_path: Option<&Path>, - external_key_path: Option<&Path>, - external_server_names: &[String], - supervisor_host_roots: Option<&[CertificateDer<'static>]>, ) -> Result> { let certs = load_certs(cert_path)?; let key = load_key(key_path)?; @@ -616,24 +387,17 @@ fn build_server_config_with_roots( external_server_names, )?; - let client_roots = client_roots - .into_iter() - .chain(supervisor_host_roots) - .flatten() - .cloned() - .collect::>(); - let mut config = if client_roots.is_empty() { - let builder = openshell_crypto::tls::server_builder().with_no_client_auth(); - if let Some(resolver) = resolver { - builder.with_cert_resolver(resolver) - } else { - builder - .with_single_cert(certs, key) - .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? + let mut config = if let Some(ca_path) = client_ca_path { + let ca_certs = load_certs(ca_path)?; + let mut root_store = rustls::RootCertStore::empty(); + for cert in ca_certs { + root_store + .add(cert) + .map_err(|e| Error::tls(format!("failed to add CA certificate: {e}")))?; } - } else { + let verifier_builder = WebPkiClientVerifier::builder_with_provider( - Arc::new(root_store(&client_roots)?), + Arc::new(root_store), openshell_crypto::tls::configuration_provider(), ); let verifier = if require_client_auth { @@ -652,6 +416,15 @@ fn build_server_config_with_roots( .with_single_cert(certs, key) .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? } + } else { + let builder = openshell_crypto::tls::server_builder().with_no_client_auth(); + if let Some(resolver) = resolver { + builder.with_cert_resolver(resolver) + } else { + builder + .with_single_cert(certs, key) + .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? + } }; config @@ -770,7 +543,6 @@ mod tests { None, None, &[], - None, ) .expect("failed to build server config"); @@ -1117,179 +889,6 @@ mod tests { let _ = tokio::time::timeout(Duration::from_secs(2), handle).await; } - /// Issue a leaf certificate for `common_name` from `ca`. - fn issue_leaf( - common_name: &str, - ca: &openshell_crypto::pki::Certificate, - ca_key: &KeyPair, - ) -> (Vec>, PrivateKeyDer<'static>) { - let mut params = CertificateParams::new(Vec::::new()).expect("leaf params"); - params - .distinguished_name - .push(rcgen::DnType::CommonName, common_name); - let key = openshell_crypto::pki::generate_keypair().expect("leaf key"); - let cert = openshell_crypto::pki::signed_by(params, &key, ca, ca_key).expect("sign leaf"); - let chain = rustls_pemfile::certs(&mut cert.pem().as_bytes()) - .collect::, _>>() - .expect("leaf chain"); - let key = rustls_pemfile::private_key(&mut key.serialize_pem().unwrap().as_bytes()) - .expect("leaf key pem") - .expect("leaf key present"); - (chain, key) - } - - /// Handshake with `acceptor`, presenting `client` when given, and - /// classify the server side of the connection. - async fn classify_with( - acceptor: &TlsAcceptor, - server_ca: &Path, - client: Option<(Vec>, PrivateKeyDer<'static>)>, - ) -> Result { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("address"); - let handshake = acceptor.handshake(); - let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.expect("accept"); - let tls = handshake.accept(stream).await?; - Ok::<_, std::io::Error>(handshake.classify(&tls)) - }); - let mut roots = rustls::RootCertStore::empty(); - for cert in load_certs(server_ca).expect("server CA") { - roots.add(cert).expect("root"); - } - let builder = openshell_crypto::tls::client_builder().with_root_certificates(roots); - let config = match client { - Some((chain, key)) => builder - .with_client_auth_cert(chain, key) - .expect("client identity"), - None => builder.with_no_client_auth(), - }; - let tcp = TcpStream::connect(address).await.expect("connect"); - let tls = tokio_rustls::TlsConnector::from(Arc::new(config)) - .connect( - rustls::pki_types::ServerName::try_from("localhost").unwrap(), - tcp, - ) - .await - .expect("client handshake"); - drop(tls); - server - .await - .expect("server task") - .expect("server handshake") - } - - /// The supervisor host CA and the ordinary client CA are separate realms: - /// a host certificate never becomes a client identity, a client - /// certificate never becomes a host, and overlapping roots are refused. - #[tokio::test] - async fn handshake_keeps_supervisor_host_and_client_realms_apart() { - let dir = tempfile::tempdir().expect("tempdir"); - let (client_ca, client_ca_key) = generate_test_certs_with_ca(dir.path()); - let mut host_ca_params = CertificateParams::new(Vec::::new()).unwrap(); - host_ca_params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); - host_ca_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; - host_ca_params - .distinguished_name - .push(rcgen::DnType::CommonName, "supervisor-host-ca"); - let host_ca_key = openshell_crypto::pki::generate_keypair().unwrap(); - let host_ca = openshell_crypto::pki::self_signed(host_ca_params, &host_ca_key).unwrap(); - write_test_file(dir.path(), "host-ca.pem", host_ca.pem().as_bytes()); - let server_ca = dir.path().join("ca.pem"); - let acceptor = || { - TlsAcceptor::from_files( - &dir.path().join("server-cert.pem"), - &dir.path().join("server-key.pem"), - Some(&server_ca), - false, - None, - None, - Vec::new(), - ) - .expect("acceptor") - }; - let two_realms = acceptor() - .with_supervisor_host_ca(&dir.path().join("host-ca.pem")) - .expect("host CA"); - - assert_eq!( - classify_with(&two_realms, &server_ca, None).await.unwrap(), - PeerRealm::Anonymous - ); - assert_eq!( - classify_with( - &two_realms, - &server_ca, - Some(issue_leaf("openshell-client", &client_ca, &client_ca_key)), - ) - .await - .unwrap(), - PeerRealm::Client - ); - match classify_with( - &two_realms, - &server_ca, - Some(issue_leaf( - "openshell-supervisor-host", - &host_ca, - &host_ca_key, - )), - ) - .await - .unwrap() - { - PeerRealm::SupervisorHost(identity) => { - assert_eq!(identity.subject, "openshell-supervisor-host"); - assert_eq!(identity.certificate_sha256.len(), 64); - } - other => panic!("host certificate classified as {other:?}"), - } - - // Without a host CA, a host certificate is simply an unknown client. - let client_only = acceptor(); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let handshake = client_only.handshake(); - let server = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - handshake.accept(stream).await.map(|_| ()) - }); - let mut roots = rustls::RootCertStore::empty(); - for cert in load_certs(&server_ca).unwrap() { - roots.add(cert).unwrap(); - } - let (chain, key) = issue_leaf("openshell-supervisor-host", &host_ca, &host_ca_key); - let config = openshell_crypto::tls::client_builder() - .with_root_certificates(roots) - .with_client_auth_cert(chain, key) - .unwrap(); - let tcp = TcpStream::connect(address).await.unwrap(); - let client = tokio_rustls::TlsConnector::from(Arc::new(config)) - .connect( - rustls::pki_types::ServerName::try_from("localhost").unwrap(), - tcp, - ) - .await; - assert!( - client.is_err() || server.await.unwrap().is_err(), - "a host certificate must not pass a handshake that trusts only the client CA" - ); - - // The same CA for both realms makes every certificate ambiguous. - let ambiguous = acceptor() - .with_supervisor_host_ca(&server_ca) - .expect("overlapping CA"); - assert!( - classify_with( - &ambiguous, - &server_ca, - Some(issue_leaf("openshell-client", &client_ca, &client_ca_key)), - ) - .await - .is_err() - ); - } - #[tokio::test] async fn test_reload_mtls_ca_rotation() { let dir = tempfile::tempdir().expect("failed to create tempdir"); diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index a39cd85afd..d1e240a782 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -123,22 +123,30 @@ backend or enable a Windows workload runtime. ## Shared supervisor host -`openshell-supervisor --mode=shared --gateway-endpoint --host-tls-dir --address ` +`openshell-supervisor --mode=shared --gateway-endpoint --token-file --address ` hosts the sandbox instances of many sandboxes in one process for the Kubernetes gateway supervisor topology. It is Unix-only. The host opens no control listener: it dials the gateway and receives its assignments from it. - **Gateway attachment.** The host dials `--gateway-endpoint` (which must be - `https`) with the client certificate in `--host-tls-dir` (`tls.crt`, - `tls.key`), verifying the gateway with `OPENSHELL_TLS_CA` and + `https`), verifying the gateway with `OPENSHELL_TLS_CA` and `OPENSHELL_GATEWAY_TLS_SERVER_NAME` like every sandbox instance does, and opens the private `openshell.supervisor.host.v1/SupervisorHost.Attach` - stream (`proto/supervisor_host.proto`). The certificate comes from a CA the - gateway trusts only for this stream; the host identity authorizes nothing - else, and the host never acts with any other identity and has no Kubernetes - identity. The stream starts with a hello carrying a per-process instance id - and a connection epoch; the gateway supersedes an older stream of the same - process and refuses a delayed older connection. + stream (`proto/supervisor_host.proto`) with the **supervisor token** in + `--token-file` as its bearer. The gateway mints that token with its + launch-signing key under a profile of its own (`typ` + `openshell-supervisor+jwt`, subject `openshell-supervisor:`, + no sandbox, generation, or epoch claims) and keeps the file fresh through + its deployment packaging (the Helm chart's `-supervisor-token` + Secret). The token authorizes nothing but this stream; sandbox, extension, + user, operator, and peer credentials are refused on it. The host re-reads + the file on every connection attempt and treats a missing, malformed, or + expired token as a reason to retry later, and the gateway ends a stream + when the token that opened it expires, so the host reconnects with the + renewed one. The host never acts with any other identity and has no + Kubernetes identity. The stream starts with a hello carrying a per-process + instance id and a connection epoch; the gateway supersedes an older stream + of the same process and refuses a delayed older connection. - **Assignments.** The gateway owns the desired set and sends it as complete `AssignmentSnapshot`s with session-scoped revisions. `shared_host::SharedHost::reconcile` validates the snapshot as a whole (at most 256 instances, no route twice), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 43cf982c95..de29e9509a 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -69,10 +69,11 @@ struct Args { #[arg(long, required_if_eq("mode", "shared"))] address: Option, - /// Directory with the shared supervisor's host client certificate - /// (`tls.crt`, `tls.key`) for its outbound gateway connection. + /// File holding the supervisor token the gateway issues for the shared + /// supervisor's assignment stream. Re-read on every connection attempt, + /// so a renewed token takes effect without a restart. #[arg(long, required_if_eq("mode", "shared"))] - host_tls_dir: Option, + token_file: Option, /// Listener for routed runtime-initiated Sandbox Protocol connections. #[arg(long, required_if_eq("mode", "shared"))] @@ -407,7 +408,7 @@ fn main() -> Result<()> { } if args.gateway_endpoint.is_some() || args.address.is_some() - || args.host_tls_dir.is_some() + || args.token_file.is_some() || args.runtime_listen.is_some() || args.session_root.is_some() { diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 354bec7883..2149355c6a 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -24,9 +24,9 @@ fn run_platform(args: crate::Args) -> Result<()> { let gateway_endpoint = args .gateway_endpoint .ok_or_else(|| miette::miette!("--gateway-endpoint is required for --mode=shared"))?; - let host_tls_dir = args - .host_tls_dir - .ok_or_else(|| miette::miette!("--host-tls-dir is required for --mode=shared"))?; + let token_file = args + .token_file + .ok_or_else(|| miette::miette!("--token-file is required for --mode=shared"))?; let runtime_listen = args .runtime_listen .ok_or_else(|| miette::miette!("--runtime-listen is required for --mode=shared"))?; @@ -34,18 +34,20 @@ fn run_platform(args: crate::Args) -> Result<()> { let instance_root = args .session_root .ok_or_else(|| miette::miette!("--session-root is required for --mode=shared"))?; - // The host verifies the gateway the way every sandbox instance does. - let gateway_ca = std::env::var_os(openshell_core::sandbox_env::TLS_CA) - .map(std::path::PathBuf::from) - .ok_or_else(|| { - miette::miette!( - "{} is required for --mode=shared", - openshell_core::sandbox_env::TLS_CA - ) - })?; - let gateway_server_name = std::env::var(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME) - .ok() - .filter(|name| !name.is_empty()); + // The supervisor token travels only over TLS; the host verifies the + // gateway the way every sandbox instance does, with `OPENSHELL_TLS_CA` + // and `OPENSHELL_GATEWAY_TLS_SERVER_NAME`. + if !gateway_endpoint.starts_with("https://") { + return Err(miette::miette!( + "--gateway-endpoint must be https for --mode=shared" + )); + } + if std::env::var_os(openshell_core::sandbox_env::TLS_CA).is_none() { + return Err(miette::miette!( + "{} is required for --mode=shared", + openshell_core::sandbox_env::TLS_CA + )); + } let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() @@ -85,9 +87,7 @@ fn run_platform(args: crate::Args) -> Result<()> { .into_diagnostic()?; let link = openshell_supervisor::shared_host::GatewayLink { endpoint: gateway_endpoint.clone(), - host_tls_dir, - gateway_ca, - gateway_server_name, + token_file, }; let host = openshell_supervisor::shared_host::SharedHost::start( openshell_supervisor::shared_host::SharedHostConfig { diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index caf1af599d..2959f2e337 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -5,12 +5,13 @@ //! //! The gateway owns which sandbox instances, one running generation of a //! stable Sandbox resource each, this process hosts. The host dials the -//! gateway with its gateway-scoped host certificate, receives complete -//! assignment snapshots over the private `openshell.supervisor.host.v1` -//! stream, reconciles the [`SandboxInstanceHost`] registry to each one, and -//! reports what it observes. An assignment carries exactly the gateway-issued -//! credentials of its generation; the host never acts with any other identity -//! toward sandbox APIs. Losing the gateway stream never stops an instance; +//! gateway presenting the supervisor token the gateway minted for it, +//! receives complete assignment snapshots over the private +//! `openshell.supervisor.host.v1` stream, reconciles the +//! [`SandboxInstanceHost`] registry to each one, and reports what it +//! observes. An assignment carries exactly the gateway-issued credentials of +//! its generation; the host never acts with any other identity toward +//! sandbox APIs. Losing the gateway stream never stops an instance; //! only a later snapshot that omits it does. Snapshots are reconciled one at //! a time, and a connection finishes the reconciliation it started before the //! next connection begins, so a superseded stream can never apply after its @@ -21,7 +22,7 @@ use std::future::Future; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use miette::Result; use openshell_core::proto::supervisor_host::v1::supervisor_host_client::SupervisorHostClient; @@ -29,6 +30,7 @@ use openshell_core::proto::supervisor_host::v1::{ AssignmentSnapshot, HostHeartbeat, HostHello, HostMessage, HostStatus, host_gateway_message, host_message, }; +use openshell_extension_core::BearerTokenSlot; use openshell_sandbox_backend::runtime_routed::RuntimeRouter; use openshell_sandbox_backend::sandbox_instance::{ SandboxInstancePhaseReport, SandboxInstanceRegistration, SandboxInstanceStatusReport, @@ -431,55 +433,43 @@ fn write_private_file(_path: &Path, _contents: &[u8]) -> Result<()> { /// How the host reaches the gateway's supervisor host service. pub struct GatewayLink { - /// Gateway gRPC endpoint; must use `https`, since the host authenticates - /// with its client certificate. + /// Gateway gRPC endpoint; must be `https`, since the supervisor token is + /// a bearer. Server-authenticated TLS follows the sandbox instance + /// conventions (`OPENSHELL_TLS_CA`, `OPENSHELL_GATEWAY_TLS_SERVER_NAME`). pub endpoint: String, - /// Directory holding the host client certificate `tls.crt` and `tls.key`. - pub host_tls_dir: PathBuf, - /// PEM CA bundle that issued the gateway's server certificate. - pub gateway_ca: PathBuf, - /// Server name expected in the gateway certificate, when it differs from - /// the endpoint host. - pub gateway_server_name: Option, + /// File holding the gateway-minted supervisor token. It is read on every + /// connection attempt, so the gateway's renewals take effect at the next + /// reconnect without a restart. + pub token_file: PathBuf, } impl GatewayLink { - fn channel_endpoint(&self) -> Result { + /// Read the current supervisor token. A missing, empty, malformed, or + /// already expired token is an error; the gateway keeps the file fresh, + /// so the caller retries later. + fn read_token(&self) -> Result { use miette::{IntoDiagnostic as _, WrapErr as _}; - if !self.endpoint.starts_with("https://") { + let contents = std::fs::read_to_string(&self.token_file) + .into_diagnostic() + .wrap_err_with(|| format!("read supervisor token {}", self.token_file.display()))?; + let token = openshell_core::jwt::SecretJwt::parse(contents.trim()) + .map_err(|_| miette::miette!("the supervisor token file holds no token yet"))?; + let expires_at = openshell_core::jwt::parse_exp_secs(token.expose_secret()) + .ok_or_else(|| miette::miette!("the supervisor token is not an expiring JWT"))?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + if expires_at <= now { return Err(miette::miette!( - "the shared supervisor requires an https gateway endpoint for its host identity" - )); - } - let read = |path: &Path| { - std::fs::read(path) - .into_diagnostic() - .wrap_err_with(|| format!("read {}", path.display())) - }; - let certificate = read(&self.host_tls_dir.join("tls.crt"))?; - let private_key = read(&self.host_tls_dir.join("tls.key"))?; - let gateway_ca = read(&self.gateway_ca)?; - let mut tls = tonic::transport::ClientTlsConfig::new() - .ca_certificate(tonic::transport::Certificate::from_pem(gateway_ca)) - .identity(tonic::transport::Identity::from_pem( - certificate, - private_key, + "the supervisor token expired {} s ago; waiting for the gateway to renew it", + now.saturating_sub(expires_at) )); - if let Some(server_name) = self.gateway_server_name.as_deref() { - tls = tls.domain_name(server_name); } - tonic::transport::Endpoint::from_shared(self.endpoint.clone()) - .into_diagnostic() - .wrap_err("invalid gateway endpoint")? - .connect_timeout(CONNECT_TIMEOUT) - .http2_keep_alive_interval(Duration::from_secs(10)) - .keep_alive_while_idle(true) - .keep_alive_timeout(Duration::from_secs(20)) - .tcp_nodelay(true) - .tls_config(tls) - .into_diagnostic() - .wrap_err("configure gateway TLS") + BearerTokenSlot::new(token.expose_secret(), expires_at.saturating_mul(1000)) + .map_err(|error| miette::miette!("{error}")) } } @@ -549,21 +539,30 @@ pub async fn attach_to_gateway( /// not end until a running reconciliation has finished. async fn run_stream(host: &Arc, link: &GatewayLink, hello: HostHello) -> StreamEnd { let lost = |accepted| StreamEnd::Lost { accepted }; - let endpoint = match link.channel_endpoint() { - Ok(endpoint) => endpoint, + if !link.endpoint.starts_with("https://") { + tracing::error!("the supervisor token requires an https gateway endpoint"); + return lost(false); + } + let token = match link.read_token() { + Ok(token) => token, Err(error) => { - tracing::error!(%error, "invalid gateway link configuration"); + tracing::warn!(%error, "no usable supervisor token; will retry"); return lost(false); } }; - let channel = match endpoint.connect().await { - Ok(channel) => channel, - Err(error) => { + let connect = openshell_core::grpc_client::connect_gateway_channel(&link.endpoint); + let channel = match tokio::time::timeout(CONNECT_TIMEOUT, connect).await { + Ok(Ok(channel)) => channel, + Ok(Err(error)) => { tracing::warn!(%error, "could not connect to the gateway"); return lost(false); } + Err(_) => { + tracing::warn!("connecting to the gateway timed out"); + return lost(false); + } }; - let mut client = SupervisorHostClient::new(channel) + let mut client = SupervisorHostClient::with_interceptor(channel, token.interceptor()) .max_decoding_message_size(MAX_GATEWAY_MESSAGE_BYTES) .max_encoding_message_size(MAX_HOST_MESSAGE_BYTES); let (tx, rx) = tokio::sync::mpsc::channel::(OUTBOUND_QUEUE); diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 8b375a08b5..12120a89ce 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -196,11 +196,6 @@ spec: mountPath: {{ dir $spiffeSocketPath | quote }} readOnly: true {{- end }} - {{- if include "openshell.supervisorEnabled" . }} - - name: supervisor-host-ca - mountPath: /etc/openshell-tls/supervisor-host - readOnly: true - {{- end }} {{- with .Values.server.extraVolumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} @@ -295,16 +290,6 @@ spec: - key: ca.crt path: ca.crt {{- end }} - {{- if include "openshell.supervisorEnabled" . }} - # CA of the shared supervisor's host certificate. Only the CA is projected; - # the host's private key stays with the supervisor. - - name: supervisor-host-ca - secret: - secretName: {{ include "openshell.supervisorName" . }}-host - items: - - key: ca.crt - path: ca.crt - {{- end }} {{- if .Values.upstreamProxy.caBundle.configMapName }} - name: upstream-proxy-ca configMap: diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 070e64dbd5..44a1a8a8bb 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -363,7 +363,7 @@ raw gateway configuration so openshell.effectiveKubernetesConfig can use it. {{- end }} {{/* -Name of the shared supervisor Deployment, Service, and host identity Secret: +Name of the shared supervisor Deployment and Service (and, with `-token`, its token Secret): the fullname with a fixed suffix, within the 63-character Service name limit. A fullname without room for the suffix keeps a prefix and a hash of the whole fullname, so the name never collapses into another release resource's name. @@ -617,9 +617,9 @@ like user-supplied schema-v2 fields during compatibility resolution. */}} {{- if not (hasKey $kubernetes "supervisor_image") -}}{{- $_ := set $kubernetes "supervisor_image" (include "openshell.supervisorImage" .) -}}{{- end -}} {{- if not (hasKey $runtimeConfig "boundary_port") -}}{{- $_ := set $runtimeConfig "boundary_port" 5500 -}}{{- end -}} {{- $_ := set $kubernetes "sandbox_runtime" $runtimeConfig -}} -{{/* The chart owns the shared supervisor and the CA of its host identity. */}} +{{/* The chart owns the shared supervisor and the Secret that delivers its token. */}} {{- if include "openshell.supervisorEnabled" . -}} -{{- $_ := set $kubernetes "shared_supervisor" (dict "service_name" (include "openshell.supervisorName" .) "service_namespace" .Release.Namespace "runtime_port" 5500 "host_ca_file" "/etc/openshell-tls/supervisor-host/ca.crt") -}} +{{- $_ := set $kubernetes "shared_supervisor" (dict "service_name" (include "openshell.supervisorName" .) "service_namespace" .Release.Namespace "runtime_port" 5500 "token_secret_name" (printf "%s-token" (include "openshell.supervisorName" .))) -}} {{- end -}} {{- toYaml $kubernetes -}} {{- end }} @@ -645,7 +645,7 @@ Validate chart values that Helm would otherwise accept silently. {{- fail "supervisor_topology=gateway does not support upstream proxy settings or provider SPIFFE identity" -}} {{- end -}} {{- if .Values.server.disableTls -}} -{{- fail "supervisor_topology=gateway requires gateway TLS; the shared supervisor authenticates with a client certificate" -}} +{{- fail "supervisor_topology=gateway requires gateway TLS; the supervisor token travels only over TLS" -}} {{- end -}} {{- if and (eq (get $kubernetes "workspace_mode" | default "shared") "operator") (not (regexMatch "^[A-Za-z0-9][-A-Za-z0-9_./]*==?[-A-Za-z0-9_.]*$" (get $kubernetes "operator_namespace_label" | default ""))) -}} {{- fail "supervisor_topology=gateway in operator workspace mode requires operator_namespace_label of the form key=value" -}} diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index 0ebc2bf7e7..9a916e5f9a 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -{{- $supervisor := include "openshell.supervisorEnabled" . }} -{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled $supervisor }} +{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} {{- $hookName := printf "%s-certgen" (include "openshell.fullname" .) }} {{- $ns := .Release.Namespace }} apiVersion: v1 @@ -122,27 +121,6 @@ spec: - --backend-ca-configmap-name={{ include "openshell.backendCaConfigMapName" . }} {{- end }} {{- end }} - {{- if $supervisor }} - {{- $supervisorName := include "openshell.supervisorName" . }} - # Private PKI for the shared supervisor's host identity: the supervisor - # presents the client certificate to the gateway, which trusts only - # the CA. - - name: supervisor-certgen - image: {{ include "openshell.image" . | quote }} - imagePullPolicy: {{ .Values.gateway.image.pullPolicy | default .Values.global.image.pullPolicy }} - securityContext: - {{- toYaml .Values.pkiInitJob.securityContext | nindent 12 }} - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: ["/usr/local/bin/openshell-gateway"] - args: - - generate-certs - - --client-only - - --client-secret-name={{ $supervisorName }}-host - {{- end }} {{- if and .Values.certManager.enabled .Values.grpcRoute.backendTLSPolicy.enabled }} --- # Post-install Job that polls for the cert-manager-issued server Secret and diff --git a/deploy/helm/openshell/templates/peer-role.yaml b/deploy/helm/openshell/templates/peer-role.yaml index 38a640cd2a..d7fa148658 100644 --- a/deploy/helm/openshell/templates/peer-role.yaml +++ b/deploy/helm/openshell/templates/peer-role.yaml @@ -29,6 +29,17 @@ rules: - {{ include "openshell.supervisorName" . }} verbs: - get + # The gateway delivers the supervisor token it mints by replacing the token + # in the chart-created Secret; it creates no Secrets here. + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - {{ include "openshell.supervisorName" . }}-token + verbs: + - get + - patch {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index d8cbb028fa..d6027d2449 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -13,10 +13,11 @@ {{- $endpointUrl := urlParse $endpoint }} {{- $gatewayTls := and (not .Values.server.disableTls) .Values.server.tls.clientTlsSecretName }} # One supervisor process hosting a sandbox instance per sandbox generation. -# The process dials the gateway with its host certificate and receives its -# assignments from it; nothing but sandbox runtimes and kubelet probes connect -# to it. The Pod holds no Kubernetes identity of its own: each instance acts -# only with the credentials the gateway issued for its generation. +# The process dials the gateway over TLS with the supervisor token the gateway +# mints for it and receives its assignments from it; nothing but sandbox +# runtimes and kubelet probes connect to it. The Pod holds no Kubernetes +# identity of its own: each instance acts only with the credentials the +# gateway issued for its generation. apiVersion: apps/v1 kind: Deployment metadata: @@ -61,8 +62,8 @@ spec: - {{ $endpoint | quote }} - --address - 0.0.0.0:5501 - - --host-tls-dir - - /etc/openshell/supervisor-host + - --token-file + - /etc/openshell/supervisor-token/token - --runtime-listen - 0.0.0.0:{{ get $shared "runtime_port" }} - --session-root @@ -112,8 +113,8 @@ spec: mountPath: /run/openshell - name: logs mountPath: /var/log - - name: supervisor-host - mountPath: /etc/openshell/supervisor-host + - name: supervisor-token + mountPath: /etc/openshell/supervisor-token readOnly: true {{- if $gatewayTls }} - name: gateway-ca @@ -125,9 +126,9 @@ spec: emptyDir: {} - name: logs emptyDir: {} - - name: supervisor-host + - name: supervisor-token secret: - secretName: {{ $name }}-host + secretName: {{ $name }}-token {{- if $gatewayTls }} - name: gateway-ca secret: @@ -190,4 +191,18 @@ spec: egress: - {} {{- end }} +--- +# Delivery of the gateway-minted supervisor token. The chart creates the +# Secret empty; the gateway replaces its `token` on every renewal, and the +# supervisor re-reads the mounted file on each connection attempt. Helm +# upgrades leave the live token in place. +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }}-token + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: supervisor +type: Opaque {{- end }} diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index db8dd4b1b7..0686a800ec 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -22,7 +22,7 @@ tests: supervisor_image: example.com/supervisor:shared asserts: - hasDocuments: - count: 3 + count: 4 - equal: path: kind value: Deployment @@ -51,8 +51,8 @@ tests: - https://openshell.gateway.svc.cluster.local:8080 - --address - 0.0.0.0:5501 - - --host-tls-dir - - /etc/openshell/supervisor-host + - --token-file + - /etc/openshell/supervisor-token/token - --runtime-listen - 0.0.0.0:5500 - --session-root @@ -107,10 +107,35 @@ tests: - contains: path: spec.template.spec.volumes content: - name: supervisor-host + name: supervisor-token secret: - secretName: openshell-supervisor-host + secretName: openshell-supervisor-token documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: supervisor-token + mountPath: /etc/openshell/supervisor-token + readOnly: true + documentIndex: 0 + - equal: + path: kind + value: Secret + documentIndex: 3 + - equal: + path: metadata.name + value: openshell-supervisor-token + documentIndex: 3 + - equal: + path: type + value: Opaque + documentIndex: 3 + - notExists: + path: data + documentIndex: 3 + - notExists: + path: stringData + documentIndex: 3 - contains: path: spec.template.spec.volumes content: @@ -212,7 +237,7 @@ tests: path: metadata.name value: openshell-gateway-release-with-a-very-very-long-name-0057 documentIndex: 1 - - it: requires gateway TLS for the host identity + - it: requires gateway TLS for the supervisor token template: templates/supervisor.yaml set: server: @@ -222,7 +247,7 @@ tests: supervisor_topology: gateway asserts: - failedTemplate: - errorMessage: supervisor_topology=gateway requires gateway TLS; the shared supervisor authenticates with a client certificate + errorMessage: supervisor_topology=gateway requires gateway TLS; the supervisor token travels only over TLS - it: renders the chart-owned shared supervisor connection into gateway TOML template: templates/gateway-config.yaml set: @@ -235,7 +260,7 @@ tests: pattern: 'supervisor_topology = "gateway"' - matchRegex: path: data["gateway.toml"] - pattern: 'shared_supervisor = \{ host_ca_file = "/etc/openshell-tls/supervisor-host/ca.crt", runtime_port = 5500, service_name = "openshell-supervisor", service_namespace = "gateway" \}' + pattern: 'shared_supervisor = \{ runtime_port = 5500, service_name = "openshell-supervisor", service_namespace = "gateway", token_secret_name = "openshell-supervisor-token" \}' - it: rejects a hand-written shared supervisor connection template: templates/supervisor.yaml set: diff --git a/deploy/helm/openshell/tests/supervisor_wiring_test.yaml b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml index 0cf1937bd3..b37c16e295 100644 --- a/deploy/helm/openshell/tests/supervisor_wiring_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml @@ -10,21 +10,22 @@ release: name: openshell namespace: gateway tests: - - it: issues the supervisor host identity with the hook + - it: issues no extra PKI for the shared supervisor template: templates/certgen.yaml set: gatewayConfig: openshell.drivers.kubernetes: supervisor_topology: gateway asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + documentIndex: 3 - equal: - path: spec.template.spec.containers[1].args - value: - - generate-certs - - --client-only - - --client-secret-name=openshell-supervisor-host + path: spec.template.spec.containers[0].name + value: certgen documentIndex: 3 - - it: issues only the supervisor host identity when the gateway PKI is external + - it: needs no hook for the shared supervisor when the gateway PKI is external template: templates/certgen.yaml set: pkiInitJob: @@ -33,14 +34,8 @@ tests: openshell.drivers.kubernetes: supervisor_topology: gateway asserts: - - lengthEqual: - path: spec.template.spec.containers - count: 1 - documentIndex: 3 - - equal: - path: spec.template.spec.containers[0].name - value: supervisor-certgen - documentIndex: 3 + - hasDocuments: + count: 0 - it: lets the gateway read only the shared supervisor Service template: templates/peer-role.yaml set: @@ -60,25 +55,52 @@ tests: verbs: - get documentIndex: 0 - - it: projects only the supervisor host CA onto the gateway - template: templates/statefulset.yaml + - it: lets the gateway replace only the supervisor token in its Secret + template: templates/peer-role.yaml set: gatewayConfig: openshell.drivers.kubernetes: supervisor_topology: gateway asserts: - contains: + path: rules + content: + apiGroups: + - "" + resources: + - secrets + resourceNames: + - openshell-supervisor-token + verbs: + - get + - patch + documentIndex: 0 + - it: grants no supervisor Secret access without the topology + template: templates/peer-role.yaml + asserts: + - notContains: + path: rules + content: + apiGroups: + - "" + resources: + - secrets + resourceNames: + - openshell-supervisor-token + verbs: + - get + - patch + documentIndex: 0 + - it: keeps the supervisor token Secret off the gateway + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - notContains: path: spec.template.spec.volumes content: - name: supervisor-host-ca + name: supervisor-token secret: - secretName: openshell-supervisor-host - items: - - key: ca.crt - path: ca.crt - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: supervisor-host-ca - mountPath: /etc/openshell-tls/supervisor-host - readOnly: true + secretName: openshell-supervisor-token diff --git a/proto/README.md b/proto/README.md index 663446e147..c66ec95990 100644 --- a/proto/README.md +++ b/proto/README.md @@ -92,8 +92,9 @@ identifies the service being exposed. a gateway and the supervisor process it shares among its sandboxes: the supervisor attaches, the gateway sends complete assignment snapshots, and the supervisor reports what it hosts. It is a version-locked deployment protocol -served on the gateway listener but authorized only for the supervisor host -certificate. It is not imported by `openshell.proto`, so SDK generation and +served on the gateway listener but authorized only for the gateway-minted +supervisor token (`auth_mode: "supervisor"`). It is not imported by +`openshell.proto`, so SDK generation and gRPC reflection exclude it; the server's schema inventory test classifies it explicitly. `SandboxInstanceAssignment.launch_material` carries the existing JSON sandbox instance registration of `openshell-sandbox-backend` and is diff --git a/proto/supervisor_host.proto b/proto/supervisor_host.proto index 4c31decaa4..2747dd5749 100644 --- a/proto/supervisor_host.proto +++ b/proto/supervisor_host.proto @@ -19,11 +19,11 @@ import "options.proto"; // registry to each snapshot and reports what it observes. Disconnecting // never stops hosted instances; only a later snapshot that omits them does. service SupervisorHost { - // Supervisor-initiated assignment stream. Only a gateway-scoped supervisor - // host certificate may call it. + // Supervisor-initiated assignment stream. Only the gateway-minted + // supervisor token may call it. rpc Attach(stream HostMessage) returns (stream HostGatewayMessage) { option (openshell.options.v1.authorization) = { - auth_mode: "supervisor_host" + auth_mode: "supervisor" }; } } diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index c038b93020..083266def1 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1280,7 +1280,7 @@ credential failures. | Gateway or `-certgen` hook pod rejected with `violates PodSecurity "restricted:latest"` | The rendered container lacks `runAsNonRoot` or `seccompProfile`: an older chart version does not set them, or an override in `securityContext` / `pkiInitJob.securityContext` unset them | `kubectl -n openshell get events --field-selector reason=FailedCreate`; upgrade the chart, then remove overrides that unset those fields | | OpenShift gateway pod fails to start with an SCC/`runAsUser` error (e.g. `unable to validate against any security context constraint`) | Chart's default `podSecurityContext`/`securityContext` hardcodes `runAsUser`/`fsGroup`, which the restricted-v2 SCC rejects; it must instead inject the namespace-assigned UID/GID range | `oc -n openshell describe pod `; deploy with `podSecurityContext: null` and clear `securityContext.runAsUser` (see `deploy/helm/openshell/ci/values-openshift-scc.yaml`) | | OpenShift `-supervisor` Deployment never creates pods after setting `supervisor_topology = "gateway"` (SCC `runAsUser`/`fsGroup` not in the namespace range) | `supervisor.podSecurityContext` defaults `runAsUser`, `runAsGroup`, and `fsGroup` to 65534, outside the namespace's assigned UID/GID ranges | `kubectl -n openshell get events --field-selector reason=FailedCreate`; compare with the `openshift.io/sa.scc.*` annotations from `oc describe ns openshell`; set `supervisor.podSecurityContext.runAsUser`, `.runAsGroup`, and `.fsGroup` to `null`, as `deploy/helm/openshell/ci/values-openshift-scc.yaml` does, and keep `runAsNonRoot` and `seccompProfile` | -| Shared sandboxes stay `Starting` or report `unavailable` readiness with `supervisor_topology = "gateway"`, gateway logs say no shared supervisor host is connected | The `-supervisor` Pod cannot reach or authenticate to the gateway: wrong `OPENSHELL_TLS_CA`, missing `-supervisor-host` Secret, or the gateway was deployed without TLS | `kubectl -n openshell logs deployment/-supervisor` for `gateway accepted the supervisor host` or a refusal; gateway logs for `supervisor host attached`; `kubectl -n openshell get secret -supervisor-host`; the gateway refuses to start the topology without TLS | +| Shared sandboxes stay `Starting` or report `unavailable` readiness with `supervisor_topology = "gateway"`, gateway logs say no shared supervisor host is connected | The `-supervisor` Pod cannot reach or authenticate to the gateway: wrong `OPENSHELL_TLS_CA`, the `-supervisor-token` Secret is missing or still empty (gateway logs `wrote the supervisor token Secret` once it has delivered a token; `no usable supervisor token` in the supervisor means it has not), or the gateway was deployed without TLS or without the sandbox-JWT signing key | `kubectl -n openshell logs deployment/-supervisor` for `gateway accepted the supervisor host` or a refusal; gateway logs for `supervisor host attached`, `wrote the supervisor token Secret`, or `failed to write the supervisor token Secret`; `kubectl -n openshell get secret -supervisor-token -o jsonpath='{.metadata.annotations}'` shows the token expiry; the gateway refuses to start the topology without TLS or `gateway_jwt` | | OpenShift sandbox pod fails to start (`unable to validate against any security context constraint`) | The `openshell-sandbox` service account lacks the privileged SCC it needs | `oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell`; remove with `remove-scc-from-user` when done | | OpenShift self-hosted Vault/OpenBao credential store pod never schedules (waits time out with `no matching resources found`) | The store's Helm chart pins `runAsUser`/`fsGroup`/seccomp, which restricted-v2 rejects, so the StatefulSet controller never creates the pod | Deploy the store's chart in its OpenShift mode (`--set global.openshift=true` for the OpenBao/Vault chart) so the namespace SCC assigns a compliant security context — no manual SCC grant needed | | Vault credential driver returns HTTP 403 / `Vault Kubernetes auth denied the configured role` on provider create | Vault's `auth/kubernetes` method or the gateway login role is not provisioned, or the role is not bound to the gateway service account and namespace | In Vault: `bao auth enable kubernetes` and `bao write auth/kubernetes/config kubernetes_host=... kubernetes_ca_cert=@...`; ensure the login role's `bound_service_account_names`/`bound_service_account_namespaces` match the gateway SA and namespace and its policy grants the credential paths | From 7754896639573937a9f6c0bd8448439443802954 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 21 Oct 2026 23:25:14 -0700 Subject: [PATCH 24/25] refactor(helm): remove redundant certgen condition Signed-off-by: Drew Newberry --- deploy/helm/openshell/templates/certgen.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/helm/openshell/templates/certgen.yaml b/deploy/helm/openshell/templates/certgen.yaml index 9a916e5f9a..aea3b8d2cc 100644 --- a/deploy/helm/openshell/templates/certgen.yaml +++ b/deploy/helm/openshell/templates/certgen.yaml @@ -84,7 +84,6 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} containers: - {{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }} - name: certgen image: {{ include "openshell.image" . | quote }} imagePullPolicy: {{ .Values.gateway.image.pullPolicy | default .Values.global.image.pullPolicy }} @@ -120,7 +119,6 @@ spec: {{- if and .Values.grpcRoute.backendTLSPolicy.enabled (not .Values.certManager.enabled) }} - --backend-ca-configmap-name={{ include "openshell.backendCaConfigMapName" . }} {{- end }} - {{- end }} {{- if and .Values.certManager.enabled .Values.grpcRoute.backendTLSPolicy.enabled }} --- # Post-install Job that polls for the cert-manager-issued server Secret and From a6cf581b856c644bcd536d5239e1530aa4b39a4d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 22 Oct 2026 11:11:55 -0700 Subject: [PATCH 25/25] refactor(supervisor): extract gateway-owned shared supervision into openshell-supervisor-control Move the shared supervisor control types out of openshell-core and openshell-sandbox-backend into a new openshell-supervisor-control crate: the SupervisorHostController (desired set, revision fencing, host sessions, observations), the SandboxInstanceRegistration launch material and SandboxInstanceStatusReport mapping, and a new SharedSupervision coordinator that owns the control contracts a compute driver previously implemented itself: validated persist-before-assign launches, recovery from durable records with the compare-and-replace baseline captured before the driver lists them, keep-previous-set on any failed or mismatched read, and one interpretation of host observations for readiness and bootstrap. Supervisor token delivery keeps its retry and newer-value policy in the crate; the write itself stays with the driver. The gateway now constructs the coordination: ComputeDriverBuildContext exposes shared_supervision() and the server keeps the controller for the Attach stream and token minting, replacing the driver-created controller that was installed into the gateway. The Kubernetes factory injects it through KubernetesComputeDriver::with_shared_supervision; new() stays the dedicated-topology constructor and refuses a configuration that selects the gateway topology without the gateway's coordination. The Kubernetes driver keeps its platform work: placement annotations, the shared supervisor Service and egress NetworkPolicy, launch Secret create and read (bytes only), its own fence and Pod evidence, and the token Secret patch. The supervisor process consumes the moved registration and status types unchanged. Wire protocol, storage JSON, annotations, and the supervisor token profile are unchanged; the private protobuf stays generated in openshell-core and is re-exported by the new crate. Signed-off-by: Drew Newberry --- AGENTS.md | 1 + Cargo.lock | 20 + crates/openshell-core/src/lib.rs | 2 - crates/openshell-driver-kubernetes/Cargo.toml | 1 + crates/openshell-driver-kubernetes/README.md | 78 ++-- .../openshell-driver-kubernetes/src/driver.rs | 377 ++++++++--------- .../src/shared_supervisor.rs | 194 +++------ crates/openshell-gateway/src/lib.rs | 27 +- crates/openshell-sandbox-backend/README.md | 10 +- crates/openshell-sandbox-backend/src/lib.rs | 1 - .../src/sandbox_instance.rs | 356 ---------------- crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/auth/principal.rs | 2 +- .../src/auth/supervisor_token.rs | 2 +- crates/openshell-server/src/compute/mod.rs | 13 +- crates/openshell-server/src/lib.rs | 37 +- crates/openshell-server/src/multiplex.rs | 2 +- .../openshell-supervisor-control/Cargo.toml | 27 ++ crates/openshell-supervisor-control/README.md | 65 +++ .../src/controller.rs} | 32 +- .../src/coordinator.rs | 390 ++++++++++++++++++ .../src/credential.rs | 66 +++ .../openshell-supervisor-control/src/lib.rs | 52 +++ .../src/registration.rs | 257 ++++++++++++ .../src/status.rs | 145 +++++++ crates/openshell-supervisor/Cargo.toml | 1 + crates/openshell-supervisor/README.md | 2 +- crates/openshell-supervisor/src/lib.rs | 2 +- .../src/sandbox_instance_host.rs | 4 +- crates/openshell-supervisor/src/shared.rs | 2 +- .../openshell-supervisor/src/shared_host.rs | 8 +- proto/README.md | 2 +- 32 files changed, 1378 insertions(+), 801 deletions(-) delete mode 100644 crates/openshell-sandbox-backend/src/sandbox_instance.rs create mode 100644 crates/openshell-supervisor-control/Cargo.toml create mode 100644 crates/openshell-supervisor-control/README.md rename crates/{openshell-core/src/supervisor_host.rs => openshell-supervisor-control/src/controller.rs} (96%) create mode 100644 crates/openshell-supervisor-control/src/coordinator.rs create mode 100644 crates/openshell-supervisor-control/src/credential.rs create mode 100644 crates/openshell-supervisor-control/src/lib.rs create mode 100644 crates/openshell-supervisor-control/src/registration.rs create mode 100644 crates/openshell-supervisor-control/src/status.rs diff --git a/AGENTS.md b/AGENTS.md index 434d235b4a..8bb2bc6bd0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ Do not rely on this file for a full inventory. The detailed public and contribut | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | | `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O | | `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking | +| `crates/openshell-supervisor-control/` | Shared supervisor control | Gateway-owned assignment controller, launch registrations, recovery coordination, and host observations for a supervisor shared by many sandboxes | | `crates/openshell-binary-identity/` | Binary identity | Shared trusted procfs executable identity resolution for isolation backends | | `crates/openshell-isolation-interface/` | Isolation backend interface | Shared data and validation for the Isolation Backend RPC contracts | | `crates/openshell-sandbox-backend/` | OpenShell sandbox backend | Concrete Isolation Backend RPC client and authenticated Sandbox Protocol shared with `openshell-sandbox` | diff --git a/Cargo.lock b/Cargo.lock index 2b85efdc05..5e5b4907da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4508,6 +4508,7 @@ dependencies = [ "openshell-otel-test-support", "openshell-policy", "openshell-sandbox-backend", + "openshell-supervisor-control", "opentelemetry", "opentelemetry_sdk", "prost", @@ -5033,6 +5034,7 @@ dependencies = [ "openshell-policy", "openshell-prover", "openshell-providers", + "openshell-supervisor-control", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", "opentelemetry", @@ -5112,6 +5114,7 @@ dependencies = [ "openshell-otel", "openshell-policy", "openshell-sandbox-backend", + "openshell-supervisor-control", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", "openshell-supervisor-network", @@ -5139,6 +5142,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "openshell-supervisor-control" +version = "0.0.0" +dependencies = [ + "openshell-core", + "openshell-otel", + "openshell-sandbox-backend", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tonic", + "tracing", +] + [[package]] name = "openshell-supervisor-middleware" version = "0.0.0" diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 2791c70292..9e70b309c2 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -61,8 +61,6 @@ pub mod secrets; pub mod settings; pub mod shell; pub mod spiffe; -#[cfg(feature = "jwt")] -pub mod supervisor_host; pub mod telemetry; pub mod time; pub mod token_grant_failure; diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 210abde9a5..f1d55cc0f0 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -19,6 +19,7 @@ openshell-crypto = { workspace = true, features = ["tls-kube"] } openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } +openshell-supervisor-control = { path = "../openshell-supervisor-control" } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 1684bb7565..7dc2478d34 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -192,48 +192,62 @@ keeps its established name) when the generation starts. Every later lifecycle de never reinterprets a running generation, and an unknown or incomplete record fails closed. -- **Assignments.** The gateway owns which instances the shared supervisor - hosts. The driver publishes that set into a - `openshell_core::supervisor_host::SupervisorHostController`, installed into - the gateway through the compute-driver build context, and the gateway serves - it to the supervisor over the private `openshell.supervisor.host.v1` stream. - The supervisor dials the gateway over TLS with a gateway-minted supervisor - token (see the `openshell-supervisor` README); that token authorizes only - the assignment stream. -- **Supervisor token delivery.** The gateway mints the supervisor token from - its `gateway_jwt` signing key and renews it at half its lifetime (the - `gateway_jwt` extension-token TTL, 15 minutes by default, bounded to one - hour). The driver's part is delivery only: it watches the controller for - each renewal and replaces the `token` key of the chart-created +- **Ownership.** The gateway owns which instances the shared supervisor + hosts. It constructs an `openshell_supervisor_control::SharedSupervision` + when the gateway binary builds this driver with + `KubernetesComputeDriver::with_shared_supervision`, serves its assignments + to the supervisor over the private `openshell.supervisor.host.v1` stream, + and mints the supervisor token the supervisor dials it with over TLS (see + the `openshell-supervisor` README); that token authorizes only the + assignment stream. The coordinator owns the control contracts (launch + material is persisted before an instance is assigned, recovery is + compare-and-replace against a baseline read before the list, an incomplete + recovery never replaces the published set, host observations are + interpreted once); this driver adapts them to Kubernetes objects. The + standalone driver binary cannot host the topology and refuses a + configuration that selects it. +- **Supervisor token delivery.** The gateway renews the token at half its + lifetime (the `gateway_jwt` extension-token TTL, 15 minutes by default, + bounded to one hour) and the coordinator retries a failed delivery until it + succeeds or a newer token supersedes it. The driver's part is the write: it + replaces the `token` key of the chart-created `shared_supervisor.token_secret_name` Secret in `service_namespace`, which - the supervisor Pod mounts. A missing Secret or a denied patch is logged and - retried; a token published while a retry waits supersedes the retry. + the supervisor Pod mounts, and names a missing Secret or a denied patch in + the retried error. - **Create and start.** The driver creates a per-sandbox egress allowance `os-shared-control-` from the workload Pod to the Pods the shared supervisor's Service selects, on the runtime port, binds its UID and - resourceVersion into the boundary evidence, persists the instance's launch - material in the generation's `os-supervisor--` Secret (key + resourceVersion into the boundary evidence, and builds the instance's + registration (route, sandbox name, main process spec and agent image from + the Sandbox resource, the provisioned runtime descriptor, the generation's + launch credentials, its proxy CA, and the current trace context). The + coordinator validates and encodes it, persists the launch material through + the driver into the generation's `os-supervisor--` Secret (key `instance.json`, owned by the workload Pod), and then assigns the instance - before releasing the workload Pod. Assignment does not wait for the - supervisor, so create still returns before the gateway persists the + before the driver releases the workload Pod. Assignment does not wait for + the supervisor, so create still returns before the gateway persists the generation. The runtime identity's last component is the instance route, which no Pod UID can equal, so `authenticate_sandbox` cannot vouch for a shared generation. -- **Readiness.** A shared generation is available while the supervisor - reports its instance as accepted by the gateway and its fence and egress - allowance exist. An instance the supervisor has reconciled past and does not - host, or reports as ended, is unavailable, and reconciliation suspends the - workload. A disconnected supervisor is unknown and a supervisor that has not - yet reconciled to the assignment is degraded; neither suspends anything. +- **Readiness.** A shared generation is available while the coordinator + reports its instance as supervised and accepted by the gateway and the + driver finds its fence and egress allowance. An instance the supervisor has + reconciled past and does not host, hosts for another sandbox, or reports as + ended is unavailable, and reconciliation suspends the workload. A + disconnected supervisor is unknown and a supervisor that has not yet + reconciled to the assignment is degraded; neither suspends anything. + Bootstrap completion waits on the same verdict within the bootstrap grace. - **Stop and delete.** The driver withdraws the instance before suspending or deleting the workload, and the generation cleanup deletes its launch - Secret. Every reconciliation also rebuilds the desired set from the Sandbox - resources that may run, reading launch material from the current assignment - or the launch Secret, so instances of deleted Sandboxes and of stops made - while the gateway was down are withdrawn, and a restarted gateway recovers - its assignments before it sends anything. The rebuild is compare-and-replace - against the revision read before the list, so it never undoes a lifecycle - edit that raced it. + Secret. Every reconciliation also recovers the desired set from the Sandbox + resources that may run: the driver begins the recovery before it lists + them, lists the shared generations that may run with their routes, and + reads each launch Secret's bytes on request; the coordinator reuses current + assignments, decodes and binds persisted material to the recorded route and + sandbox, keeps the previous set when any read fails, and refuses a list + that raced a lifecycle edit. Instances of deleted Sandboxes and of stops + made while the gateway was down are withdrawn, and a restarted gateway + recovers its assignments before it sends anything. - **Isolation.** The supervisor token is a profile of its own, separate from every instance's credentials; the gateway refuses it on every other RPC and refuses every other identity on the assignment stream, and ends a stream diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 62660f48e9..91a1ff416b 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -24,7 +24,7 @@ use crate::sandbox_runtime::{ }; use crate::shared_supervisor::{ ANNOTATION_SUPERVISOR_SESSION, ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY, - SharedSupervisor, SupervisorPlacement, reported, supervisor_placement, + SharedSupervisor, SupervisorPlacement, supervisor_placement, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::authentication::v1::{ @@ -69,12 +69,15 @@ use openshell_core::proto::compute::v1::{ WatchSandboxesSandboxEvent, watch_sandboxes_event, }; use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; -use openshell_core::supervisor_host::{AssignmentError, ObservedInstance}; use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use openshell_sandbox_backend::boundary_protocol::{ GatewayVerificationKey, SandboxTlsClientConfig, SandboxTlsServerConfig, generate_sandbox_tls_material, }; +use openshell_supervisor_control::{ + InstanceSupervision, LaunchError, RecoverableInstance, RecoveryBaseline, + RegistrationTraceContext, SandboxInstanceRegistration, SharedSupervision, +}; use serde::Deserialize; use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; @@ -947,9 +950,47 @@ impl KubernetesComputeDriver { } } + /// A driver whose new generations get a dedicated supervisor Pod each. + /// A configuration that places them on the gateway's shared supervisor + /// needs [`Self::with_shared_supervision`]; without the gateway's + /// coordination it is refused here, before anything runs. pub async fn new( config: KubernetesComputeConfig, shutdown_rx: tokio::sync::watch::Receiver, + ) -> Result { + if config.shared_supervisor.is_some() { + return Err(KubernetesDriverError::Precondition( + "the gateway supervisor topology requires the gateway's shared supervision, which only the driver linked into the gateway receives" + .to_string(), + )); + } + Self::build(config, shutdown_rx, None).await + } + + /// A driver whose new generations become sandbox instances on the shared + /// supervisor the gateway coordinates through `supervision`. The driver + /// adapts that coordination to Kubernetes: it records placements, + /// persists and reads back launch material, provisions egress to the + /// supervisor's Service, and delivers the supervisor token into its + /// Secret. + pub async fn with_shared_supervision( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + supervision: SharedSupervision, + ) -> Result { + if config.shared_supervisor.is_none() { + return Err(KubernetesDriverError::Precondition( + "shared supervision was provided but the configuration does not place generations on the shared supervisor" + .to_string(), + )); + } + Self::build(config, shutdown_rx, Some(supervision)).await + } + + async fn build( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + supervision: Option, ) -> Result { config .resource_admission @@ -1027,7 +1068,11 @@ impl KubernetesComputeDriver { None }; - let shared_supervisor = config.shared_supervisor.clone().map(SharedSupervisor::new); + let shared_supervisor = config + .shared_supervisor + .clone() + .zip(supervision) + .map(|(config, supervision)| SharedSupervisor::new(config, supervision)); if let Some(shared) = &shared_supervisor { shared.spawn_token_secret_writer(client.clone(), shutdown_rx.clone()); } @@ -3037,7 +3082,7 @@ impl KubernetesComputeDriver { return; }; if let Some(shared) = &self.shared_supervisor { - shared.withdraw(route); + shared.supervision().withdraw(route); } else { warn!( route, @@ -3046,17 +3091,6 @@ impl KubernetesComputeDriver { } } - /// The gateway-owned assignments of the shared supervisor host, when the - /// gateway topology is configured. The gateway serves them to the host. - #[must_use] - pub fn supervisor_host_controller( - &self, - ) -> Option { - self.shared_supervisor - .as_ref() - .map(|shared| shared.controller().clone()) - } - /// Read operator-provided bootstrap material once per generation, so the /// whole generation stages, mounts, and references the same bytes, and so /// unreadable material fails before any Service, Pod, or Secret is created. @@ -3564,29 +3598,41 @@ impl KubernetesComputeDriver { launch_authentication, proxy_ca, )?; - // The launch material is persisted with the generation first, - // so a restarted gateway rebuilds the host's assignments from - // the generation's own resources; assignment then publishes it - // to the host without waiting for the runtime. - let launch_material = serde_json::to_vec(®istration).map_err(|error| { - KubernetesDriverError::Message(format!( - "encode sandbox instance registration: {error}" - )) - })?; - secrets - .create( - &PostParams::default(), - &shared_instance_secret( - namespace, - names, - sandbox_id, - launch_material, - instance_owner, - ), - ) + // The coordinator persists the launch material with the + // generation before it assigns the instance, so a restarted + // gateway rebuilds the host's assignments from the + // generation's own resources; assignment then publishes it to + // the host without waiting for the runtime. + shared + .supervision() + .launch(®istration, |launch_material| async { + secrets + .create( + &PostParams::default(), + &shared_instance_secret( + namespace, + names, + sandbox_id, + launch_material, + instance_owner, + ), + ) + .await + .map(|_| ()) + .map_err(KubernetesDriverError::from_kube) + }) .await - .map_err(KubernetesDriverError::from_kube)?; - shared.assign(®istration)?; + .map_err(|error| match error { + LaunchError::Persist(error) => error, + // The host is at capacity; the create fails before + // its workload runs. + LaunchError::Refused(error) => { + KubernetesDriverError::Precondition(error.to_string()) + } + LaunchError::Invalid(error) => { + KubernetesDriverError::Message(error.to_string()) + } + })?; } } pods.patch( @@ -3606,7 +3652,11 @@ impl KubernetesComputeDriver { name: names.supervisor_pod.clone(), }, GenerationLaunchTarget::Shared { shared, route, .. } => { - BootstrapControl::SharedInstance { shared, route } + BootstrapControl::SharedInstance { + shared, + route, + sandbox_id: sandbox_id.to_string(), + } } }; spawn_sandbox_runtime_bootstrap_completion( @@ -4489,12 +4539,12 @@ impl KubernetesComputeDriver { return; } }; - // Read the desired revision before the list so a lifecycle edit that - // lands in between cannot be undone by the resync below. - let shared_baseline = self + // Begin the recovery before the list so a lifecycle edit that lands + // in between cannot be undone by the completion below. + let shared_recovery = self .shared_supervisor .as_ref() - .map(SharedSupervisor::revision); + .map(|shared| shared.supervision().begin_recovery()); let list = match tokio::time::timeout( KUBE_API_TIMEOUT, lookup_api @@ -4517,8 +4567,8 @@ impl KubernetesComputeDriver { // The shared supervisor's desired set is derived from the Sandbox // resources that may run, so instances of deleted Sandboxes and of // stops that happened while this gateway was down are withdrawn here. - if let Some(baseline) = shared_baseline { - self.resync_shared_instances(baseline, &list.items).await; + if let Some(baseline) = shared_recovery { + self.recover_shared_instances(baseline, &list.items).await; } for object in list.items { @@ -4772,74 +4822,60 @@ impl KubernetesComputeDriver { } } - /// Rebuild the shared supervisor's desired set from the generations that - /// may run: their launch material comes from the current assignment or - /// from the generation's launch Secret. A failed read keeps the previous - /// set, because an incomplete set would stop running instances, and a - /// list that raced a lifecycle edit is refused by the controller. - async fn resync_shared_instances(&self, baseline: u64, sandboxes: &[DynamicObject]) { + /// Complete a recovery of the shared supervisor's desired set from the + /// Sandbox resources that may run. The coordinator reuses current + /// assignments and otherwise reads each generation's launch Secret + /// through [`Self::read_shared_instance_material`]; it keeps the previous + /// set when any read fails and refuses a list that raced a lifecycle + /// edit. + async fn recover_shared_instances( + &self, + baseline: RecoveryBaseline, + sandboxes: &[DynamicObject], + ) { let Some(shared) = &self.shared_supervisor else { return; }; - let mut assignments = Vec::new(); - for (object, route) in runnable_shared_generations(sandboxes) { - if let Some(assignment) = shared.assignment(&route) { - assignments.push(assignment); - continue; - } - let (Ok(sandbox_id), Some(generation)) = ( - sandbox_id_from_object(object), - sandbox_runtime_generation(object), - ) else { - continue; - }; - let namespace = object - .metadata - .namespace - .as_deref() - .unwrap_or(&self.config.namespace); - match self - .read_shared_instance_registration(namespace, &sandbox_id, generation, &route) - .await - .and_then(|registration| { - registration - .map(|registration| registration.to_assignment()) - .transpose() - .map_err(|error| KubernetesDriverError::Message(error.to_string())) - }) { - Ok(Some(assignment)) => assignments.push(assignment), - Ok(None) => warn!( - sandbox_id, - route, "shared sandbox instance has no launch material; it stays unassigned" - ), - Err(error) => { - warn!(sandbox_id, route, %error, "could not read shared sandbox instance launch material; keeping the previous assignments"); - return; - } - } - } - match shared.resync(baseline, assignments) { - Ok(_) | Err(AssignmentError::Conflict { .. }) => {} - Err(error) => { - warn!(%error, "could not publish the shared sandbox instances; keeping the previous assignments"); - } - } + let candidates = runnable_shared_generations(sandboxes) + .into_iter() + .filter_map(|(object, route)| { + let sandbox_id = sandbox_id_from_object(object).ok()?; + let generation = sandbox_runtime_generation(object)?.to_string(); + let namespace = object + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + Some(RecoverableInstance { + route, + sandbox_id: sandbox_id.clone(), + record: SharedInstanceSecretRecord { + namespace, + sandbox_id, + generation, + }, + }) + }) + .collect(); + // Outcomes are logged by the coordinator; a conflict is derived again + // on the next reconciliation. + shared + .supervision() + .complete_recovery(baseline, candidates, |record| { + self.read_shared_instance_material(record.clone()) + }) + .await; } /// Launch material persisted with a shared generation, or `None` when its - /// Secret is gone or belongs to a dedicated generation. - async fn read_shared_instance_registration( + /// Secret is gone or belongs to a dedicated generation. The bytes are + /// decoded and bound to the generation by the coordinator. + async fn read_shared_instance_material( &self, - namespace: &str, - sandbox_id: &str, - generation: &str, - route: &str, - ) -> Result< - Option, - KubernetesDriverError, - > { - let names = SandboxRuntimeNames::for_generation(sandbox_id, generation); - let secrets = Api::::namespaced(self.client.clone(), namespace); + record: SharedInstanceSecretRecord, + ) -> Result>, KubernetesDriverError> { + let names = SandboxRuntimeNames::for_generation(&record.sandbox_id, &record.generation); + let secrets = Api::::namespaced(self.client.clone(), &record.namespace); let secret = tokio::time::timeout(KUBE_API_TIMEOUT, secrets.get_opt(&names.supervisor_secret)) .await @@ -4849,30 +4885,10 @@ impl KubernetesComputeDriver { ) })? .map_err(KubernetesDriverError::from_kube)?; - let Some(material) = secret - .as_ref() - .and_then(|secret| secret.data.as_ref()) - .and_then(|data| data.get(SHARED_INSTANCE_KEY)) - else { - return Ok(None); - }; - // Decoder errors can quote field values, including credentials. - let registration: openshell_sandbox_backend::sandbox_instance::SandboxInstanceRegistration = - serde_json::from_slice(&material.0).map_err(|_| { - KubernetesDriverError::Message( - "shared sandbox instance launch Secret is not a valid registration".to_string(), - ) - })?; - if registration.route != route || registration.sandbox_id != sandbox_id { - return Err(KubernetesDriverError::Precondition( - "shared sandbox instance launch Secret does not match the recorded generation" - .to_string(), - )); - } - registration - .validate() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - Ok(Some(registration)) + Ok(secret + .and_then(|secret| secret.data) + .and_then(|mut data| data.remove(SHARED_INSTANCE_KEY)) + .map(|material| material.0)) } async fn reconcile_sandbox_runtime_suspending( @@ -5418,8 +5434,18 @@ fn sandbox_runtime_bootstrap_completion_patch(resource_version: &str) -> serde_j }) } -/// Trusted launch material for one shared-supervisor sandbox instance, taken -/// from the generation's own Sandbox resource and gateway-issued credentials. +/// Where a shared generation's launch material is persisted: the +/// generation's supervisor Secret, read back by recovery. +#[derive(Clone)] +struct SharedInstanceSecretRecord { + namespace: String, + sandbox_id: String, + generation: String, +} + +/// Launch material of one shared-supervisor sandbox instance, taken from the +/// generation's own Sandbox resource and gateway-issued credentials. The +/// coordinator validates, encodes, persists, and assigns it. fn shared_instance_registration( object: &DynamicObject, route: &str, @@ -5427,21 +5453,8 @@ fn shared_instance_registration( runtime_descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor, launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, proxy_ca: crate::sandbox_runtime::ProxyCaMaterial, -) -> Result< - openshell_sandbox_backend::sandbox_instance::SandboxInstanceRegistration, - KubernetesDriverError, -> { - use openshell_sandbox_backend::sandbox_instance::{ - ProxyCaMaterial, RegistrationTraceContext, SandboxInstanceRegistration, SecretPem, - }; - let trace = openshell_otel::current_trace_context_environment(); - let trace_value = |name: &str| { - trace - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| value.clone()) - }; - let registration = SandboxInstanceRegistration { +) -> Result { + Ok(SandboxInstanceRegistration { route: route.to_string(), sandbox_id: sandbox_id.to_string(), sandbox_name: annotation_or_label(object, LABEL_SANDBOX_NAME).ok_or_else(|| { @@ -5469,21 +5482,12 @@ fn shared_instance_registration( .and_then(serde_json::Value::as_str) .unwrap_or_default() .to_string(), - proxy_ca: ProxyCaMaterial { + proxy_ca: openshell_supervisor_control::ProxyCaMaterial { certificate_pem: proxy_ca.certificate_pem, - private_key_pem: SecretPem::new(proxy_ca.private_key_pem), + private_key_pem: openshell_supervisor_control::SecretPem::new(proxy_ca.private_key_pem), }, - trace_context: trace_value(openshell_otel::TRACEPARENT_ENV).map(|traceparent| { - RegistrationTraceContext { - traceparent, - tracestate: trace_value(openshell_otel::TRACESTATE_ENV), - } - }), - }; - registration - .validate() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - Ok(registration) + trace_context: RegistrationTraceContext::current(), + }) } /// What bootstrap completion waits for before the runtime is declared ready. @@ -5494,6 +5498,7 @@ enum BootstrapControl { SharedInstance { shared: SharedSupervisor, route: String, + sandbox_id: String, }, } @@ -5516,24 +5521,18 @@ impl BootstrapControl { } } } - Self::SharedInstance { shared, route } => loop { - match shared.observed(&route) { - ObservedInstance::Reported(status) => { - let status = reported(&status); - if status.ready { - return true; - } - if status.phase.is_terminal() { - return false; - } - } - ObservedInstance::Absent => return false, - // A disconnected or still-reconciling host is not a - // verdict; keep waiting within the bootstrap grace. - ObservedInstance::Pending | ObservedInstance::NoHost => {} - } - tokio::time::sleep(Duration::from_secs(1)).await; - }, + // A disconnected or still-reconciling host is not a verdict; the + // coordinator keeps waiting within the bootstrap grace. + Self::SharedInstance { + shared, + route, + sandbox_id, + } => { + shared + .supervision() + .wait_for_ready(&route, &sandbox_id) + .await + } } } } @@ -6538,23 +6537,13 @@ async fn shared_instance_control_availability( } } }; - // Only a host that has reconciled to the assigning snapshot can render a - // verdict on the instance; a disconnected or still-reconciling host must - // not read as a terminal failure and suspend a running sandbox. - let instance = match shared.observed(route) { - ObservedInstance::Reported(status) => { - let status = reported(&status); - if status.sandbox_id != sandbox_id || status.phase.is_terminal() { - SandboxRuntimeControlAvailability::Unavailable - } else if status.ready { - SandboxRuntimeControlAvailability::Available - } else { - SandboxRuntimeControlAvailability::Degraded - } - } - ObservedInstance::Pending => SandboxRuntimeControlAvailability::Degraded, - ObservedInstance::Absent => SandboxRuntimeControlAvailability::Unavailable, - ObservedInstance::NoHost => { + // The coordinator renders the host's verdict; a disconnected or + // still-reconciling host is not one and must not suspend a running sandbox. + let instance = match shared.supervision().supervision(route, sandbox_id) { + InstanceSupervision::Available => SandboxRuntimeControlAvailability::Available, + InstanceSupervision::Degraded => SandboxRuntimeControlAvailability::Degraded, + InstanceSupervision::Unavailable => SandboxRuntimeControlAvailability::Unavailable, + InstanceSupervision::Unknown => { warn!( sandbox_id, "no shared supervisor host is connected; sandbox instance availability is unknown" diff --git a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs index b2f370b079..850e261e34 100644 --- a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs +++ b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs @@ -7,18 +7,14 @@ //! Every later lifecycle decision reads that record, so changing the gateway //! topology setting never reinterprets an existing generation. //! -//! The gateway owns which sandbox instances the shared supervisor hosts. The -//! driver publishes that set into a [`SupervisorHostController`], which the -//! gateway serves to the supervisor over the private supervisor host stream, -//! and reads the host's observations back from it. The durable source of the -//! set is the generation's Sandbox resource plus its launch Secret, so a -//! restarted gateway rebuilds it before publishing anything, and a full -//! resync is compare-and-replace against the revision it started from. -//! -//! The gateway also mints the supervisor token the supervisor presents on -//! that stream and publishes each renewal into the controller; the driver's -//! only part in authentication is delivery, writing the current token into -//! the Secret the supervisor Pod mounts. +//! The gateway owns which sandbox instances the shared supervisor hosts and +//! coordinates them through `openshell_supervisor_control::SharedSupervision`, +//! which it constructs and hands to this driver. The driver's part is +//! Kubernetes: the placement record on the Sandbox resource, the launch +//! Secret the coordinator persists launch material into and reads it back +//! from, the egress allowance from the workload to the supervisor's Service, +//! and delivery of the gateway-minted supervisor token into the Secret the +//! supervisor Pod mounts. use std::collections::BTreeMap; use std::time::Duration; @@ -26,16 +22,8 @@ use std::time::Duration; use k8s_openapi::api::core::v1::Secret; use kube::api::{Api, ObjectMeta, Patch, PatchParams}; use kube::core::DynamicObject; -use openshell_core::proto::supervisor_host::v1::{ - SandboxInstanceAssignment, SandboxInstanceStatus, -}; -use openshell_core::supervisor_host::{ - AssignmentError, ObservedInstance, SupervisorCredential, SupervisorHostController, -}; -use openshell_sandbox_backend::sandbox_instance::{ - SandboxInstanceRegistration, SandboxInstanceStatusReport, -}; -use tracing::{info, warn}; +use openshell_supervisor_control::{SharedSupervision, SupervisorCredential, deliver_credentials}; +use tracing::info; use crate::config::SharedSupervisorConfig; use crate::driver::KubernetesDriverError; @@ -91,163 +79,62 @@ pub fn supervisor_placement( } } -/// The gateway's desired set of shared sandbox instances and the host's -/// observations of them. The controller is the only owner of that state; -/// this type only translates between the driver's launch material and its -/// wire form. +/// The shared supervisor as this driver reaches it: the Helm-deployed +/// Service and token Secret it is configured with, and the gateway's +/// coordination of its sandbox instances. #[derive(Clone, Debug)] pub struct SharedSupervisor { config: SharedSupervisorConfig, - controller: SupervisorHostController, + supervision: SharedSupervision, } impl SharedSupervisor { - pub fn new(config: SharedSupervisorConfig) -> Self { + pub fn new(config: SharedSupervisorConfig, supervision: SharedSupervision) -> Self { Self { config, - controller: SupervisorHostController::new(), + supervision, } } /// Keep the supervisor token Secret current with every token the gateway /// publishes, until shutdown. The Secret itself is deployment packaging - /// (the chart creates it empty); the driver only replaces its token. A - /// failed write is retried until it succeeds or a newer token supersedes - /// it. + /// (the chart creates it empty); the driver only replaces its token. pub fn spawn_token_secret_writer( &self, client: kube::Client, - mut shutdown_rx: tokio::sync::watch::Receiver, + shutdown_rx: tokio::sync::watch::Receiver, ) { let secrets = Api::::namespaced(client, &self.config.service_namespace); let name = self.config.token_secret_name.clone(); - let mut credential = self.controller.credential(); - tokio::spawn(async move { - loop { - let current = credential.borrow_and_update().clone(); - if let Some(current) = current { - while let Err(error) = write_token_secret(&secrets, &name, ¤t).await { - match &error { - kube::Error::Api(response) if response.code == 404 => warn!( - secret = %name, - "the supervisor token Secret does not exist; the chart creates it, \ - restore it (helm upgrade) so the shared supervisor can authenticate" - ), - kube::Error::Api(response) if response.code == 403 => warn!( - secret = %name, - "the gateway may not patch the supervisor token Secret; check the \ - chart's gateway Role" - ), - _ => warn!( - secret = %name, - %error, - "failed to write the supervisor token Secret; retrying" - ), - } - tokio::select! { - () = tokio::time::sleep(TOKEN_WRITE_RETRY) => {} - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - } - } - if credential.has_changed().unwrap_or(true) { - break; - } - } - } - tokio::select! { - changed = credential.changed() => { - if changed.is_err() { - return; - } - } - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - } - } - } - }); + tokio::spawn(deliver_credentials( + self.supervision.credentials(), + shutdown_rx, + TOKEN_WRITE_RETRY, + move |credential| { + let secrets = secrets.clone(); + let name = name.clone(); + async move { write_token_secret(&secrets, &name, &credential).await } + }, + )); } pub fn config(&self) -> &SharedSupervisorConfig { &self.config } - /// The controller the gateway serves to the supervisor host. - pub fn controller(&self) -> &SupervisorHostController { - &self.controller - } - - /// Assign a generation's sandbox instance; an identical assignment - /// changes nothing. Refused as a precondition when the host is at - /// capacity, so the create fails before its workload runs. - pub fn assign( - &self, - registration: &SandboxInstanceRegistration, - ) -> Result<(), KubernetesDriverError> { - let assignment = registration - .to_assignment() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - self.controller - .assign(assignment) - .map_err(|error| KubernetesDriverError::Precondition(error.to_string())) - } - - /// Withdraw a generation's sandbox instance so the host stops it. - pub fn withdraw(&self, route: &str) { - self.controller.unassign(route); - } - - /// Current desired revision, read before deriving a full list for - /// [`Self::resync`]. - #[must_use] - pub fn revision(&self) -> u64 { - self.controller.revision() + /// The gateway's coordination of the host's sandbox instances. + pub fn supervision(&self) -> &SharedSupervision { + &self.supervision } - - /// The current assignment of a route, if any, so a resync keeps the - /// launch material it already holds instead of reading the Secret again. - #[must_use] - pub fn assignment(&self, route: &str) -> Option { - self.controller.assignment(route) - } - - /// Replace the desired set with the generations that may run, derived - /// from Sandbox resources and their launch Secrets while the revision was - /// `baseline`. The first call initializes the gateway's view; until then - /// nothing is sent to the host. A list that raced a lifecycle edit is - /// refused and derived again on the next reconciliation. - pub fn resync( - &self, - baseline: u64, - assignments: Vec, - ) -> Result { - self.controller.replace_assignments(baseline, assignments) - } - - /// The host's view of one route. - #[must_use] - pub fn observed(&self, route: &str) -> ObservedInstance { - self.controller.observed(route) - } -} - -/// The host's report of one instance, in the driver's terms. -#[must_use] -pub fn reported(status: &SandboxInstanceStatus) -> SandboxInstanceStatusReport { - SandboxInstanceStatusReport::from_proto(status) } -/// Replace the token in the chart-created Secret. +/// Replace the token in the chart-created Secret. The error names what an +/// operator must restore when the write cannot succeed on its own. async fn write_token_secret( secrets: &Api, name: &str, credential: &SupervisorCredential, -) -> Result<(), kube::Error> { +) -> Result<(), String> { let secret = Secret { metadata: ObjectMeta { annotations: Some(BTreeMap::from([( @@ -264,7 +151,18 @@ async fn write_token_secret( }; secrets .patch(name, &PatchParams::default(), &Patch::Merge(&secret)) - .await?; + .await + .map_err(|error| match &error { + kube::Error::Api(response) if response.code == 404 => format!( + "the supervisor token Secret {name} does not exist; the chart creates it, \ + restore it (helm upgrade) so the shared supervisor can authenticate" + ), + kube::Error::Api(response) if response.code == 403 => format!( + "the gateway may not patch the supervisor token Secret {name}; check the \ + chart's gateway Role" + ), + _ => format!("failed to write the supervisor token Secret {name}: {error}"), + })?; info!( secret = %name, expires_at = credential.expires_at, diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index b5f44bc5e8..3f060ae9f7 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -285,17 +285,24 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { .map(str::to_string); config.supervisor_ocsf_export = supervisor_ocsf_export(context.otlp_config()); config.supervisor_gateway_name = Some(context.gateway_name().to_string()); - let driver = openshell_driver_kubernetes::KubernetesComputeDriver::new( - config, - context.shutdown_receiver(), - ) - .await - .map_err(|error| openshell_core::Error::execution(error.to_string()))?; - // With the gateway supervisor topology, the driver owns the shared - // supervisor's assignments and the gateway serves them to the host. - if let Some(controller) = driver.supervisor_host_controller() { - context.install_supervisor_host(controller)?; + // With the gateway supervisor topology, the gateway owns the shared + // supervisor's assignments and token; the driver adapts them to + // Kubernetes objects. + let driver = if config.shared_supervisor.is_some() { + openshell_driver_kubernetes::KubernetesComputeDriver::with_shared_supervision( + config, + context.shutdown_receiver(), + context.shared_supervision(), + ) + .await + } else { + openshell_driver_kubernetes::KubernetesComputeDriver::new( + config, + context.shutdown_receiver(), + ) + .await } + .map_err(|error| openshell_core::Error::execution(error.to_string()))?; let driver = openshell_driver_kubernetes::ComputeDriverService::new_in_process(driver); Ok(openshell_server::ComputeDriverInstance::InProcess( std::sync::Arc::new(driver), diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index 93b24ad451..9045a8edf8 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -136,12 +136,10 @@ handshake. Handshakes are bounded globally and per route, and an authenticated connection only fills its own instance's spare queue. A routed instance waits up to five minutes for its first connection, since it is registered before its workload Pod is scheduled, and the usual connect window -afterwards. `sandbox_instance` defines the launch material and status of one -hosted instance: `SandboxInstanceRegistration` is carried as the JSON -`launch_material` of a `SandboxInstanceAssignment` on the gateway's private -`openshell.supervisor.host.v1` stream, validated against the assignment's -route, sandbox, and generation, and `SandboxInstanceStatusReport` converts to -and from the stream's status message. +afterwards. The launch material and status of one hosted instance +(`SandboxInstanceRegistration`, `SandboxInstanceStatusReport`) are defined by +`openshell-supervisor-control`, which depends on this crate for the runtime +descriptor and route validation. Runtime-initiated transport supports TCP, protected Unix sockets, and host vsock listeners. These share bounded TLS handshakes, the authenticated spare diff --git a/crates/openshell-sandbox-backend/src/lib.rs b/crates/openshell-sandbox-backend/src/lib.rs index 2fda81f66d..6e1bc80872 100644 --- a/crates/openshell-sandbox-backend/src/lib.rs +++ b/crates/openshell-sandbox-backend/src/lib.rs @@ -20,7 +20,6 @@ mod runtime; mod runtime_initiated; pub mod runtime_routed; pub mod sandbox_auth; -pub mod sandbox_instance; pub mod stream_adapter; pub use openshell_core::proto::sandbox::backend::v1 as backend_proto; diff --git a/crates/openshell-sandbox-backend/src/sandbox_instance.rs b/crates/openshell-sandbox-backend/src/sandbox_instance.rs deleted file mode 100644 index 6ce1606842..0000000000 --- a/crates/openshell-sandbox-backend/src/sandbox_instance.rs +++ /dev/null @@ -1,356 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Launch material and status of a sandbox instance hosted by a supervisor -//! shared by many sandboxes. -//! -//! A sandbox instance is one running generation of a stable Sandbox resource. -//! The gateway assigns instances to the shared supervisor over the private -//! `openshell.supervisor.host.v1` stream; each assignment carries one -//! instance's [`SandboxInstanceRegistration`] as JSON launch material, and the -//! host reports each instance's [`SandboxInstanceStatusReport`] back. A -//! registration grants the supervisor exactly the per-generation gateway and -//! Sandbox Protocol credentials it carries; it confers no other identity and -//! is never available to a sandbox. - -use std::time::Duration; - -use openshell_core::proto::supervisor_host::v1::{ - SandboxInstanceAssignment, SandboxInstancePhase, SandboxInstanceStatus, -}; -use openshell_isolation_interface::contract::BackendError; -use serde::{Deserialize, Serialize}; - -use crate::boundary_protocol::{SandboxRuntimeDescriptor, SandboxTransport}; - -/// Largest launch material one assignment may carry. -pub const MAX_LAUNCH_MATERIAL_BYTES: usize = 256 * 1024; -/// Time a shared supervisor gives a withdrawn sandbox instance's workload to -/// stop before cancelling the instance. -pub const INSTANCE_REMOVAL_GRACE: Duration = Duration::from_secs(20); - -/// PEM private key that never appears in `Debug` output. -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(transparent)] -pub struct SecretPem(String); - -impl SecretPem { - #[must_use] - pub fn new(pem: String) -> Self { - Self(pem) - } - - #[must_use] - pub fn expose_secret(&self) -> &str { - &self.0 - } -} - -impl std::fmt::Debug for SecretPem { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("SecretPem([REDACTED])") - } -} - -/// Proxy interception CA issued for one generation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ProxyCaMaterial { - pub certificate_pem: String, - pub private_key_pem: SecretPem, -} - -/// W3C trace context of the operation that created the generation. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct RegistrationTraceContext { - pub traceparent: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tracestate: Option, -} - -/// Trusted launch material for one sandbox instance on a shared supervisor. -#[derive(Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SandboxInstanceRegistration { - /// Route handle the runtime names; also the instance's key on the host. - pub route: String, - pub sandbox_id: String, - pub sandbox_name: String, - /// Supervisor-side launch descriptor using the routed transport. - pub runtime_descriptor: SandboxRuntimeDescriptor, - /// Gateway-issued credentials for exactly this generation. - pub auth: openshell_core::jwt::SupervisorAuthBundle, - /// Encoded `MainProcessConfig` for the canonical process. - pub main_process_spec: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub workdir: Option, - /// Workload image recorded in the instance's OCSF events. - #[serde(default)] - pub container_image: String, - pub proxy_ca: ProxyCaMaterial, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub trace_context: Option, -} - -impl std::fmt::Debug for SandboxInstanceRegistration { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("SandboxInstanceRegistration") - .field("route", &self.route) - .field("sandbox_id", &self.sandbox_id) - .field("sandbox_name", &self.sandbox_name) - .field("runtime_descriptor", &self.runtime_descriptor) - .field("auth", &self.auth) - .field("workdir", &self.workdir) - .field("container_image", &self.container_image) - .finish_non_exhaustive() - } -} - -impl SandboxInstanceRegistration { - /// Check that every part of the registration names the same sandbox, - /// generation, launch session, and route. - pub fn validate(&self) -> Result<(), BackendError> { - crate::runtime_routed::validate_route_handle(&self.route)?; - let descriptor = &self.runtime_descriptor; - match &descriptor.transport { - SandboxTransport::RuntimeInitiatedRouted { route, .. } if *route == self.route => {} - _ => { - return Err(BackendError::Descriptor( - "shared supervisor sandbox instances require this route's routed transport" - .to_string(), - )); - } - } - crate::runtime::validate_runtime_coordinates(descriptor)?; - if self.sandbox_id.is_empty() || descriptor.boundary_id != self.sandbox_id { - return Err(BackendError::Descriptor( - "runtime descriptor does not match the registered sandbox".to_string(), - )); - } - if descriptor.generation != self.auth.runtime_generation.as_str() - || descriptor.session_id != self.auth.session_id - { - return Err(BackendError::Descriptor( - "launch credentials do not match the runtime generation".to_string(), - )); - } - self.auth - .validate() - .map_err(|error| BackendError::Descriptor(error.to_string()))?; - openshell_core::sandbox_env::MainProcessConfig::decode(&self.main_process_spec) - .map_err(BackendError::Descriptor)?; - if self.proxy_ca.certificate_pem.is_empty() - || self.proxy_ca.private_key_pem.expose_secret().is_empty() - { - return Err(BackendError::Descriptor( - "proxy CA material is required".to_string(), - )); - } - Ok(()) - } - - /// Encode this registration as the launch material of one assignment. - pub fn to_assignment(&self) -> Result { - let launch_material = serde_json::to_vec(self).map_err(|error| { - BackendError::Descriptor(format!("encode sandbox instance registration: {error}")) - })?; - if launch_material.len() > MAX_LAUNCH_MATERIAL_BYTES { - return Err(BackendError::Descriptor( - "sandbox instance launch material is too large".to_string(), - )); - } - Ok(SandboxInstanceAssignment { - route: self.route.clone(), - sandbox_id: self.sandbox_id.clone(), - generation: self.auth.runtime_generation.to_string(), - launch_material, - }) - } - - /// Decode and validate one assignment's launch material, checking that - /// the assignment's own identifiers agree with it. - pub fn from_assignment(assignment: &SandboxInstanceAssignment) -> Result { - if assignment.launch_material.len() > MAX_LAUNCH_MATERIAL_BYTES { - return Err(BackendError::Descriptor( - "sandbox instance launch material is too large".to_string(), - )); - } - // Decoder errors can quote field values, including credentials. - let registration: Self = - serde_json::from_slice(&assignment.launch_material).map_err(|_| { - BackendError::Descriptor( - "sandbox instance launch material is not a valid registration".to_string(), - ) - })?; - if registration.route != assignment.route - || registration.sandbox_id != assignment.sandbox_id - || registration.auth.runtime_generation.as_str() != assignment.generation - { - return Err(BackendError::Descriptor( - "sandbox instance assignment does not match its launch material".to_string(), - )); - } - registration.validate()?; - Ok(registration) - } -} - -/// Lifecycle phase of a hosted sandbox instance. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(tag = "phase", rename_all = "snake_case")] -pub enum SandboxInstancePhaseReport { - Starting, - Supervising, - Stopping, - Exited { exit_code: i32 }, - Failed { error: String }, -} - -impl SandboxInstancePhaseReport { - /// Supervision has ended and cannot resume for this generation. - #[must_use] - pub fn is_terminal(&self) -> bool { - matches!(self, Self::Exited { .. } | Self::Failed { .. }) - } -} - -/// Status of one hosted sandbox instance. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SandboxInstanceStatusReport { - pub route: String, - pub sandbox_id: String, - #[serde(flatten)] - pub phase: SandboxInstancePhaseReport, - /// The gateway currently accepts this instance's supervision. - pub ready: bool, -} - -impl SandboxInstanceStatusReport { - /// The wire form of this report. - #[must_use] - pub fn to_proto(&self) -> SandboxInstanceStatus { - let (phase, exit_code, error) = match &self.phase { - SandboxInstancePhaseReport::Starting => { - (SandboxInstancePhase::Starting, 0, String::new()) - } - SandboxInstancePhaseReport::Supervising => { - (SandboxInstancePhase::Supervising, 0, String::new()) - } - SandboxInstancePhaseReport::Stopping => { - (SandboxInstancePhase::Stopping, 0, String::new()) - } - SandboxInstancePhaseReport::Exited { exit_code } => { - (SandboxInstancePhase::Exited, *exit_code, String::new()) - } - SandboxInstancePhaseReport::Failed { error } => { - (SandboxInstancePhase::Failed, 0, error.clone()) - } - }; - SandboxInstanceStatus { - route: self.route.clone(), - sandbox_id: self.sandbox_id.clone(), - phase: phase.into(), - ready: self.ready, - exit_code, - error, - } - } - - /// Read a wire status. An unknown phase reads as a failure, so a newer - /// host can never look healthy to an older gateway. - #[must_use] - pub fn from_proto(status: &SandboxInstanceStatus) -> Self { - let phase = match SandboxInstancePhase::try_from(status.phase) { - Ok(SandboxInstancePhase::Starting) => SandboxInstancePhaseReport::Starting, - Ok(SandboxInstancePhase::Supervising) => SandboxInstancePhaseReport::Supervising, - Ok(SandboxInstancePhase::Stopping) => SandboxInstancePhaseReport::Stopping, - Ok(SandboxInstancePhase::Exited) => SandboxInstancePhaseReport::Exited { - exit_code: status.exit_code, - }, - Ok(SandboxInstancePhase::Failed) => SandboxInstancePhaseReport::Failed { - error: status.error.clone(), - }, - Ok(SandboxInstancePhase::Unspecified) | Err(_) => SandboxInstancePhaseReport::Failed { - error: "sandbox instance reported an unknown phase".to_string(), - }, - }; - let ready = status.ready && !phase.is_terminal(); - Self { - route: status.route.clone(), - sandbox_id: status.sandbox_id.clone(), - phase, - ready, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn status_reports_round_trip_and_unknown_phases_fail_closed() { - for report in [ - SandboxInstanceStatusReport { - route: "rt-a".to_string(), - sandbox_id: "sandbox-a".to_string(), - phase: SandboxInstancePhaseReport::Supervising, - ready: true, - }, - SandboxInstanceStatusReport { - route: "rt-a".to_string(), - sandbox_id: "sandbox-a".to_string(), - phase: SandboxInstancePhaseReport::Exited { exit_code: 3 }, - ready: false, - }, - SandboxInstanceStatusReport { - route: "rt-a".to_string(), - sandbox_id: "sandbox-a".to_string(), - phase: SandboxInstancePhaseReport::Failed { - error: "boom".to_string(), - }, - ready: false, - }, - ] { - assert_eq!( - SandboxInstanceStatusReport::from_proto(&report.to_proto()), - report - ); - } - let unknown = SandboxInstanceStatus { - route: "rt-a".to_string(), - sandbox_id: "sandbox-a".to_string(), - phase: 99, - ready: true, - exit_code: 0, - error: String::new(), - }; - let report = SandboxInstanceStatusReport::from_proto(&unknown); - assert!(report.phase.is_terminal()); - assert!(!report.ready); - } - - #[test] - fn assignments_must_agree_with_their_launch_material() { - let mismatched = SandboxInstanceAssignment { - route: "rt-0123456789abcdef".to_string(), - sandbox_id: "sandbox".to_string(), - generation: "gen".to_string(), - launch_material: b"{}".to_vec(), - }; - let error = SandboxInstanceRegistration::from_assignment(&mismatched).unwrap_err(); - assert!(error.to_string().contains("not a valid registration")); - let oversized = SandboxInstanceAssignment { - launch_material: vec![b' '; MAX_LAUNCH_MATERIAL_BYTES + 1], - ..mismatched - }; - assert!( - SandboxInstanceRegistration::from_assignment(&oversized) - .unwrap_err() - .to_string() - .contains("too large") - ); - } -} diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 0066cd8d5b..9160719bf9 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -25,6 +25,7 @@ openshell-policy = { path = "../openshell-policy" } openshell-prover = { path = "../openshell-prover" } openshell-providers = { path = "../openshell-providers" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } +openshell-supervisor-control = { path = "../openshell-supervisor-control" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } # Kubernetes client used by ServiceAccount bootstrap authentication and the diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs index 825cd8a0e4..bc5d31381a 100644 --- a/crates/openshell-server/src/auth/principal.rs +++ b/crates/openshell-server/src/auth/principal.rs @@ -89,7 +89,7 @@ pub struct SandboxPrincipal { #[derive(Debug, Clone)] pub struct SupervisorPrincipal { /// Claims of the verified supervisor token. - pub identity: openshell_core::supervisor_host::HostIdentity, + pub identity: openshell_supervisor_control::HostIdentity, } /// Gateway peer caller. diff --git a/crates/openshell-server/src/auth/supervisor_token.rs b/crates/openshell-server/src/auth/supervisor_token.rs index c3274693d4..b8c77ec233 100644 --- a/crates/openshell-server/src/auth/supervisor_token.rs +++ b/crates/openshell-server/src/auth/supervisor_token.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use async_trait::async_trait; use jsonwebtoken::decode_header; use openshell_core::jwt::SUPERVISOR_JWT_TYPE; -use openshell_core::supervisor_host::HostIdentity; +use openshell_supervisor_control::HostIdentity; use super::authenticator::{AuthRejection, Authentication, Authenticator, bearer_credential}; use super::principal::{Principal, SupervisorPrincipal}; diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index d69db21753..7c31f30cfe 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -763,9 +763,9 @@ pub struct ComputeRuntime { restart_notify: Arc, ssh_identities: Arc>, supervisor_ocsf_headers: Arc>, - /// Desired assignments of the shared supervisor host, when the in-process - /// driver installed one. - supervisor_host: Option, + /// Desired assignments of the shared supervisor host, when the gateway + /// coordinates one for the in-process driver. + supervisor_host: Option, } impl fmt::Debug for ComputeRuntime { @@ -878,10 +878,11 @@ impl ComputeRuntime { }) } - /// Serve the shared supervisor host that the in-process driver installed. + /// Serve the shared supervisor host the gateway coordinates for the + /// in-process driver. pub(crate) fn with_supervisor_host( mut self, - supervisor_host: Option, + supervisor_host: Option, ) -> Self { self.supervisor_host = supervisor_host; self @@ -891,7 +892,7 @@ impl ComputeRuntime { #[must_use] pub fn supervisor_host( &self, - ) -> Option<&openshell_core::supervisor_host::SupervisorHostController> { + ) -> Option<&openshell_supervisor_control::SupervisorHostController> { self.supervisor_host.as_ref() } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 417312fc25..f1f991e4ae 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -248,7 +248,7 @@ fn spawn_gateway_extension_token_refresh( fn spawn_supervisor_token_refresh( authority: Arc, ttl: Duration, - controller: openshell_core::supervisor_host::SupervisorHostController, + controller: openshell_supervisor_control::SupervisorHostController, mut shutdown_rx: watch::Receiver, ) { tokio::spawn(async move { @@ -264,7 +264,7 @@ fn spawn_supervisor_token_refresh( .unwrap_or(i64::MAX), ); controller.publish_credential( - openshell_core::supervisor_host::SupervisorCredential { + openshell_supervisor_control::SupervisorCredential { token: minted.token, expires_at: minted.expires_at, }, @@ -1695,21 +1695,20 @@ impl ComputeDriverConfigContext<'_> { pub struct ComputeDriverBuildContext<'a> { config: ComputeDriverConfigContext<'a>, shutdown_rx: watch::Receiver, - supervisor_host: &'a OnceLock, + shared_supervision: &'a OnceLock, } impl ComputeDriverBuildContext<'_> { - /// Let the gateway serve the shared supervisor whose desired assignments - /// this in-process driver owns. The gateway mints the supervisor token, - /// publishes each renewal into the controller for the driver to deliver, - /// and routes streams authenticated by that token to it. - pub fn install_supervisor_host( - &self, - controller: openshell_core::supervisor_host::SupervisorHostController, - ) -> Result<()> { - self.supervisor_host.set(controller).map_err(|_| { - Error::config("a compute driver may install only one supervisor host controller") - }) + /// The gateway's coordination of a shared supervisor host, for an + /// in-process driver whose configuration places new generations on it. + /// The gateway owns the desired set, serves it to the host over the + /// private assignment stream, mints the supervisor token, and publishes + /// each renewal for the driver to deliver. Created on first request; every + /// call returns the same coordination. + pub fn shared_supervision(&self) -> openshell_supervisor_control::SharedSupervision { + self.shared_supervision + .get_or_init(openshell_supervisor_control::SharedSupervision::new) + .clone() } #[must_use] @@ -1829,7 +1828,7 @@ async fn build_compute_runtime( info!(driver = %driver.name(), "Using compute driver"); let runtime = match driver { ConfiguredComputeDriver::Registered(registration) => { - let supervisor_host = OnceLock::new(); + let shared_supervision = OnceLock::new(); let build_context = ComputeDriverBuildContext { config: ComputeDriverConfigContext { driver_name: ®istration.name, @@ -1839,10 +1838,12 @@ async fn build_compute_runtime( driver_startup, }, shutdown_rx, - supervisor_host: &supervisor_host, + shared_supervision: &shared_supervision, }; let instance = registration.factory.build(build_context).await?; - let supervisor_host = supervisor_host.into_inner(); + let supervisor_host = shared_supervision + .into_inner() + .map(|supervision| supervision.controller().clone()); match instance { ComputeDriverInstance::InProcess(driver) => ComputeRuntime::from_driver( registration.name, @@ -1861,7 +1862,7 @@ async fn build_compute_runtime( .with_supervisor_host(supervisor_host), ComputeDriverInstance::ManagedRemote(_) if supervisor_host.is_some() => { return Err(Error::config( - "only an in-process compute driver can install a supervisor host controller", + "only an in-process compute driver can use the gateway's shared supervision", )); } ComputeDriverInstance::ManagedRemote(mut endpoint) => { diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index ff1ca10da2..baf9b53ef3 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -3566,7 +3566,7 @@ mod tests { async fn supervisor_tokens_reach_only_the_assignment_stream() { let attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; let supervisor = Principal::Supervisor(crate::auth::principal::SupervisorPrincipal { - identity: openshell_core::supervisor_host::HostIdentity { + identity: openshell_supervisor_control::HostIdentity { subject: "openshell-supervisor:test".to_string(), token_id: "token".to_string(), expires_at: i64::MAX, diff --git a/crates/openshell-supervisor-control/Cargo.toml b/crates/openshell-supervisor-control/Cargo.toml new file mode 100644 index 0000000000..b61564c2a5 --- /dev/null +++ b/crates/openshell-supervisor-control/Cargo.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-supervisor-control" +description = "Gateway-side control of a supervisor shared by many sandboxes: assignments, launch registrations, recovery, and host observations" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt"] } +openshell-otel = { path = "../openshell-otel" } +openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } +prost-types = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tonic = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-supervisor-control/README.md b/crates/openshell-supervisor-control/README.md new file mode 100644 index 0000000000..124caa5080 --- /dev/null +++ b/crates/openshell-supervisor-control/README.md @@ -0,0 +1,65 @@ +# Shared supervisor control + +`openshell-supervisor-control` is the gateway side of a supervisor shared by +many sandboxes. One shared supervisor process hosts the sandbox instances (one +running generation of a stable Sandbox each) of a gateway; the gateway owns +which instances it hosts and serves that set to it over the private +`openshell.supervisor.host.v1` stream, which the supervisor opens with a +gateway-minted supervisor token. Everything about that control relationship +that is not specific to a compute platform lives here, so a compute driver +adds a shared supervisor by adapting storage and infrastructure, not by +re-implementing the control contracts. + +## Modules + +- `controller`: `SupervisorHostController` holds the desired set of + `SandboxInstanceAssignment`s and serves it to the host. Desired state is + uninitialized until the complete set has been derived from durable records + once, so a restarted gateway never presents an empty set as authoritative; + an explicit empty set after initialization does stop instances. Full + replacements are compare-and-replace against the revision read before the + records were listed. One host session at a time: a newer connection epoch + of the same host process or a new process supersedes the current stream, a + delayed older hello is refused, and reports from superseded streams, acks + of revisions never sent, regressing acks, and oversized reports are + dropped. A stream ends when the token that opened it expires; hosted + instances keep running. At most 256 instances are assigned. +- `registration`: `SandboxInstanceRegistration` is the JSON launch material + one assignment carries and the driver persists with the generation. It + grants the supervisor exactly that generation's gateway and Sandbox Protocol + credentials. `validate` binds route, sandbox, generation, launch session, + and proxy CA together; `from_launch_material` and `from_assignment` decode + bounded material without quoting values, which include credentials. +- `status`: `SandboxInstanceStatusReport` and its phases, converted to and + from the stream's status message; an unknown phase reads as a failure. +- `coordinator`: `SharedSupervision` is what the gateway constructs and hands + to the compute driver. `launch` validates the driver's registration, + persists the encoded launch material through the driver's operation, and + only then assigns the instance; a host at capacity is refused before the + workload runs. `begin_recovery` captures + the revision before the driver lists its durable records and + `complete_recovery` reuses current assignments, reads the rest through the + driver's operation, binds each to its recorded route and sandbox, and keeps + the previous set when any read fails; a list that raced a lifecycle edit is + refused and derived again. `supervision` and `wait_for_ready` interpret the + host's observations in one place: a disconnected host is unknown, a host + that has not reconciled to the assignment yet is degraded, and only a host + that reconciled past it without hosting the instance, hosts it for another + sandbox, or reports it ended makes it unavailable. +- `credential`: `deliver_credentials` delivers the current supervisor token + and every renewal through the driver's operation, retrying a failed + delivery until it succeeds or a newer token supersedes it. +- `proto`: the private wire protocol, generated in `openshell-core`. + +## Ownership + +The gateway (`openshell-server`) constructs the `SharedSupervision` when the +selected in-process driver asks for it, keeps the controller to serve the +host's `Attach` stream and to publish the token it mints and renews at half +its lifetime, and gives the coordination to the driver. The driver +(`openshell-driver-kubernetes` today) records placements, persists and reads +back launch material, provisions the workload's path to the supervisor, checks +its own infrastructure evidence alongside the coordinator's verdict, and +delivers the token into the Secret the supervisor Pod mounts. The supervisor +(`openshell-supervisor`) consumes only the registration and status types; it +owns admission, per-instance runtime, routing, and cleanup. diff --git a/crates/openshell-core/src/supervisor_host.rs b/crates/openshell-supervisor-control/src/controller.rs similarity index 96% rename from crates/openshell-core/src/supervisor_host.rs rename to crates/openshell-supervisor-control/src/controller.rs index 25d032ba14..2d0aa885d9 100644 --- a/crates/openshell-core/src/supervisor_host.rs +++ b/crates/openshell-supervisor-control/src/controller.rs @@ -5,19 +5,20 @@ //! (`openshell.supervisor.host.v1`). //! //! The gateway owns the desired set of sandbox instances a shared supervisor -//! runs. A compute driver publishes that set here; the gateway's gRPC server -//! hands each host stream authenticated by a supervisor token to +//! runs. The [`SharedSupervision`](crate::SharedSupervision) coordinator +//! publishes that set here on behalf of a compute driver; the gateway's gRPC +//! server hands each host stream authenticated by a supervisor token to //! [`SupervisorHostController::attach`], which sends complete snapshots and //! records what the host reports. The gateway also mints that token and //! publishes each renewal here for the driver to deliver to the host. //! -//! Desired state is uninitialized until the driver has derived it from its -//! durable records once. Nothing is sent before that, so a restarted gateway -//! can never present an empty map as authoritative and stop running +//! Desired state is uninitialized until the complete set has been derived +//! from durable records once. Nothing is sent before that, so a restarted +//! gateway can never present an empty map as authoritative and stop running //! instances. An explicit empty snapshot, after initialization, does stop -//! them. Full resyncs are compare-and-replace against the revision the driver -//! read before it started, so a list that raced a lifecycle edit never -//! resurrects a withdrawn instance or discards a new one. +//! them. Full resyncs are compare-and-replace against the revision read +//! before the records were listed, so a list that raced a lifecycle edit +//! never resurrects a withdrawn instance or discards a new one. //! //! Exactly one host serves a gateway: a new attach from a newer connection of //! the same host process, or from a new host process, supersedes the current @@ -33,7 +34,7 @@ use tokio_stream::StreamExt as _; use tokio_stream::wrappers::ReceiverStream; use tonic::{Status, Streaming}; -use crate::proto::supervisor_host::v1::{ +use crate::proto::{ AssignmentSnapshot, HostAccepted, HostGatewayHeartbeat, HostGatewayMessage, HostHello, HostMessage, HostStatus, SandboxInstanceAssignment, SandboxInstanceStatus, host_gateway_message, host_message, @@ -77,7 +78,7 @@ pub struct HostIdentity { /// its deployment packaging provides; the Kubernetes driver writes a Secret. #[derive(Clone, Debug)] pub struct SupervisorCredential { - pub token: crate::jwt::SecretJwt, + pub token: openshell_core::jwt::SecretJwt, /// Unix seconds. pub expires_at: i64, } @@ -234,14 +235,17 @@ impl SupervisorHostController { self.inner.credential.subscribe() } - /// The driver has derived the complete desired set at least once. + /// The complete desired set has been derived from durable records at + /// least once. #[must_use] pub fn is_initialized(&self) -> bool { lock(&self.inner.desired).initialized } /// Current desired revision: 0 while uninitialized. Read it before - /// deriving a full list for [`Self::replace_assignments`]. + /// deriving a full list for [`Self::replace_assignments`]; the + /// coordinator does so through + /// [`SharedSupervision::begin_recovery`](crate::SharedSupervision::begin_recovery). #[must_use] pub fn revision(&self) -> u64 { lock(&self.inner.desired).revision @@ -256,8 +260,8 @@ impl SupervisorHostController { .map(|assigned| assigned.assignment.clone()) } - /// Replace the whole desired set with `assignments`, derived from the - /// driver's durable records while the revision was `baseline`. + /// Replace the whole desired set with `assignments`, derived from + /// durable records while the revision was `baseline`. /// /// Initializes the controller; edits made before initialization are /// applied on top of the list. After initialization the replacement is diff --git a/crates/openshell-supervisor-control/src/coordinator.rs b/crates/openshell-supervisor-control/src/coordinator.rs new file mode 100644 index 0000000000..cedbb42bc2 --- /dev/null +++ b/crates/openshell-supervisor-control/src/coordinator.rs @@ -0,0 +1,390 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway-owned coordination of one shared supervisor host. +//! +//! The gateway constructs a [`SharedSupervision`] and hands it to the compute +//! driver that provisions the host's sandbox instances. The driver supplies +//! each instance's registration and operations over its own durable records; +//! this module owns the contracts those operations are sequenced by: launch +//! material is persisted before an instance is assigned, a recovery baseline +//! is captured before the driver lists its records, an incomplete recovery +//! never replaces the published set, and the host's observations are +//! interpreted in one place. The driver keeps everything that is specific to +//! its infrastructure: where the records live, how its workloads reach the +//! host, and which of its own objects must exist for an instance to count as +//! supervised. + +use std::fmt::Display; +use std::future::Future; +use std::time::Duration; + +use tokio::sync::watch; + +use crate::controller::{ + AssignmentError, ObservedInstance, SupervisorCredential, SupervisorHostController, +}; +use crate::registration::{RegistrationError, SandboxInstanceRegistration}; +use crate::status::SandboxInstanceStatusReport; + +/// Interval at which [`SharedSupervision::wait_for_ready`] re-reads the +/// host's observations. +const READINESS_POLL: Duration = Duration::from_secs(1); + +/// Why a launch was not assigned. +#[derive(Debug, thiserror::Error)] +pub enum LaunchError { + /// The registration is not valid; nothing was persisted or assigned. + #[error("{0}")] + Invalid(RegistrationError), + /// The driver could not persist the launch material; nothing was assigned. + #[error("{0}")] + Persist(E), + /// The host is at capacity. The material was persisted; the caller rolls + /// the generation back before its workload runs. + #[error("{0}")] + Refused(AssignmentError), +} + +/// Revision of the desired set read before the driver listed its durable +/// records. Completing a recovery with it makes the replacement +/// compare-and-replace against that revision. +#[must_use = "complete the recovery with the records listed after this baseline was read"] +pub struct RecoveryBaseline { + revision: u64, +} + +/// A generation that may run on the shared supervisor, as the driver's +/// durable records describe it, with the record the driver needs to read its +/// persisted launch material. +pub struct RecoverableInstance { + pub route: String, + pub sandbox_id: String, + pub record: R, +} + +/// Result of a recovery pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecoveryOutcome { + /// The desired set matches the records; carries the revision in force. + Published(u64), + /// The desired set changed while the records were listed; the next pass + /// derives them again. + Conflict, + /// Launch material could not be read or does not match its record, or the + /// set exceeds the host's capacity; the previous set stays published. + Kept, +} + +/// What the host's observations say about one instance's supervision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InstanceSupervision { + /// The host supervises the instance and the gateway accepts it. + Available, + /// The host has not reconciled to the assigning snapshot yet, or hosts + /// the instance but it is still starting. + Degraded, + /// The host reconciled past the assignment and does not host the + /// instance, hosts it for another sandbox, or reports it ended. + Unavailable, + /// No host is connected, so nothing is known. + Unknown, +} + +/// The gateway's coordination of one shared supervisor host: the desired set +/// it publishes, the token it delivers, and the observations it reads back. +/// +/// Clones share one controller. The gateway keeps one for serving the host's +/// stream and minting its token, and gives one to the compute driver. +#[derive(Clone, Debug, Default)] +pub struct SharedSupervision { + controller: SupervisorHostController, +} + +impl SharedSupervision { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// The controller the gateway serves to the host and publishes the + /// supervisor token into. + #[must_use] + pub fn controller(&self) -> &SupervisorHostController { + &self.controller + } + + /// The current supervisor token and every renewal after it, for the + /// driver to deliver to the host (see + /// [`deliver_credentials`](crate::credential::deliver_credentials)). + #[must_use] + pub fn credentials(&self) -> watch::Receiver> { + self.controller.credential() + } + + /// Register a generation's sandbox instance: validate `registration`, + /// persist its encoded launch material through `persist`, and only then + /// assign the instance so the host starts it. The launch material is + /// persisted first so a restarted gateway recovers the assignment from the + /// generation's own records. An identical assignment changes nothing. + /// Capacity is refused as [`LaunchError::Refused`], so the caller fails + /// its create before the workload runs. + pub async fn launch( + &self, + registration: &SandboxInstanceRegistration, + persist: impl FnOnce(Vec) -> F + Send, + ) -> Result<(), LaunchError> + where + E: Send, + F: Future> + Send, + { + registration.validate().map_err(LaunchError::Invalid)?; + let assignment = registration.to_assignment().map_err(LaunchError::Invalid)?; + persist(assignment.launch_material.clone()) + .await + .map_err(LaunchError::Persist)?; + self.controller + .assign(assignment) + .map_err(LaunchError::Refused) + } + + /// Withdraw a generation's sandbox instance so the host stops it. Nothing + /// happens for a route that is not assigned. + pub fn withdraw(&self, route: &str) { + self.controller.unassign(route); + } + + /// Start a recovery pass. Call it before listing the durable records the + /// pass derives the desired set from, so a lifecycle edit that lands + /// during the list is detected by [`Self::complete_recovery`] instead of + /// being undone. + pub fn begin_recovery(&self) -> RecoveryBaseline { + RecoveryBaseline { + revision: self.controller.revision(), + } + } + + /// Replace the desired set with `candidates`, the generations that may + /// run according to the records listed after `baseline` was read. Each + /// candidate's launch material comes from its current assignment or, for + /// a generation this gateway has not assigned since it started, from + /// `read` on the driver's record, which returns the persisted bytes or + /// `None` when they are gone. Material is decoded without quoting values, + /// bound to the recorded route and sandbox, and validated. + /// + /// The first completed pass initializes the desired set; until then + /// nothing is sent to the host. A failed or mismatched read keeps the + /// previous set, because an incomplete set would stop running instances. + /// A pass that raced a lifecycle edit is refused and derived again next + /// time. + pub async fn complete_recovery( + &self, + baseline: RecoveryBaseline, + candidates: Vec>, + read: impl Fn(&R) -> F + Send + Sync, + ) -> RecoveryOutcome + where + R: Send + Sync, + E: Display, + F: Future>, E>> + Send, + { + let mut assignments = Vec::with_capacity(candidates.len()); + for candidate in &candidates { + if let Some(assignment) = self.controller.assignment(&candidate.route) { + assignments.push(assignment); + continue; + } + let material = match read(&candidate.record).await { + Ok(Some(material)) => material, + Ok(None) => { + tracing::warn!( + sandbox_id = %candidate.sandbox_id, + route = %candidate.route, + "shared sandbox instance has no launch material; it stays unassigned" + ); + continue; + } + Err(error) => { + tracing::warn!( + sandbox_id = %candidate.sandbox_id, + route = %candidate.route, + %error, + "could not read shared sandbox instance launch material; keeping the previous assignments" + ); + return RecoveryOutcome::Kept; + } + }; + match recovered_assignment(candidate, &material) { + Ok(assignment) => assignments.push(assignment), + Err(error) => { + tracing::warn!( + sandbox_id = %candidate.sandbox_id, + route = %candidate.route, + %error, + "shared sandbox instance launch material is unusable; keeping the previous assignments" + ); + return RecoveryOutcome::Kept; + } + } + } + match self + .controller + .replace_assignments(baseline.revision, assignments) + { + Ok(revision) => RecoveryOutcome::Published(revision), + Err(AssignmentError::Conflict { .. }) => RecoveryOutcome::Conflict, + Err(error) => { + tracing::warn!(%error, "could not publish the shared sandbox instances; keeping the previous assignments"); + RecoveryOutcome::Kept + } + } + } + + /// What the host's observations say about the instance of `sandbox_id` + /// on `route`. Only a host that has reconciled to the assigning snapshot + /// renders a verdict; a disconnected or still-reconciling host is not + /// evidence that a running instance is gone. + #[must_use] + pub fn supervision(&self, route: &str, sandbox_id: &str) -> InstanceSupervision { + match self.controller.observed(route) { + ObservedInstance::Reported(status) => { + let status = SandboxInstanceStatusReport::from_proto(&status); + if status.sandbox_id != sandbox_id || status.phase.is_terminal() { + InstanceSupervision::Unavailable + } else if status.ready { + InstanceSupervision::Available + } else { + InstanceSupervision::Degraded + } + } + ObservedInstance::Pending => InstanceSupervision::Degraded, + ObservedInstance::Absent => InstanceSupervision::Unavailable, + ObservedInstance::NoHost => InstanceSupervision::Unknown, + } + } + + /// Wait for the instance to become available: `true` once the host + /// reports it ready, `false` once the host has reconciled past the + /// assignment without hosting it or reports it ended. A disconnected or + /// still-reconciling host is not a verdict, so the caller bounds the wait + /// with its own bootstrap grace. + pub async fn wait_for_ready(&self, route: &str, sandbox_id: &str) -> bool { + loop { + match self.supervision(route, sandbox_id) { + InstanceSupervision::Available => return true, + InstanceSupervision::Unavailable => return false, + InstanceSupervision::Degraded | InstanceSupervision::Unknown => { + tokio::time::sleep(READINESS_POLL).await; + } + } + } + } +} + +/// Decode persisted launch material for `candidate`, binding it to the +/// recorded route and sandbox before trusting it. +fn recovered_assignment( + candidate: &RecoverableInstance, + material: &[u8], +) -> Result { + let registration = SandboxInstanceRegistration::from_launch_material(material)?; + if registration.route != candidate.route || registration.sandbox_id != candidate.sandbox_id { + return Err(RegistrationError::new( + "shared sandbox instance launch material does not match the recorded generation", + )); + } + registration.validate()?; + registration.to_assignment() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto::SandboxInstanceAssignment; + + fn candidate(route: &str, sandbox_id: &str) -> RecoverableInstance<&'static str> { + RecoverableInstance { + route: route.to_string(), + sandbox_id: sandbox_id.to_string(), + record: "record", + } + } + + #[tokio::test] + async fn recovery_keeps_the_previous_set_unless_every_record_is_usable() { + let supervision = SharedSupervision::new(); + let assigned = SandboxInstanceAssignment { + route: "rt-assigned-0123456789".to_string(), + sandbox_id: "sandbox-a".to_string(), + generation: "gen".to_string(), + launch_material: b"opaque".to_vec(), + }; + // An assignment made before the first recovery is carried by it. + supervision.controller().assign(assigned.clone()).unwrap(); + + // A record whose material cannot be read keeps the set uninitialized. + let baseline = supervision.begin_recovery(); + let outcome = supervision + .complete_recovery( + baseline, + vec![ + candidate(&assigned.route, &assigned.sandbox_id), + candidate("rt-unreadable-0123456789", "sandbox-b"), + ], + |_record| async { Err::>, _>("boom") }, + ) + .await; + assert_eq!(outcome, RecoveryOutcome::Kept); + assert!(!supervision.controller().is_initialized()); + + // Material that is not a registration is refused without quoting it. + let baseline = supervision.begin_recovery(); + let outcome = supervision + .complete_recovery( + baseline, + vec![candidate("rt-garbage-0123456789", "sandbox-c")], + |_record| async { Ok::<_, &str>(Some(b"{\"secret\":1}".to_vec())) }, + ) + .await; + assert_eq!(outcome, RecoveryOutcome::Kept); + assert!(!supervision.controller().is_initialized()); + + // A record without material is skipped; the current assignment is + // reused without a read; the pass initializes the set. + let baseline = supervision.begin_recovery(); + let outcome = supervision + .complete_recovery( + baseline, + vec![ + candidate(&assigned.route, &assigned.sandbox_id), + candidate("rt-gone-0123456789", "sandbox-d"), + ], + |_record| async { Ok::>, &str>(None) }, + ) + .await; + assert_eq!(outcome, RecoveryOutcome::Published(1)); + assert!(supervision.controller().is_initialized()); + assert_eq!( + supervision.controller().assignment(&assigned.route), + Some(assigned.clone()) + ); + + // A baseline read before a lifecycle edit is refused, and the edit stands. + let baseline = supervision.begin_recovery(); + supervision.withdraw(&assigned.route); + let outcome = supervision + .complete_recovery( + baseline, + vec![candidate(&assigned.route, &assigned.sandbox_id)], + |_record| async { Ok::>, &str>(None) }, + ) + .await; + assert_eq!(outcome, RecoveryOutcome::Conflict); + assert!( + supervision + .controller() + .assignment(&assigned.route) + .is_none() + ); + } +} diff --git a/crates/openshell-supervisor-control/src/credential.rs b/crates/openshell-supervisor-control/src/credential.rs new file mode 100644 index 0000000000..7a2d9423e1 --- /dev/null +++ b/crates/openshell-supervisor-control/src/credential.rs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Delivery of the gateway-minted supervisor token to the shared supervisor. +//! +//! The gateway mints and renews the token and publishes each renewal into +//! the controller. How the token reaches the supervisor process is +//! deployment packaging (the Kubernetes driver patches a Secret the +//! supervisor Pod mounts); the retry policy around that delivery is the +//! same for every packaging and lives here. + +use std::fmt::Display; +use std::future::Future; +use std::time::Duration; + +use tokio::sync::watch; + +use crate::controller::SupervisorCredential; + +/// Deliver the current supervisor token and every renewal after it. +/// +/// The initial value is delivered too. A failed delivery is retried every +/// `retry` until it succeeds or a newer token supersedes it. Returns at +/// shutdown or once the publisher is gone. +pub async fn deliver_credentials( + mut credential: watch::Receiver>, + mut shutdown: watch::Receiver, + retry: Duration, + mut deliver: F, +) where + E: Display, + F: FnMut(SupervisorCredential) -> Fut, + Fut: Future>, +{ + loop { + let current = credential.borrow_and_update().clone(); + if let Some(current) = current { + while let Err(error) = deliver(current.clone()).await { + tracing::warn!(%error, "supervisor token delivery failed; retrying"); + tokio::select! { + () = tokio::time::sleep(retry) => {} + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return; + } + } + } + if credential.has_changed().unwrap_or(true) { + break; + } + } + } + tokio::select! { + changed = credential.changed() => { + if changed.is_err() { + return; + } + } + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return; + } + } + } + } +} diff --git a/crates/openshell-supervisor-control/src/lib.rs b/crates/openshell-supervisor-control/src/lib.rs new file mode 100644 index 0000000000..4401ff3d84 --- /dev/null +++ b/crates/openshell-supervisor-control/src/lib.rs @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway-side control of a supervisor shared by many sandboxes. +//! +//! One shared supervisor hosts the sandbox instances (one running generation +//! of a stable Sandbox each) of a gateway. The gateway owns which instances +//! it hosts and serves that set to it over the private +//! `openshell.supervisor.host.v1` stream, which the supervisor opens with a +//! gateway-minted supervisor token. This crate holds everything about that +//! control relationship that is not specific to a compute platform: +//! +//! - [`controller`]: the desired set, its revision fencing, the host +//! sessions, and the observations they report. +//! - [`registration`] and [`status`]: the launch material one assignment +//! carries and the status the host reports, with their validation and wire +//! mapping. +//! - [`coordinator`]: the gateway-owned [`SharedSupervision`] a compute +//! driver is handed: validated persist-before-assign launches, recovery +//! from durable records, and interpretation of observations. +//! - [`credential`]: delivery of the supervisor token with retry. +//! +//! The private wire protocol is generated in `openshell-core` and re-exported +//! as [`proto`]. The supervisor process consumes the registration and status +//! types; it never links the controller or coordinator. + +pub mod controller; +pub mod coordinator; +pub mod credential; +pub mod registration; +pub mod status; + +/// The private gateway-to-shared-supervisor protocol +/// (`openshell.supervisor.host.v1`). +pub mod proto { + pub use openshell_core::proto::supervisor_host::v1::*; +} + +pub use controller::{ + AssignmentError, HOST_HEARTBEAT_INTERVAL, HostIdentity, MAX_ASSIGNED_INSTANCES, + MAX_REPORTED_INSTANCES, ObservedInstance, SupervisorCredential, SupervisorHostController, +}; +pub use coordinator::{ + InstanceSupervision, LaunchError, RecoverableInstance, RecoveryBaseline, RecoveryOutcome, + SharedSupervision, +}; +pub use credential::deliver_credentials; +pub use registration::{ + INSTANCE_REMOVAL_GRACE, MAX_LAUNCH_MATERIAL_BYTES, ProxyCaMaterial, RegistrationError, + RegistrationTraceContext, SandboxInstanceRegistration, SecretPem, +}; +pub use status::{SandboxInstancePhaseReport, SandboxInstanceStatusReport}; diff --git a/crates/openshell-supervisor-control/src/registration.rs b/crates/openshell-supervisor-control/src/registration.rs new file mode 100644 index 0000000000..508227554f --- /dev/null +++ b/crates/openshell-supervisor-control/src/registration.rs @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Launch material of a sandbox instance hosted by a supervisor shared by +//! many sandboxes. +//! +//! A sandbox instance is one running generation of a stable Sandbox resource. +//! The gateway assigns instances to the shared supervisor over the private +//! `openshell.supervisor.host.v1` stream; each assignment carries one +//! instance's [`SandboxInstanceRegistration`] as JSON launch material, which +//! the compute driver also persists with the generation so a restarted +//! gateway can recover its assignments. A registration grants the supervisor +//! exactly the per-generation gateway and Sandbox Protocol credentials it +//! carries; it confers no other identity and is never available to a sandbox. + +use std::time::Duration; + +use openshell_sandbox_backend::boundary_protocol::{SandboxRuntimeDescriptor, SandboxTransport}; +use serde::{Deserialize, Serialize}; + +use crate::proto::SandboxInstanceAssignment; + +/// Largest launch material one assignment may carry. +pub const MAX_LAUNCH_MATERIAL_BYTES: usize = 256 * 1024; +/// Time a shared supervisor gives a withdrawn sandbox instance's workload to +/// stop before cancelling the instance. +pub const INSTANCE_REMOVAL_GRACE: Duration = Duration::from_secs(20); + +/// Why launch material was refused. The message never quotes field values, +/// which include credentials. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{0}")] +pub struct RegistrationError(String); + +impl RegistrationError { + pub(crate) fn new(message: impl Into) -> Self { + Self(message.into()) + } +} + +/// PEM private key that never appears in `Debug` output. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct SecretPem(String); + +impl SecretPem { + #[must_use] + pub fn new(pem: String) -> Self { + Self(pem) + } + + #[must_use] + pub fn expose_secret(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Debug for SecretPem { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("SecretPem([REDACTED])") + } +} + +/// Proxy interception CA issued for one generation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProxyCaMaterial { + pub certificate_pem: String, + pub private_key_pem: SecretPem, +} + +/// W3C trace context of the operation that created the generation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RegistrationTraceContext { + pub traceparent: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tracestate: Option, +} + +impl RegistrationTraceContext { + /// The trace context of the current span, for the generation created in + /// it; `None` outside an exported trace. + #[must_use] + pub fn current() -> Option { + let environment = openshell_otel::current_trace_context_environment(); + let value = |name: &str| { + environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.clone()) + }; + value(openshell_otel::TRACEPARENT_ENV).map(|traceparent| Self { + traceparent, + tracestate: value(openshell_otel::TRACESTATE_ENV), + }) + } +} + +/// Trusted launch material for one sandbox instance on a shared supervisor. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SandboxInstanceRegistration { + /// Route handle the runtime names; also the instance's key on the host. + pub route: String, + pub sandbox_id: String, + pub sandbox_name: String, + /// Supervisor-side launch descriptor using the routed transport. + pub runtime_descriptor: SandboxRuntimeDescriptor, + /// Gateway-issued credentials for exactly this generation. + pub auth: openshell_core::jwt::SupervisorAuthBundle, + /// Encoded `MainProcessConfig` for the canonical process. + pub main_process_spec: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workdir: Option, + /// Workload image recorded in the instance's OCSF events. + #[serde(default)] + pub container_image: String, + pub proxy_ca: ProxyCaMaterial, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trace_context: Option, +} + +impl std::fmt::Debug for SandboxInstanceRegistration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SandboxInstanceRegistration") + .field("route", &self.route) + .field("sandbox_id", &self.sandbox_id) + .field("sandbox_name", &self.sandbox_name) + .field("runtime_descriptor", &self.runtime_descriptor) + .field("auth", &self.auth) + .field("workdir", &self.workdir) + .field("container_image", &self.container_image) + .finish_non_exhaustive() + } +} + +impl SandboxInstanceRegistration { + /// Check that every part of the registration names the same sandbox, + /// generation, launch session, and route. + pub fn validate(&self) -> Result<(), RegistrationError> { + openshell_sandbox_backend::runtime_routed::validate_route_handle(&self.route) + .map_err(|error| RegistrationError::new(error.to_string()))?; + let descriptor = &self.runtime_descriptor; + match &descriptor.transport { + SandboxTransport::RuntimeInitiatedRouted { route, .. } if *route == self.route => {} + _ => { + return Err(RegistrationError::new( + "shared supervisor sandbox instances require this route's routed transport", + )); + } + } + openshell_sandbox_backend::validate_runtime_coordinates(descriptor) + .map_err(|error| RegistrationError::new(error.to_string()))?; + if self.sandbox_id.is_empty() || descriptor.boundary_id != self.sandbox_id { + return Err(RegistrationError::new( + "runtime descriptor does not match the registered sandbox", + )); + } + if descriptor.generation != self.auth.runtime_generation.as_str() + || descriptor.session_id != self.auth.session_id + { + return Err(RegistrationError::new( + "launch credentials do not match the runtime generation", + )); + } + self.auth + .validate() + .map_err(|error| RegistrationError::new(error.to_string()))?; + openshell_core::sandbox_env::MainProcessConfig::decode(&self.main_process_spec) + .map_err(RegistrationError::new)?; + if self.proxy_ca.certificate_pem.is_empty() + || self.proxy_ca.private_key_pem.expose_secret().is_empty() + { + return Err(RegistrationError::new("proxy CA material is required")); + } + Ok(()) + } + + /// Encode this registration as the launch material of one assignment. + pub fn to_assignment(&self) -> Result { + let launch_material = serde_json::to_vec(self).map_err(|error| { + RegistrationError::new(format!("encode sandbox instance registration: {error}")) + })?; + if launch_material.len() > MAX_LAUNCH_MATERIAL_BYTES { + return Err(RegistrationError::new( + "sandbox instance launch material is too large", + )); + } + Ok(SandboxInstanceAssignment { + route: self.route.clone(), + sandbox_id: self.sandbox_id.clone(), + generation: self.auth.runtime_generation.to_string(), + launch_material, + }) + } + + /// Decode persisted or transmitted launch material without validating + /// it. Decoder errors are not reported, because they can quote field + /// values, including credentials. + pub fn from_launch_material(material: &[u8]) -> Result { + if material.len() > MAX_LAUNCH_MATERIAL_BYTES { + return Err(RegistrationError::new( + "sandbox instance launch material is too large", + )); + } + serde_json::from_slice(material).map_err(|_| { + RegistrationError::new("sandbox instance launch material is not a valid registration") + }) + } + + /// Decode and validate one assignment's launch material, checking that + /// the assignment's own identifiers agree with it. + pub fn from_assignment( + assignment: &SandboxInstanceAssignment, + ) -> Result { + let registration = Self::from_launch_material(&assignment.launch_material)?; + if registration.route != assignment.route + || registration.sandbox_id != assignment.sandbox_id + || registration.auth.runtime_generation.as_str() != assignment.generation + { + return Err(RegistrationError::new( + "sandbox instance assignment does not match its launch material", + )); + } + registration.validate()?; + Ok(registration) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn assignments_must_agree_with_their_launch_material() { + let mismatched = SandboxInstanceAssignment { + route: "rt-0123456789abcdef".to_string(), + sandbox_id: "sandbox".to_string(), + generation: "gen".to_string(), + launch_material: b"{}".to_vec(), + }; + let error = SandboxInstanceRegistration::from_assignment(&mismatched).unwrap_err(); + assert!(error.to_string().contains("not a valid registration")); + let oversized = SandboxInstanceAssignment { + launch_material: vec![b' '; MAX_LAUNCH_MATERIAL_BYTES + 1], + ..mismatched + }; + assert!( + SandboxInstanceRegistration::from_assignment(&oversized) + .unwrap_err() + .to_string() + .contains("too large") + ); + } +} diff --git a/crates/openshell-supervisor-control/src/status.rs b/crates/openshell-supervisor-control/src/status.rs new file mode 100644 index 0000000000..6f5e0e20b7 --- /dev/null +++ b/crates/openshell-supervisor-control/src/status.rs @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Status of a hosted sandbox instance, as the shared supervisor reports it +//! on the private `openshell.supervisor.host.v1` stream. + +use serde::{Deserialize, Serialize}; + +use crate::proto::{SandboxInstancePhase, SandboxInstanceStatus}; + +/// Lifecycle phase of a hosted sandbox instance. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case")] +pub enum SandboxInstancePhaseReport { + Starting, + Supervising, + Stopping, + Exited { exit_code: i32 }, + Failed { error: String }, +} + +impl SandboxInstancePhaseReport { + /// Supervision has ended and cannot resume for this generation. + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Exited { .. } | Self::Failed { .. }) + } +} + +/// Status of one hosted sandbox instance. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxInstanceStatusReport { + pub route: String, + pub sandbox_id: String, + #[serde(flatten)] + pub phase: SandboxInstancePhaseReport, + /// The gateway currently accepts this instance's supervision. + pub ready: bool, +} + +impl SandboxInstanceStatusReport { + /// The wire form of this report. + #[must_use] + pub fn to_proto(&self) -> SandboxInstanceStatus { + let (phase, exit_code, error) = match &self.phase { + SandboxInstancePhaseReport::Starting => { + (SandboxInstancePhase::Starting, 0, String::new()) + } + SandboxInstancePhaseReport::Supervising => { + (SandboxInstancePhase::Supervising, 0, String::new()) + } + SandboxInstancePhaseReport::Stopping => { + (SandboxInstancePhase::Stopping, 0, String::new()) + } + SandboxInstancePhaseReport::Exited { exit_code } => { + (SandboxInstancePhase::Exited, *exit_code, String::new()) + } + SandboxInstancePhaseReport::Failed { error } => { + (SandboxInstancePhase::Failed, 0, error.clone()) + } + }; + SandboxInstanceStatus { + route: self.route.clone(), + sandbox_id: self.sandbox_id.clone(), + phase: phase.into(), + ready: self.ready, + exit_code, + error, + } + } + + /// Read a wire status. An unknown phase reads as a failure, so a newer + /// host can never look healthy to an older gateway. + #[must_use] + pub fn from_proto(status: &SandboxInstanceStatus) -> Self { + let phase = match SandboxInstancePhase::try_from(status.phase) { + Ok(SandboxInstancePhase::Starting) => SandboxInstancePhaseReport::Starting, + Ok(SandboxInstancePhase::Supervising) => SandboxInstancePhaseReport::Supervising, + Ok(SandboxInstancePhase::Stopping) => SandboxInstancePhaseReport::Stopping, + Ok(SandboxInstancePhase::Exited) => SandboxInstancePhaseReport::Exited { + exit_code: status.exit_code, + }, + Ok(SandboxInstancePhase::Failed) => SandboxInstancePhaseReport::Failed { + error: status.error.clone(), + }, + Ok(SandboxInstancePhase::Unspecified) | Err(_) => SandboxInstancePhaseReport::Failed { + error: "sandbox instance reported an unknown phase".to_string(), + }, + }; + let ready = status.ready && !phase.is_terminal(); + Self { + route: status.route.clone(), + sandbox_id: status.sandbox_id.clone(), + phase, + ready, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_reports_round_trip_and_unknown_phases_fail_closed() { + for report in [ + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Supervising, + ready: true, + }, + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Exited { exit_code: 3 }, + ready: false, + }, + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Failed { + error: "boom".to_string(), + }, + ready: false, + }, + ] { + assert_eq!( + SandboxInstanceStatusReport::from_proto(&report.to_proto()), + report + ); + } + let unknown = SandboxInstanceStatus { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: 99, + ready: true, + exit_code: 0, + error: String::new(), + }; + let report = SandboxInstanceStatusReport::from_proto(&unknown); + assert!(report.phase.is_terminal()); + assert!(!report.ready); + } +} diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index b5f08b699b..e6e7fcd6ba 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -20,6 +20,7 @@ openshell-core = { path = "../openshell-core", default-features = false, feature openshell-extension-core = { path = "../openshell-extension-core" } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } +openshell-supervisor-control = { path = "../openshell-supervisor-control" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index d1e240a782..10964b8542 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -151,7 +151,7 @@ control listener: it dials the gateway and receives its assignments from it. `AssignmentSnapshot`s with session-scoped revisions. `shared_host::SharedHost::reconcile` validates the snapshot as a whole (at most 256 instances, no route twice), decodes each assignment's launch material (the JSON - `SandboxInstanceRegistration` of `openshell-sandbox-backend`), admits it into + `SandboxInstanceRegistration` of `openshell-supervisor-control`), admits it into the `SandboxInstanceHost` registry keyed by route with the registration digest as its fingerprint, and removes every hosted route the snapshot omits. An identical assignment is a no-op; one that cannot be admitted diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 11bdfd0885..46eb9e8839 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -38,7 +38,7 @@ use std::time::Duration; use tracing::{Instrument as _, debug, info, warn}; use openshell_core::PolicyValidationFailureMode; -use openshell_sandbox_backend::sandbox_instance::RegistrationTraceContext; +use openshell_supervisor_control::RegistrationTraceContext; use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, diff --git a/crates/openshell-supervisor/src/sandbox_instance_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs index 8e06713f35..66aa30e57e 100644 --- a/crates/openshell-supervisor/src/sandbox_instance_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -28,9 +28,7 @@ use std::time::Duration; use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use openshell_core::grpc_client::GatewayCredentials; use openshell_ocsf::OcsfJsonSettings; -use openshell_sandbox_backend::sandbox_instance::{ - SandboxInstancePhaseReport, SandboxInstanceStatusReport, -}; +use openshell_supervisor_control::{SandboxInstancePhaseReport, SandboxInstanceStatusReport}; use openshell_supervisor_network::policy_local::{DenialLogSource, SandboxInstanceDenialLog}; use openshell_supervisor_process::log_push::LogSink; use tokio::sync::watch; diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 2149355c6a..4dc3e521ef 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -94,7 +94,7 @@ fn run_platform(args: crate::Args) -> Result<()> { gateway_endpoint, instance_root, runtime_listen, - removal_grace: openshell_sandbox_backend::sandbox_instance::INSTANCE_REMOVAL_GRACE, + removal_grace: openshell_supervisor_control::INSTANCE_REMOVAL_GRACE, }, )?; let health = tokio::net::TcpListener::bind(address) diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index 2959f2e337..b78ce7469c 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -32,7 +32,7 @@ use openshell_core::proto::supervisor_host::v1::{ }; use openshell_extension_core::BearerTokenSlot; use openshell_sandbox_backend::runtime_routed::RuntimeRouter; -use openshell_sandbox_backend::sandbox_instance::{ +use openshell_supervisor_control::{ SandboxInstancePhaseReport, SandboxInstanceRegistration, SandboxInstanceStatusReport, }; use tokio::sync::watch; @@ -59,11 +59,11 @@ const RECONNECT_BACKOFF_MAX: Duration = Duration::from_secs(30); const OUTBOUND_QUEUE: usize = 16; /// Most sandbox instances one snapshot may assign to a host; the gateway /// enforces the same bound before it assigns. -pub const MAX_ASSIGNED_INSTANCES: usize = openshell_core::supervisor_host::MAX_ASSIGNED_INSTANCES; +pub const MAX_ASSIGNED_INSTANCES: usize = openshell_supervisor_control::MAX_ASSIGNED_INSTANCES; /// Largest gateway message the host decodes: a full snapshot of /// `MAX_ASSIGNED_INSTANCES` assignments at their largest, plus framing. -const MAX_GATEWAY_MESSAGE_BYTES: usize = MAX_ASSIGNED_INSTANCES - * (openshell_sandbox_backend::sandbox_instance::MAX_LAUNCH_MATERIAL_BYTES + 1024); +const MAX_GATEWAY_MESSAGE_BYTES: usize = + MAX_ASSIGNED_INSTANCES * (openshell_supervisor_control::MAX_LAUNCH_MATERIAL_BYTES + 1024); /// Largest host message: a status report is a few hundred bytes per instance. const MAX_HOST_MESSAGE_BYTES: usize = 1024 * 1024; diff --git a/proto/README.md b/proto/README.md index c66ec95990..f029280f8d 100644 --- a/proto/README.md +++ b/proto/README.md @@ -97,7 +97,7 @@ supervisor token (`auth_mode: "supervisor"`). It is not imported by `openshell.proto`, so SDK generation and gRPC reflection exclude it; the server's schema inventory test classifies it explicitly. `SandboxInstanceAssignment.launch_material` carries the existing -JSON sandbox instance registration of `openshell-sandbox-backend` and is +JSON sandbox instance registration of `openshell-supervisor-control` and is marked secret. ## Isolation backend protocol