From d94fb6a0e7062de0378ad16b9090511e34ff73ff Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 26 Oct 2026 23:23:46 -0700 Subject: [PATCH] feat(supervisor): add independent application listener and endpoints Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 11 + Cargo.lock | 2 + crates/openshell-core/src/app_listener.rs | 71 + crates/openshell-core/src/config.rs | 2 + crates/openshell-core/src/grpc_client.rs | 3 + crates/openshell-core/src/lib.rs | 1 + .../src/app_listener.rs | 325 ++++ .../openshell-driver-kubernetes/src/config.rs | 47 + .../openshell-driver-kubernetes/src/driver.rs | 89 +- crates/openshell-driver-kubernetes/src/lib.rs | 1 + .../openshell-driver-kubernetes/src/main.rs | 2 + .../src/sandbox_runtime.rs | 171 +- crates/openshell-gateway/src/lib.rs | 1 + crates/openshell-server/src/cli.rs | 8 + crates/openshell-server/src/config_file.rs | 2 + crates/openshell-server/src/grpc/policy.rs | 18 +- crates/openshell-server/src/grpc/service.rs | 171 ++ crates/openshell-server/src/lib.rs | 8 + crates/openshell-server/src/storage_proto.rs | 4 +- crates/openshell-supervisor/Cargo.toml | 5 +- crates/openshell-supervisor/README.md | 33 + .../openshell-supervisor/src/app_listener.rs | 1448 +++++++++++++++++ .../src/backend_setup/tests.rs | 2 + .../src/config_runtime.rs | 56 +- .../src/legacy_startup_regression_tests.rs | 1 + crates/openshell-supervisor/src/lib.rs | 32 + crates/openshell-supervisor/src/main.rs | 7 + .../openshell-supervisor/src/shared_host.rs | 10 + .../openshell/ci/values-app-listener.yaml | 15 + deploy/helm/openshell/templates/_helpers.tpl | 37 + deploy/helm/openshell/templates/_toml.tpl | 3 + .../helm/openshell/templates/clusterrole.yaml | 2 +- deploy/helm/openshell/templates/role.yaml | 2 +- .../helm/openshell/templates/supervisor.yaml | 37 + .../tests/app_listener_rbac_test.yaml | 61 + .../openshell/tests/app_listener_test.yaml | 76 + deploy/helm/test-gateway-config-parser.sh | 7 + docs/how-it-works/gateways/configuration.mdx | 48 + e2e/kubernetes/app-listener/README.md | 77 + proto/sandbox.proto | 13 + sdk/go/proto/sandboxv1/sandbox.pb.go | 230 ++- skills/debug-openshell-cluster/SKILL.md | 32 + 42 files changed, 2987 insertions(+), 184 deletions(-) create mode 100644 crates/openshell-core/src/app_listener.rs create mode 100644 crates/openshell-driver-kubernetes/src/app_listener.rs create mode 100644 crates/openshell-supervisor/src/app_listener.rs create mode 100644 deploy/helm/openshell/ci/values-app-listener.yaml create mode 100644 deploy/helm/openshell/tests/app_listener_rbac_test.yaml create mode 100644 deploy/helm/openshell/tests/app_listener_test.yaml create mode 100644 e2e/kubernetes/app-listener/README.md diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 47df909f32..d0fcc7d278 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -611,3 +611,14 @@ for dependencies still declared in `Chart.yaml`. | `deploy/kube/manifests/envoy-gateway-openshell.yaml` | GatewayClass and BackendTrafficPolicy for Envoy Gateway (`mise run helm:gateway:apply`) | | `tasks/scripts/helm-k3s-local.sh` | k3d cluster create/delete/start/stop/status | | `tasks/scripts/keycloak-k8s-setup.sh` | Keycloak deploy, realm import, and development TLS trust anchor | + +## App listener fixture + +The opt-in `deploy/helm/openshell/ci/values-app-listener.yaml` enables the +supervisor listener at port 9443 without discovery or a proxy. Provision the +operator TLS Secret and an ingress-client NetworkPolicy first. Follow +`e2e/kubernetes/app-listener/README.md` for direct in-cluster verification, +shared-member selection, and the HA follow-up checks. Keep the fixture disabled +in default profiles. Shared supervisors expose app ports on their Pod IPs +without a shared app Service. Select the member owning the sandbox; automatic +owner selection belongs to the later discovery API. diff --git a/Cargo.lock b/Cargo.lock index a36b1edd43..20d2782aa7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5095,6 +5095,7 @@ dependencies = [ name = "openshell-supervisor" version = "0.0.0" dependencies = [ + "axum", "bytes", "clap", "futures", @@ -5124,6 +5125,7 @@ dependencies = [ "prost-types", "rustix 1.1.4", "rustls", + "rustls-pemfile", "serde", "serde_json", "socket2", diff --git a/crates/openshell-core/src/app_listener.rs b/crates/openshell-core/src/app_listener.rs new file mode 100644 index 0000000000..6fe2d1e2b4 --- /dev/null +++ b/crates/openshell-core/src/app_listener.rs @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Driver-independent application listener configuration. + +use serde::{Deserialize, Serialize}; + +/// Explicitly enabled application ingress. Deployment details such as TLS +/// Secret names belong to the driver, not to this common configuration. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct AppListenerConfig { + pub enabled: bool, + pub port: u16, + pub domain: String, +} + +impl Default for AppListenerConfig { + fn default() -> Self { + Self { + enabled: false, + port: 8443, + domain: String::new(), + } + } +} + +impl AppListenerConfig { + pub fn validate(&self) -> Result<(), String> { + if !self.enabled { + return Ok(()); + } + if self.port < 1024 { + return Err("supervisor.app_listener.port must be at least 1024".into()); + } + if self.domain.len() > 253 + || !self.domain.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + }) + { + return Err("supervisor.app_listener.domain must be a lowercase DNS name".into()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_explicit_enablement_requires_a_valid_endpoint() { + assert!(!AppListenerConfig::default().enabled); + assert!(AppListenerConfig::default().validate().is_ok()); + let mut config = AppListenerConfig { + enabled: true, + ..Default::default() + }; + assert!(config.validate().is_err()); + config.domain = "services.example.com".into(); + assert!(config.validate().is_ok()); + config.port = 0; + assert!(config.validate().is_err()); + } +} diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 27e5b80dd9..7305d667e5 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -332,6 +332,7 @@ pub struct Config { /// Browser-facing sandbox service routing configuration. pub service_routing: ServiceRoutingConfig, + pub supervisor_app_listener: crate::app_listener::AppListenerConfig, } /// Browser-facing sandbox service routing configuration. @@ -1015,6 +1016,7 @@ impl Config { grpc_rate_limit_requests: None, grpc_rate_limit_window_secs: None, service_routing: ServiceRoutingConfig::default(), + supervisor_app_listener: crate::app_listener::AppListenerConfig::default(), } } diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 63dabaf678..763d1d4c00 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -1863,6 +1863,7 @@ pub struct CachedOpenShellClient { /// Settings poll result returned by [`CachedOpenShellClient::poll_settings`]. #[derive(Clone, Debug)] pub struct SettingsPollResult { + pub service_routes: Vec, pub configuration_instance_id: String, pub configuration_admitted: bool, pub configuration_error: String, @@ -1890,6 +1891,7 @@ pub struct SettingsPollResult { fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> SettingsPollResult { SettingsPollResult { + service_routes: inner.service_routes, configuration_instance_id: inner.configuration_instance_id, configuration_admitted: inner.configuration_admitted, configuration_error: inner.configuration_error, @@ -1919,6 +1921,7 @@ fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> Settin impl From for SettingsPollResult { fn from(snapshot: crate::proto::SandboxConfigSnapshot) -> Self { settings_poll_result(crate::proto::GetSandboxConfigResponse { + service_routes: snapshot.service_routes, policy: snapshot.policy, version: snapshot.version, policy_hash: snapshot.policy_hash, diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 0588a9974d..f1fb416222 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -10,6 +10,7 @@ //! - Build version metadata pub mod activity; +pub mod app_listener; pub mod auth; pub mod config; pub mod container_paths; diff --git a/crates/openshell-driver-kubernetes/src/app_listener.rs b/crates/openshell-driver-kubernetes/src/app_listener.rs new file mode 100644 index 0000000000..738296f9a7 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/app_listener.rs @@ -0,0 +1,325 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reachable supervisor app endpoints. External ingress routes are owned by +//! ingress integrations, not by this compute driver. + +use crate::config::KubernetesAppListenerConfig; +use crate::driver::KubernetesDriverError; +use k8s_openapi::api::core::v1::{ + ContainerPort, EnvVar, Pod, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, + VolumeMount, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{ObjectMeta, OwnerReference}; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use kube::api::{DeleteParams, Patch, PatchParams, PostParams, Preconditions}; +use kube::{Api, Client}; +use std::collections::BTreeMap; + +const TLS_PATH: &str = "/etc/openshell/app-tls"; + +pub fn configure_supervisor_pod(mut pod: Pod, config: &KubernetesAppListenerConfig) -> Pod { + if !config.common.enabled { + return pod; + } + let spec = pod.spec.as_mut().expect("supervisor Pod spec"); + let container = spec + .containers + .iter_mut() + .find(|c| c.name == "supervisor") + .expect("supervisor container"); + let env = container.env.get_or_insert_default(); + for (name, value) in [ + ("OPENSHELL_APP_LISTENER_ENABLED", "true".to_string()), + ( + "OPENSHELL_APP_LISTEN_ADDRESS", + format!("0.0.0.0:{}", config.common.port), + ), + ("OPENSHELL_APP_SERVICE_DOMAIN", config.common.domain.clone()), + ("OPENSHELL_APP_TLS_CERT", format!("{TLS_PATH}/tls.crt")), + ("OPENSHELL_APP_TLS_KEY", format!("{TLS_PATH}/tls.key")), + ] { + env.push(EnvVar { + name: name.into(), + value: Some(value), + ..Default::default() + }); + } + container.ports.get_or_insert_default().push(ContainerPort { + name: Some("app".into()), + container_port: i32::from(config.common.port), + protocol: Some("TCP".into()), + ..Default::default() + }); + container + .volume_mounts + .get_or_insert_default() + .push(VolumeMount { + name: "app-tls".into(), + mount_path: TLS_PATH.into(), + read_only: Some(true), + ..Default::default() + }); + spec.volumes.get_or_insert_default().push(Volume { + name: "app-tls".into(), + secret: Some(SecretVolumeSource { + secret_name: Some(config.tls_secret_name.clone()), + ..Default::default() + }), + ..Default::default() + }); + pod +} + +pub fn app_service(namespace: &str, sandbox_id: &str, port: u16, owner: OwnerReference) -> Service { + Service { + metadata: ObjectMeta { + name: Some(format!("os-app-{sandbox_id}")), + namespace: Some(namespace.into()), + owner_references: Some(vec![owner]), + labels: Some(BTreeMap::from([ + ("openshell.ai/managed-by".into(), "openshell".into()), + ("openshell.ai/sandbox-id".into(), sandbox_id.into()), + ("openshell.ai/component".into(), "app-service".into()), + ])), + ..Default::default() + }, + spec: Some(ServiceSpec { + cluster_ip: Some("None".into()), + selector: Some(BTreeMap::from([ + ("openshell.ai/sandbox-id".into(), sandbox_id.into()), + ("openshell.ai/component".into(), "supervisor".into()), + ])), + ports: Some(vec![ServicePort { + name: Some("app".into()), + port: i32::from(port), + target_port: Some(IntOrString::Int(i32::from(port))), + protocol: Some("TCP".into()), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + } +} + +pub(crate) async fn ensure_service( + client: Client, + namespace: &str, + sandbox_id: &str, + config: &KubernetesAppListenerConfig, + owner: OwnerReference, +) -> Result<(), KubernetesDriverError> { + if !config.create_service { + return Ok(()); + } + let api: Api = Api::namespaced(client, namespace); + let expected = app_service(namespace, sandbox_id, config.common.port, owner.clone()); + let name = expected.metadata.name.as_deref().expect("app Service name"); + if !config.common.enabled { + if let Some(current) = api + .get_opt(name) + .await + .map_err(KubernetesDriverError::from_kube)? + && current + .metadata + .owner_references + .as_ref() + .is_some_and(|owners| owners.iter().any(|o| same_owner(o, &owner))) + { + let params = DeleteParams { + preconditions: Some(Preconditions { + uid: current.metadata.uid, + resource_version: current.metadata.resource_version, + }), + ..Default::default() + }; + match api.delete(name, ¶ms).await { + Ok(_) => {} + Err(kube::Error::Api(error)) if error.code == 404 => {} + Err(error) => return Err(KubernetesDriverError::from_kube(error)), + } + } + return Ok(()); + } + match api.create(&PostParams::default(), &expected).await { + Ok(_) => Ok(()), + Err(kube::Error::Api(error)) if error.code == 409 => { + let current = api + .get(name) + .await + .map_err(KubernetesDriverError::from_kube)?; + if !current + .metadata + .owner_references + .as_ref() + .is_some_and(|owners| owners.iter().any(|o| same_owner(o, &owner))) + { + return Err(KubernetesDriverError::Precondition(format!( + "app Service {name} is not owned by this sandbox" + ))); + } + api.patch( + name, + &PatchParams::default(), + &Patch::Merge(serde_json::json!({ + "metadata": {"resourceVersion": current.metadata.resource_version}, + "spec": expected.spec, + })), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + Ok(()) + } + Err(error) => Err(KubernetesDriverError::from_kube(error)), + } +} + +fn same_owner(current: &OwnerReference, expected: &OwnerReference) -> bool { + // Runtime companions deliberately use non-controller references: the + // Agent Sandbox controller owns the workload, while this driver owns its + // companion resources. Preserve that relationship on repeated reconcile. + current.uid == expected.uid + && current.name == expected.name + && current.kind == expected.kind + && current.api_version == expected.api_version + && current.controller.unwrap_or(false) == expected.controller.unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn reconciles_non_controller_owned_service_without_adopting_foreign_service() { + use http_body_util::BodyExt as _; + use std::sync::{Arc, Mutex}; + + for enabled in [true, false] { + for owned in [true, false] { + let owner = OwnerReference { + api_version: "agents.x-k8s.io/v1beta1".into(), + kind: "Sandbox".into(), + name: "default--demo".into(), + uid: "sandbox-uid".into(), + controller: Some(false), + block_owner_deletion: Some(false), + }; + let mut current = app_service("workspace", "sandbox-a", 9443, owner.clone()); + current.metadata.uid = Some("service-uid".into()); + current.metadata.resource_version = Some("42".into()); + if !owned { + current.metadata.owner_references.as_mut().unwrap()[0].uid = "foreign".into(); + } + let calls = Arc::new(Mutex::new(Vec::new())); + let recorded = calls.clone(); + let service = tower::service_fn( + move |request: http::Request| { + let current = current.clone(); + let recorded = recorded.clone(); + async move { + let method = request.method().to_string(); + recorded.lock().unwrap().push(method.clone()); + let (status, body) = match method.as_str() { + "POST" => ( + 409, + serde_json::json!({ + "kind": "Status", "apiVersion": "v1", "status": "Failure", + "reason": "AlreadyExists", "message": "exists", "code": 409, + }), + ), + "GET" | "PATCH" => (200, serde_json::to_value(¤t).unwrap()), + "DELETE" => { + let bytes = + request.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = + serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["preconditions"]["uid"], "service-uid"); + assert_eq!(body["preconditions"]["resourceVersion"], "42"); + ( + 200, + serde_json::json!({"kind": "Status", "apiVersion": "v1", "status": "Success"}), + ) + } + _ => panic!("unexpected method {method}"), + }; + Ok::<_, std::convert::Infallible>( + http::Response::builder() + .status(status) + .body(kube::client::Body::from(body.to_string().into_bytes())) + .unwrap(), + ) + } + }, + ); + let config = KubernetesAppListenerConfig { + create_service: true, + common: openshell_core::app_listener::AppListenerConfig { + enabled, + port: 9443, + domain: "services.example.com".into(), + }, + ..Default::default() + }; + let result = ensure_service( + Client::new(service, "workspace"), + "workspace", + "sandbox-a", + &config, + owner, + ) + .await; + assert_eq!(result.is_ok(), !enabled || owned); + let expected = match (enabled, owned) { + (true, true) => vec!["POST", "GET", "PATCH"], + (true, false) => vec!["POST", "GET"], + (false, true) => vec!["GET", "DELETE"], + (false, false) => vec!["GET"], + }; + assert_eq!(*calls.lock().unwrap(), expected); + } + } + } + + #[test] + fn disabled_listener_does_not_add_ports_or_private_key_mounts() { + let pod = Pod { + spec: Some(k8s_openapi::api::core::v1::PodSpec { + containers: vec![k8s_openapi::api::core::v1::Container { + name: "supervisor".into(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + let config = KubernetesAppListenerConfig { + tls_secret_name: "app-tls".into(), + create_service: true, + ..Default::default() + }; + assert_eq!(configure_supervisor_pod(pod.clone(), &config), pod); + } + + #[test] + fn app_endpoint_is_headless_and_selects_only_its_supervisor() { + let owner = OwnerReference { + api_version: "v1".into(), + kind: "Sandbox".into(), + name: "demo".into(), + uid: "uid-a".into(), + controller: Some(true), + ..Default::default() + }; + let service = app_service("workspace", "sandbox-a", 9443, owner.clone()); + assert_eq!(service.metadata.owner_references, Some(vec![owner])); + let spec = service.spec.unwrap(); + assert_eq!(spec.cluster_ip.as_deref(), Some("None")); + assert_eq!( + spec.selector.unwrap()["openshell.ai/component"], + "supervisor" + ); + assert_eq!(spec.ports.unwrap()[0].port, 9443); + assert_ne!(spec.publish_not_ready_addresses, Some(true)); + } +} diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a983459477..930667af15 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -118,6 +118,7 @@ impl FromStr for KubernetesControlConnection { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesSandboxRuntimeConfig { + pub app_listener: KubernetesAppListenerConfig, /// TCP port of the Sandbox Protocol listener: the workload boundary for /// supervisor-initiated sandboxes, the supervisor for runtime-initiated ones. pub boundary_port: u16, @@ -127,12 +128,33 @@ impl Default for KubernetesSandboxRuntimeConfig { fn default() -> Self { Self { boundary_port: 5500, + app_listener: KubernetesAppListenerConfig::default(), } } } +/// Driver-owned TLS and Service deployment inputs. Common listener settings +/// are injected by the gateway so there is only one port/domain source. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesAppListenerConfig { + pub tls_secret_name: String, + pub create_service: bool, + #[serde(skip)] + pub common: openshell_core::app_listener::AppListenerConfig, +} + impl KubernetesSandboxRuntimeConfig { pub fn validate(&self) -> Result<(), String> { + self.app_listener.common.validate()?; + if self.app_listener.common.enabled { + if !is_dns_subdomain(&self.app_listener.tls_secret_name) { + return Err("sandbox_runtime.app_listener.tls_secret_name is required when the app listener is enabled".into()); + } + if [self.boundary_port, 5501].contains(&self.app_listener.common.port) { + return Err("supervisor.app_listener.port conflicts with a supervisor runtime or health port".into()); + } + } if self.boundary_port < 1024 { return Err("sandbox_runtime.boundary_port must be at least 1024".to_string()); } @@ -557,6 +579,11 @@ impl KubernetesComputeConfig { "supervisor_topology=gateway requires shared_supervisor".to_string() })?; shared.validate()?; + if self.sandbox_runtime.app_listener.common.enabled + && self.sandbox_runtime.app_listener.common.port == shared.runtime_port + { + return Err("supervisor.app_listener.port conflicts with shared_supervisor.runtime_port".into()); + } if self.provider_spiffe_enabled() { return Err("supervisor_topology=gateway does not support provider_spiffe_workload_api_socket_path; a shared supervisor has no per-sandbox workload identity".into()); } @@ -1118,6 +1145,26 @@ mod tests { use super::*; use std::collections::BTreeMap as HashMap; + #[test] + fn app_endpoint_configuration_has_one_port_source_and_explicit_enablement() { + let mut config = KubernetesSandboxRuntimeConfig::default(); + config.app_listener.tls_secret_name = "app-tls".into(); + config.app_listener.create_service = true; + assert!(!config.app_listener.common.enabled); + assert!(config.validate().is_ok()); + config.app_listener.common = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + domain: "services.example.com".into(), + }; + assert!(config.validate().is_ok()); + config.app_listener.common.port = config.boundary_port; + assert!(config.validate().is_err()); + config.app_listener.common.port = 9443; + config.app_listener.tls_secret_name.clear(); + assert!(config.validate().is_err()); + } + #[test] fn gateway_supervisor_topology_requires_its_connection_and_refuses_unsupported_inputs() { let shared = || SharedSupervisorConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 65be627dc9..3b07dfe098 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -486,7 +486,7 @@ pub enum KubernetesDriverError { } impl KubernetesDriverError { - fn from_kube(err: KubeError) -> Self { + pub(crate) fn from_kube(err: KubeError) -> Self { match err { KubeError::Api(api) if api.code == 409 && api.reason == "AlreadyExists" => { Self::AlreadyExists @@ -3269,6 +3269,16 @@ impl KubernetesComputeDriver { owner: OwnerReference, ) -> Result<(), KubernetesDriverError> { let services: Api = Api::namespaced(self.client.clone(), namespace); + if self.shared_supervisor.is_none() { + crate::app_listener::ensure_service( + self.client.clone(), + namespace, + sandbox_id, + &self.config.sandbox_runtime.app_listener, + owner.clone(), + ) + .await?; + } let expected = boundary_service( namespace, names, @@ -3362,44 +3372,47 @@ impl KubernetesComputeDriver { let pods: Api = Api::namespaced(self.client.clone(), namespace); pods.create( &PostParams::default(), - &supervisor_pod( - namespace, - names, - sandbox_id, - sandbox_name, - &self.config.gateway_id, - &self.config.supervisor_image, - self.config.supervisor_image_pull_policy, - &self.config.service_account_name, - agent_uid, - agent_gid, - &self.generation_image_pull_secret_names(names), - &self.config.grpc_endpoint, - self.config.supervisor_client_tls(), - main_process_spec, - log_level, - self.config.effective_sa_token_ttl_secs(), - self.config.https_proxy.as_deref(), - self.config.no_proxy.as_deref(), - self.config - .proxy_auth_secret_name - .as_deref() - .zip(self.config.proxy_auth_secret_key.as_deref()), - self.config.proxy_auth_allow_insecure == Some(true), - self.config.proxy_connect_by_hostname == Some(true), - upstream_proxy_ca_bundle_staged, - self.config.provider_spiffe_enabled().then_some( + &crate::app_listener::configure_supervisor_pod( + supervisor_pod( + namespace, + names, + sandbox_id, + sandbox_name, + &self.config.gateway_id, + &self.config.supervisor_image, + self.config.supervisor_image_pull_policy, + &self.config.service_account_name, + agent_uid, + agent_gid, + &self.generation_image_pull_secret_names(names), + &self.config.grpc_endpoint, + self.config.supervisor_client_tls(), + main_process_spec, + log_level, + self.config.effective_sa_token_ttl_secs(), + self.config.https_proxy.as_deref(), + self.config.no_proxy.as_deref(), self.config - .provider_spiffe_workload_api_socket_path - .as_str(), - ), - network_additional_ca - .as_ref() - .map(|generation| (generation.name.as_str(), generation.digest.as_str())), - owner, - &self.supervisor_otlp_environment(), - ) - .map_err(KubernetesDriverError::Message)?, + .proxy_auth_secret_name + .as_deref() + .zip(self.config.proxy_auth_secret_key.as_deref()), + self.config.proxy_auth_allow_insecure == Some(true), + self.config.proxy_connect_by_hostname == Some(true), + upstream_proxy_ca_bundle_staged, + self.config.provider_spiffe_enabled().then_some( + self.config + .provider_spiffe_workload_api_socket_path + .as_str(), + ), + network_additional_ca + .as_ref() + .map(|generation| (generation.name.as_str(), generation.digest.as_str())), + owner, + &self.supervisor_otlp_environment(), + ) + .map_err(KubernetesDriverError::Message)?, + &self.config.sandbox_runtime.app_listener, + ), ) .await .map_err(KubernetesDriverError::from_kube)? diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 4d21fbff51..f3a1e419ec 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +pub mod app_listener; pub mod config; pub mod driver; pub mod grpc; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 7803d1561a..888cd698b3 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -292,6 +292,8 @@ async fn main() -> Result<()> { .unwrap_or_else(openshell_core::config::default_supervisor_image), supervisor_image_pull_policy: args.supervisor_image_pull_policy, sandbox_runtime: KubernetesSandboxRuntimeConfig { + app_listener: + openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(), boundary_port: args.sandbox_runtime_boundary_port, }, https_proxy: args.https_proxy, diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 111b6ba7ec..3ae1d837fc 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -1371,84 +1371,105 @@ mod tests { #[test] fn destination_trust_is_mounted_only_into_the_dedicated_supervisor_pod() { - let names = SandboxRuntimeNames::new("pair"); - let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; - let pod = supervisor_pod( - "sandbox", - &names, - "pair", - "demo", - "gateway", - "supervisor:latest", - None, - "sandbox-sa", - 1000, - 1000, - &[], - "https://gateway:8080", - SupervisorClientTls::Disabled, - "{}", - "info", - 600, - None, - None, - None, - false, - false, - false, - None, - Some(("openshell-additional-ca-gateway-generation", digest)), - owner(), - &[], - ) - .expect("render supervisor Pod with destination trust"); - let spec = pod.spec.expect("Pod spec"); - let container = &spec.containers[0]; - assert!(container.command.as_deref().is_some_and(|command| { - command - .windows(2) - .any(|args| args == ["--additional-ca-bundle", ADDITIONAL_CA_MOUNT_PATH]) - })); - assert_eq!( - container - .volume_mounts + for enabled in [false, true] { + let names = SandboxRuntimeNames::new("pair"); + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let pod = supervisor_pod( + "sandbox", + &names, + "pair", + "demo", + "gateway", + "supervisor:latest", + None, + "sandbox-sa", + 1000, + 1000, + &[], + "https://gateway:8080", + SupervisorClientTls::Disabled, + "{}", + "info", + 600, + None, + None, + None, + false, + false, + false, + None, + Some(("openshell-additional-ca-gateway-generation", digest)), + owner(), + &[], + ) + .expect("render supervisor Pod with destination trust"); + let pod = crate::app_listener::configure_supervisor_pod( + pod, + &crate::config::KubernetesAppListenerConfig { + tls_secret_name: "app-tls-secret".into(), + common: openshell_core::app_listener::AppListenerConfig { + enabled, + ..Default::default() + }, + ..Default::default() + }, + ); + let spec = pod.spec.expect("Pod spec"); + assert_eq!( + spec.volumes + .as_ref() + .unwrap() + .iter() + .any(|v| v.name == "app-tls"), + enabled + ); + let container = &spec.containers[0]; + assert!(container.command.as_deref().is_some_and(|command| { + command + .windows(2) + .any(|args| args == ["--additional-ca-bundle", ADDITIONAL_CA_MOUNT_PATH]) + })); + assert_eq!( + container + .volume_mounts + .as_deref() + .unwrap_or_default() + .iter() + .filter(|mount| mount.name == ADDITIONAL_CA_VOLUME_NAME) + .collect::>() + .as_slice(), + &[&VolumeMount { + name: ADDITIONAL_CA_VOLUME_NAME.to_string(), + mount_path: ADDITIONAL_CA_MOUNT_PATH.to_string(), + read_only: Some(true), + sub_path: Some(ADDITIONAL_CA_KEY.to_string()), + ..Default::default() + }] + ); + let volume = spec + .volumes .as_deref() .unwrap_or_default() .iter() - .filter(|mount| mount.name == ADDITIONAL_CA_VOLUME_NAME) - .collect::>() - .as_slice(), - &[&VolumeMount { - name: ADDITIONAL_CA_VOLUME_NAME.to_string(), - mount_path: ADDITIONAL_CA_MOUNT_PATH.to_string(), - read_only: Some(true), - sub_path: Some(ADDITIONAL_CA_KEY.to_string()), - ..Default::default() - }] - ); - let volume = spec - .volumes - .as_deref() - .unwrap_or_default() - .iter() - .find(|volume| volume.name == ADDITIONAL_CA_VOLUME_NAME) - .expect("destination trust ConfigMap volume"); - assert_eq!( - volume - .config_map - .as_ref() - .map(|source| source.name.as_str()), - Some("openshell-additional-ca-gateway-generation") - ); - assert_eq!( - volume - .config_map - .as_ref() - .and_then(|source| source.items.as_deref()) - .and_then(|items| items.first()) - .map(|item| (item.key.as_str(), item.path.as_str())), - Some((ADDITIONAL_CA_KEY, ADDITIONAL_CA_KEY)) - ); + .find(|volume| volume.name == ADDITIONAL_CA_VOLUME_NAME) + .expect("destination trust ConfigMap volume"); + assert_eq!( + volume + .config_map + .as_ref() + .map(|source| source.name.as_str()), + Some("openshell-additional-ca-gateway-generation") + ); + assert_eq!( + volume + .config_map + .as_ref() + .and_then(|source| source.items.as_deref()) + .and_then(|items| items.first()) + .map(|item| (item.key.as_str(), item.path.as_str())), + Some((ADDITIONAL_CA_KEY, ADDITIONAL_CA_KEY)) + ); + } } #[test] diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index a6a50774b6..45abaf2607 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -343,6 +343,7 @@ fn kubernetes_config( ) -> openshell_core::Result { let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = context.driver_config()?; + config.sandbox_runtime.app_listener.common = context.app_listener_config(); config.sandbox_runtime_image = prefer_environment( config.sandbox_runtime_image, std::env::var(openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV).ok(), diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 9b16b53914..f4465ced1d 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -594,6 +594,14 @@ fn prepare_server_config_with_drivers( .service_routing .validate(&config.name) .map_err(|error| miette::miette!("invalid service routing configuration: {error}"))?; + config.supervisor_app_listener = file + .as_ref() + .map(|f| f.openshell.supervisor.app_listener.clone()) + .unwrap_or_default(); + config + .supervisor_app_listener + .validate() + .map_err(|e| miette::miette!(e))?; if let Some(sources) = file .as_ref() .and_then(|file| file.openshell.gateway.provider_profile_sources.clone()) diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 5c4eedf9ac..4601666eac 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -508,6 +508,8 @@ impl TryFrom for OtlpConfig { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SupervisorFileSection { + #[serde(default)] + pub app_listener: openshell_core::app_listener::AppListenerConfig, /// Operator-owned external isolation routes, reachable by supervisors. #[serde(default)] pub isolation_backends: Vec, diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index b09a6df792..5b97da32cd 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -3053,7 +3053,7 @@ pub async fn build_sandbox_config_snapshot_from_inputs( let settings = merge_effective_settings(global_settings, &sandbox_settings)?; let supervisor_middleware_services = state.middleware_registry.required_services(policy.as_ref()); - let config_revision = compute_config_revision_with_validation_mode( + let mut config_revision = compute_config_revision_with_validation_mode( policy.as_ref(), &settings, policy_source, @@ -3078,7 +3078,22 @@ pub async fn build_sandbox_config_snapshot_from_inputs( .filter(|_| matches!(policy_source, PolicySource::Global)), )?; + let service_routes = super::service::configured_app_routes(state, sandbox).await?; + // Include the allowlist in delivery identity so a service edit is not + // coalesced away as a duplicate policy/settings snapshot. + if !service_routes.is_empty() { + let mut bytes = config_revision.to_le_bytes().to_vec(); + for route in &service_routes { + route + .encode_length_delimited(&mut bytes) + .expect("encode service route"); + } + let hash = openshell_crypto::sha256(&bytes) + .map_err(|_| Status::internal("service configuration fingerprint failed"))?; + config_revision = u64::from_le_bytes(hash[..8].try_into().expect("hash prefix")); + } Ok(SandboxConfigSnapshot { + service_routes, configuration_instance_id: sandbox .status .as_ref() @@ -3113,6 +3128,7 @@ pub async fn build_sandbox_config_snapshot_from_inputs( fn sandbox_config_response(snapshot: SandboxConfigSnapshot) -> GetSandboxConfigResponse { GetSandboxConfigResponse { + service_routes: snapshot.service_routes, policy: snapshot.policy, version: snapshot.version, policy_hash: snapshot.policy_hash, diff --git a/crates/openshell-server/src/grpc/service.rs b/crates/openshell-server/src/grpc/service.rs index bf514da414..a47791e8d8 100644 --- a/crates/openshell-server/src/grpc/service.rs +++ b/crates/openshell-server/src/grpc/service.rs @@ -186,6 +186,9 @@ pub(super) async fn expose_service_endpoint( } }; + if state.config.supervisor_app_listener.enabled { + crate::config_delivery::publish_sandbox(state, sandbox.object_id()); + } let mut endpoint = endpoint; if let Some(ref mut meta) = endpoint.metadata { meta.resource_version = result.resource_version; @@ -360,6 +363,9 @@ pub(super) async fn handle_delete_service( state .service_route_index .remove(workspace, sandbox_name, &req.name); + if state.config.supervisor_app_listener.enabled { + crate::config_delivery::publish_sandbox(state, &endpoint.sandbox_id); + } service_routing::emit_service_endpoint_delete_event(&endpoint); } @@ -368,6 +374,56 @@ pub(super) async fn handle_delete_service( })) } +/// Complete bounded allowlist for the supervisor configuration. Disabled +/// ingress leaves existing gateway-only configurations and fingerprints alone. +pub(super) async fn configured_app_routes( + state: &ServerState, + sandbox: &Sandbox, +) -> Result, Status> { + if !state.config.supervisor_app_listener.enabled { + return Ok(Vec::new()); + } + let selector = format!("sandbox={}", sandbox.object_name()); + let page = state + .store + .list_message_page::( + ObjectListQuery::WorkspaceSelector { + workspace: sandbox.object_workspace(), + label_selector: &selector, + }, + None, + 1024, + ) + .await + .map_err(|_| Status::unavailable("service declarations unavailable"))?; + if page.next_cursor.is_some() { + return Err(Status::resource_exhausted( + "too many application service declarations", + )); + } + let mut service_routes: Vec<_> = page + .messages + .into_iter() + .filter(|ep| { + ep.sandbox_id == sandbox.object_id() + && ep.domain + && ep + .metadata + .as_ref() + .is_none_or(|m| m.deletion_time.is_none()) + }) + .map(|ep| openshell_core::proto::SandboxServiceRoute { + endpoint_id: ep.object_id().to_string(), + name: ep.name.clone(), + target_port: ep.target_port, + authorization_mode: ep.authorization_mode, + resource_version: ep.metadata.as_ref().map_or(0, |m| m.resource_version), + }) + .collect(); + service_routes.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(service_routes) +} + async fn get_service_endpoint( state: &Arc, workspace: &str, @@ -485,6 +541,121 @@ mod tests { validate_endpoint_name("service", "web-api", 28).unwrap(); } + #[tokio::test] + async fn service_declarations_update_configuration_identity_and_stay_sandbox_scoped() { + let mut state = test_server_state().await; + Arc::get_mut(&mut state) + .expect("unshared test state") + .config + .supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + domain: "services.example.com".into(), + }; + seed_sandbox(&state, "first").await; + seed_sandbox(&state, "second").await; + let sandbox = state + .store + .get_message::("sandbox-first") + .await + .unwrap() + .unwrap(); + let before = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + for name in ["first", "second"] { + handle_expose_service( + &state, + authed_request(ExposeServiceRequest { + sandbox: name.into(), + name: "web".into(), + target_port: 8080, + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + } + let installed = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert_eq!(installed.service_routes.len(), 1); + assert_eq!(installed.service_routes[0].target_port, 8080); + assert_ne!(installed.config_revision, before.config_revision); + + handle_expose_service( + &state, + authed_request(ExposeServiceRequest { + sandbox: "first".into(), + name: "web".into(), + target_port: 9090, + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + let edited = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert_eq!(edited.service_routes[0].target_port, 9090); + assert_eq!( + edited.service_routes[0].endpoint_id, + installed.service_routes[0].endpoint_id + ); + assert_ne!(edited.config_revision, installed.config_revision); + + handle_delete_service( + &state, + authed_request(DeleteServiceRequest { + sandbox: "first".into(), + name: "web".into(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + let removed = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert!(removed.service_routes.is_empty()); + assert_eq!(removed.config_revision, before.config_revision); + } + + #[tokio::test] + async fn gateway_only_exposure_does_not_change_supervisor_configuration() { + let state = test_server_state().await; + seed_sandbox(&state, "gateway-only").await; + let sandbox = state + .store + .get_message::("sandbox-gateway-only") + .await + .unwrap() + .unwrap(); + let before = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + handle_expose_service( + &state, + authed_request(ExposeServiceRequest { + sandbox: "gateway-only".into(), + name: "web".into(), + target_port: 8080, + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + let after = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert!(after.service_routes.is_empty()); + assert_eq!(after.config_revision, before.config_revision); + } + #[test] fn validates_empty_optional_service_name() { validate_optional_endpoint_name("service", "", 28).unwrap(); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 06bbf42736..954302c695 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1697,6 +1697,14 @@ pub struct ComputeDriverConfigContext<'a> { } impl ComputeDriverConfigContext<'_> { + #[must_use] + pub fn app_listener_config(&self) -> openshell_core::app_listener::AppListenerConfig { + self.driver_startup + .file + .map(|file| file.openshell.supervisor.app_listener.clone()) + .unwrap_or_default() + } + #[must_use] pub fn driver_name(&self) -> &str { self.driver_name diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 49d158637a..76cb2fc701 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -164,7 +164,7 @@ mod tests { // SandboxPhase enum. Stored rows never carried those values, so prior // payloads decode unchanged. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "f24c9b77a85f59ae9ab60f783d45654d78bf566fcf260f9131f1aba39fa4b59b"; + "73e29104ad78d7682b859edfc15ebe2c7319f53d72948cab4dd2c99daed8d11c"; const DURABLE_SCHEMA_SHA256: &str = "ffb20595e36032a9476d1c6956e2dcdc814f85286fc4735751e5ec6f499ea9ed"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = @@ -702,7 +702,7 @@ mod tests { overlap_hash.as_str(), ), ( - (330, 31), + (331, 31), (95, 22), (83, 22), PUBLIC_RPC_SCHEMA_SHA256, diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index 645b813604..21940d9f60 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -29,7 +29,9 @@ openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-mid openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } openshell-supervisor-process = { path = "../openshell-supervisor-process" } +axum = { workspace = true } bytes = { workspace = true } +rustls-pemfile = { workspace = true } clap = { workspace = true } futures = { workspace = true } http = { workspace = true } @@ -70,10 +72,7 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } openshell-otel-test-support = { path = "../openshell-otel-test-support" } bytes = { workspace = true } -http = { workspace = true } http-body-util = { workspace = true } -hyper = { workspace = true } -hyper-util = { workspace = true } tonic-prost = { workspace = true } tokio = { workspace = true, features = ["test-util"] } temp-env = "0.3" diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 5956ef43f7..4082257b32 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -257,3 +257,36 @@ cannot export each instance's records with that instance's collector headers and identity. Its console logs and each instance's gateway log push remain, and the Kubernetes driver warns at startup when the gateway configures supervisor export. + +## Application listener + +The opt-in application TLS listener serves declared HTTP/1.1 and WebSocket +services through each sandbox instance's `BoundaryLoopbackConnector`. The host +binds one listener and dispatches accepted TLS streams by SNI to a bounded queue +on the owning instance's runtime. That runtime owns request handling, credentials, +policy state, limits, and OCSF context. Stopping an instance unregisters only its +routes and closes its streams; it does not stop another instance's ingress. + +Gateway configuration snapshots carry the declared-service allowlist. Successful +poll and push application installs that allowlist; rejected configuration +suspends it. Polling renews a 30-second freshness lease. In push mode, an accepted +live supervisor session renews already installed, unexpired routes even when no +configuration changes. Rejection or a lease lapse cannot revive an old stream. +Ending the configuration loop suspends routes. + +`OPENSHELL_APP_LISTENER_ENABLED=true` explicitly enables the process listener. +`OPENSHELL_APP_LISTEN_ADDRESS`, `OPENSHELL_APP_SERVICE_DOMAIN`, +`OPENSHELL_APP_TLS_CERT`, and `OPENSHELL_APP_TLS_KEY` are then required. Hosted +instances receive the process router explicitly and never read another +instance's configuration from process environment. TLS configuration uses +`openshell-crypto`; certificates are read at process startup. Restart the +supervisor to reload application certificates. + +This path uses application or trusted-edge authentication independently from +gateway authentication. It validates Host against SNI, preserves public +Host/Origin and application cookies, strips gateway/edge identity material, and +retains service-specific bearer passthrough. It has no discovery API or public +URL selection. Shared supervisors expose the app listener on each member's Pod +IP without a shared app Service. Clients must select the member owning their +sandbox. HA takeover may preserve the workload generation; existing ingress streams +are not migrated. Runtime ownership fencing must reject stale connectors. diff --git a/crates/openshell-supervisor/src/app_listener.rs b/crates/openshell-supervisor/src/app_listener.rs new file mode 100644 index 0000000000..ed8d8943cb --- /dev/null +++ b/crates/openshell-supervisor/src/app_listener.rs @@ -0,0 +1,1448 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Opt-in application ingress, separate from the gateway control transport. + +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use axum::body::Body; +use http::{Request, Response, StatusCode, header}; +use hyper::body::Incoming; +use hyper_util::rt::{TokioIo, TokioTimer}; +use miette::{IntoDiagnostic, Result}; +use openshell_core::grpc_client::SettingsPollResult; +use openshell_core::proto::SandboxServiceRoute; +use openshell_isolation_interface::contract::LoopbackTarget; +use openshell_ocsf::{ + ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DispositionId, + HttpActivityBuilder, HttpResponse, NetworkActivityBuilder, SeverityId, StatusId, ocsf_emit, +}; +use openshell_sandbox_backend::BoundaryLoopbackConnector; +use std::sync::Mutex; +use tokio::sync::{Semaphore, mpsc, watch}; +use tokio::task::JoinSet; +use tokio::time::Instant; +use tokio_rustls::TlsAcceptor; + +const FRESHNESS: Duration = Duration::from_secs(30); +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +pub struct Config { + address: SocketAddr, + domain: String, + certificate: PathBuf, + key: PathBuf, +} + +impl Config { + pub fn from_env() -> Result> { + Self::from_values(|name| std::env::var(name).ok()) + } + + fn from_values(read: impl Fn(&str) -> Option) -> Result> { + match read("OPENSHELL_APP_LISTENER_ENABLED").as_deref() { + None | Some("false") => return Ok(None), + Some("true") => {} + _ => { + return Err(miette::miette!( + "OPENSHELL_APP_LISTENER_ENABLED must be true or false" + )); + } + } + let address = read("OPENSHELL_APP_LISTEN_ADDRESS"); + let domain = read("OPENSHELL_APP_SERVICE_DOMAIN"); + let certificate = read("OPENSHELL_APP_TLS_CERT"); + let key = read("OPENSHELL_APP_TLS_KEY"); + let (Some(address), Some(domain), Some(certificate), Some(key)) = + (address, domain, certificate, key) + else { + return Err(miette::miette!( + "app listener requires address, domain, certificate, and key" + )); + }; + let address: SocketAddr = address.parse().into_diagnostic()?; + openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: address.port(), + domain: domain.clone(), + } + .validate() + .map_err(|error| miette::miette!(error))?; + Ok(Some(Self { + address, + domain, + certificate: certificate.into(), + key: key.into(), + })) + } + + pub fn bind(self) -> Result { + let certificates = rustls_pemfile::certs(&mut std::io::BufReader::new( + std::fs::File::open(self.certificate).into_diagnostic()?, + )) + .collect::>>() + .into_diagnostic()?; + let key = rustls_pemfile::private_key(&mut std::io::BufReader::new( + std::fs::File::open(self.key).into_diagnostic()?, + )) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("app TLS private key missing"))?; + let mut tls = openshell_crypto::tls::server_builder() + .with_no_client_auth() + .with_single_cert(certificates, key) + .into_diagnostic()?; + tls.alpn_protocols = vec![b"http/1.1".to_vec()]; + let socket = std::net::TcpListener::bind(self.address).into_diagnostic()?; + socket.set_nonblocking(true).into_diagnostic()?; + let listener = tokio::net::TcpListener::from_std(socket).into_diagnostic()?; + Ok(BoundListener { + listener, + acceptor: TlsAcceptor::from(Arc::new(tls)), + domain: self.domain, + }) + } +} + +fn valid_label(label: &str) -> bool { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') +} + +#[derive(Clone, Default)] +struct Snapshot { + routes: BTreeMap, + generations: BTreeMap, + next_generation: u64, + expires: Option, + workspace: String, + // Whole-sandbox suspension fence, independent of individual service edits. + epoch: u64, +} + +#[derive(Clone, Copy)] +struct RouteToken { + epoch: u64, + generation: u64, +} + +/// Shared by policy reconciliation and ingress. Only accepted configuration +/// refreshes this lease; route changes cannot restart or replace the workload. +#[derive(Clone)] +pub struct Routes { + registration: Option>, + sender: watch::Sender, + stopped: Arc, +} + +impl Default for Routes { + fn default() -> Self { + Self { + sender: watch::channel(Snapshot::default()).0, + stopped: Arc::new(AtomicBool::new(false)), + registration: None, + } + } +} + +pub(crate) struct ConfigurationGuard(Routes); +impl Drop for ConfigurationGuard { + fn drop(&mut self) { + self.0.suspend(); + } +} + +impl Routes { + pub(crate) fn configuration_guard(self) -> ConfigurationGuard { + ConfigurationGuard(self) + } + + fn valid(&self, route: &SandboxServiceRoute, token: RouteToken) -> bool { + let snapshot = self.sender.borrow(); + snapshot + .expires + .is_some_and(|expiry| expiry > Instant::now()) + && snapshot.epoch == token.epoch + && snapshot.generations.get(&route.name) == Some(&token.generation) + && snapshot.routes.get(&route.name) == Some(route) + } + pub(crate) fn install(&self, result: &SettingsPollResult) { + if self.stopped.load(Ordering::Acquire) { + return; + } + let mut routes = BTreeMap::new(); + if !result.configuration_admitted { + self.suspend(); + return; + } + if result.configuration_admitted + && valid_label(&result.workspace) + && !result.workspace.contains("--") + { + for route in &result.service_routes { + if route.endpoint_id.is_empty() + || route.target_port == 0 + || route.target_port > 65535 + || !matches!(route.authorization_mode, 0..=2) + || (!route.name.is_empty() + && (!valid_label(&route.name) || route.name.contains("--"))) + { + self.suspend(); + return; + } + if routes.insert(route.name.clone(), route.clone()).is_some() { + self.suspend(); + return; + } + } + } + if routes.len() > 1024 || !valid_label(&result.workspace) || result.workspace.contains("--") + { + self.suspend(); + return; + } + if let Some(registration) = &self.registration + && !registration.bind(&result.workspace) + { + self.suspend(); + return; + } + let changed = self.sender.borrow().routes != routes; + self.sender.send_modify(|snapshot| { + // Renewal after a lapse cannot revive a connection whose lease expired. + if snapshot + .expires + .is_some_and(|expiry| expiry <= Instant::now()) + { + snapshot.epoch = snapshot.epoch.wrapping_add(1); + } + let generations = routes + .iter() + .map(|(name, route)| { + let previous = (snapshot.routes.get(name) == Some(route)) + .then(|| snapshot.generations.get(name).copied()) + .flatten(); + let generation = previous.unwrap_or_else(|| { + snapshot.next_generation = snapshot + .next_generation + .checked_add(1) + .expect("app route generation exhausted"); + snapshot.next_generation + }); + (name.clone(), generation) + }) + .collect(); + if snapshot.workspace != result.workspace { + snapshot.epoch = snapshot.epoch.wrapping_add(1); + } + snapshot.workspace.clone_from(&result.workspace); + snapshot.routes = routes; + snapshot.generations = generations; + snapshot.expires = Some(Instant::now() + FRESHNESS); + }); + if changed { + ocsf_emit!( + ConfigStateChangeBuilder::new(crate::ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .message("App listener service routes updated") + .build() + ); + } + } + + /// Renew only already installed routes while a live push session supplies + /// freshness. A rejected configuration is suspended and cannot be revived. + pub(crate) fn renew(&self) { + if self.stopped.load(Ordering::Acquire) { + return; + } + self.sender.send_modify(|snapshot| { + if snapshot + .expires + .is_some_and(|expiry| expiry > Instant::now()) + { + snapshot.expires = Some(Instant::now() + FRESHNESS); + } + }); + } + + pub(crate) fn suspend(&self) { + self.sender.send_modify(|snapshot| { + snapshot.routes.clear(); + snapshot.generations.clear(); + snapshot.expires = None; + snapshot.epoch = snapshot.epoch.wrapping_add(1); + }); + } + + fn resolve( + &self, + sni: &str, + prefix: &str, + domain: &str, + ) -> Option<(SandboxServiceRoute, RouteToken)> { + let encoded = sni.strip_suffix(&format!(".{domain}"))?; + let service = if encoded == prefix { + "" + } else { + encoded.strip_prefix(&format!("{prefix}--"))? + }; + let snapshot = self.sender.borrow(); + if snapshot.expires? <= Instant::now() { + return None; + } + Some(( + snapshot.routes.get(service)?.clone(), + RouteToken { + epoch: snapshot.epoch, + generation: *snapshot.generations.get(service)?, + }, + )) + } + + async fn revoked(&self, route: &SandboxServiceRoute, token: RouteToken) { + let mut receiver = self.sender.subscribe(); + loop { + let expiry = { + let snapshot = receiver.borrow_and_update(); + if snapshot.epoch != token.epoch + || snapshot.generations.get(&route.name) != Some(&token.generation) + || snapshot.routes.get(&route.name) != Some(route) + { + return; + } + let Some(expiry) = snapshot.expires else { + return; + }; + expiry + }; + tokio::select! { + () = tokio::time::sleep_until(expiry) => return, + result = receiver.changed() => if result.is_err() { return; }, + } + } + } +} + +pub struct BoundListener { + listener: tokio::net::TcpListener, + acceptor: TlsAcceptor, + domain: String, +} + +/// Host-level dispatcher. It never shares a sandbox connector or policy +/// between instances: TLS streams are handed to the owning instance runtime. +#[derive(Clone, Default)] +pub struct Router { + domain: String, + registrations: Arc>>>, +} + +struct Accepted { + tls: tokio_rustls::server::TlsStream, + permit: tokio::sync::OwnedSemaphorePermit, +} + +struct Registration { + router: Router, + sandbox: String, + prefix: Mutex, + sender: mpsc::Sender, +} + +impl Registration { + fn bind(&self, workspace: &str) -> bool { + let next = format!("{workspace}--{}", self.sandbox); + let mut prefix = self + .prefix + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut entries = self + .router + .registrations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if entries + .get(&next) + .is_some_and(|sender| !sender.same_channel(&self.sender)) + { + return false; + } + if *prefix != next { + if entries + .get(&*prefix) + .is_some_and(|s| s.same_channel(&self.sender)) + { + entries.remove(&*prefix); + } + prefix.clone_from(&next); + } + entries.insert(next, self.sender.clone()); + true + } + + fn remove(&self) { + let prefix = self + .prefix + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut entries = self + .router + .registrations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if entries + .get(&*prefix) + .is_some_and(|s| s.same_channel(&self.sender)) + { + entries.remove(&*prefix); + } + } +} + +/// Process-owned listener. Dropping it stops acceptance without affecting +/// another host process. Individual instance lifetimes are independent. +pub struct HostHandle { + task: tokio::task::JoinHandle<()>, + router: Router, +} + +impl HostHandle { + #[must_use] + pub fn router(&self) -> Router { + self.router.clone() + } +} + +impl Drop for HostHandle { + fn drop(&mut self) { + self.task.abort(); + } +} + +pub struct InstanceListener { + pub(crate) routes: Routes, + receiver: mpsc::Receiver, + domain: String, +} + +pub struct Handle { + task: tokio::task::JoinHandle<()>, + routes: Routes, +} + +impl Drop for Handle { + fn drop(&mut self) { + self.routes.stopped.store(true, Ordering::Release); + self.routes.suspend(); + if let Some(registration) = &self.routes.registration { + registration.remove(); + } + self.task.abort(); + } +} + +impl Router { + pub(crate) fn instance(&self, sandbox: &str) -> Result { + if !valid_label(sandbox) || sandbox.contains("--") { + return Err(miette::miette!( + "app listener requires a canonical sandbox name" + )); + } + let (sender, receiver) = mpsc::channel(16); + let registration = Arc::new(Registration { + router: self.clone(), + sandbox: sandbox.to_string(), + prefix: Mutex::new(String::new()), + sender, + }); + Ok(InstanceListener { + routes: Routes { + registration: Some(registration), + ..Default::default() + }, + receiver, + domain: self.domain.clone(), + }) + } +} + +impl BoundListener { + #[must_use] + pub fn start(self) -> HostHandle { + let router = Router { + domain: self.domain.clone(), + ..Default::default() + }; + let dispatch = router.clone(); + let task = tokio::spawn(async move { + let permits = Arc::new(Semaphore::new(1024)); + let mut connections = JoinSet::new(); + loop { + tokio::select! { + Some(_) = connections.join_next(), if !connections.is_empty() => {}, + accepted = self.listener.accept() => { + let Ok((stream, _)) = accepted else { break; }; + let Ok(permit) = permits.clone().try_acquire_owned() else { continue; }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + let acceptor = self.acceptor.clone(); + let domain = self.domain.clone(); + let dispatch = dispatch.clone(); + connections.spawn(async move { + let Ok(Ok(tls)) = tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await else { return; }; + let Some(sni) = tls.get_ref().1.server_name() else { return; }; + let Some(label) = sni.strip_suffix(&format!(".{domain}")) else { return; }; + let mut segments = label.split("--"); + let (Some(workspace), Some(sandbox)) = (segments.next(), segments.next()) else { return; }; + let prefix = format!("{workspace}--{sandbox}"); + let sender = dispatch.registrations.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&prefix).cloned(); + if let Some(sender) = sender { let _ = sender.try_send(Accepted { tls, permit }); } + }); + } + } + } + }); + HostHandle { task, router } + } +} + +struct Permits { + _host: tokio::sync::OwnedSemaphorePermit, + _instance: tokio::sync::OwnedSemaphorePermit, +} + +impl InstanceListener { + /// Called on the sandbox instance runtime, preserving its OCSF/logging + /// identity even when multiple sandboxes share one listener process. + pub(crate) fn start(mut self, connector: Arc) -> Handle { + let retained_routes = self.routes.clone(); + let task = tokio::spawn(async move { + let routes = self.routes.clone(); + let permits = Arc::new(Semaphore::new(64)); + let mut connections = JoinSet::new(); + ocsf_emit!( + AppLifecycleBuilder::new(crate::ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .message("Supervisor app ingress ready") + .build() + ); + loop { + tokio::select! { + Some(_) = connections.join_next(), if !connections.is_empty() => {}, + accepted = self.receiver.recv() => { + let Some(Accepted { tls, permit: host_permit }) = accepted else { break; }; + let Ok(instance_permit) = permits.clone().try_acquire_owned() else { continue; }; + let Some(sni) = tls.get_ref().1.server_name().map(str::to_ascii_lowercase) else { continue; }; + let prefix = routes.registration.as_ref().expect("instance registration").prefix.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone(); + let Some((route, token)) = routes.resolve(&sni, &prefix, &self.domain) else { + emit_denial("App listener unknown or expired SNI route"); continue; + }; + let routes = routes.clone(); + let connector = connector.clone(); + connections.spawn(async move { + let connection_routes = routes.clone(); + let connection_route = route.clone(); + let permit = Arc::new(Permits { _host: host_permit, _instance: instance_permit }); + let service = hyper::service::service_fn(move |req| { + proxy(req, sni.clone(), route.clone(), routes.clone(), connector.clone(), permit.clone(), token) + }); + let connection = hyper::server::conn::http1::Builder::new() + .timer(TokioTimer::new()).header_read_timeout(HANDSHAKE_TIMEOUT) + .max_buf_size(65536).max_headers(64) + .serve_connection(TokioIo::new(tls), service).with_upgrades(); + tokio::select! { + _ = connection => {}, + () = connection_routes.revoked(&connection_route, token) => {}, + } + }); + } + } + } + }); + Handle { + task, + routes: retained_routes, + } + } +} + +fn error(status: StatusCode) -> Response { + ocsf_emit!( + HttpActivityBuilder::new(crate::ocsf_ctx()) + .http_response(HttpResponse { + code: status.as_u16() + }) + .activity(ActivityId::Other) + .action(ActionId::Denied) + .disposition(if status.is_server_error() { + DispositionId::Error + } else { + DispositionId::Blocked + }) + .severity(if status.is_server_error() { + SeverityId::Low + } else { + SeverityId::Medium + }) + .status(StatusId::Failure) + .message(format!( + "App listener request rejected [status:{}]", + status.as_u16() + )) + .build() + ); + Response::builder() + .status(status) + .body(Body::empty()) + .expect("valid error response") +} + +fn emit_denial(message: &str) { + ocsf_emit!( + NetworkActivityBuilder::new(crate::ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(openshell_ocsf::Endpoint { + domain: None, + ip: None, + port: None, + name: None, + }) + .message(message) + .build() + ); +} + +fn authority_matches(req: &Request, sni: &str) -> bool { + let mut values = req.headers().get_all(header::HOST).iter(); + let Some(value) = values.next() else { + return false; + }; + if values.next().is_some() { + return false; + } + let Ok(value) = value.to_str() else { + return false; + }; + if value.contains('@') { + return false; + } + let Ok(authority) = value.parse::() else { + return false; + }; + if authority.port().is_some() && authority.port_u16().is_none() { + return false; + } + authority.host().eq_ignore_ascii_case(sni) + && req + .uri() + .authority() + .is_none_or(|uri| uri.as_str().eq_ignore_ascii_case(value)) +} + +fn gateway_header(name: &str) -> bool { + matches!( + name, + "proxy-authorization" + | "cf-access-jwt-assertion" + | "cf-access-authenticated-user-email" + | "x-openshell-principal" + | "x-openshell-user" + | "x-openshell-roles" + | "x-forwarded-client-cert" + | "x-ssl-client-cert" + | "x-client-cert" + ) +} + +fn hop_header(name: &str) -> bool { + matches!( + name, + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) +} + +fn sanitize_headers(headers: &mut http::HeaderMap, passthrough: bool, websocket: bool) -> bool { + if passthrough { + let values: Vec<_> = headers.get_all(header::AUTHORIZATION).iter().collect(); + if values.len() > 1 { + return false; + } + if let Some(value) = values.first() { + let Ok(value) = value.to_str() else { + return false; + }; + let Some(token) = openshell_core::auth::strip_bearer_scheme(value) else { + return false; + }; + if token.is_empty() + || !token + .trim_end_matches('=') + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-._~+/".contains(&byte)) + || token.trim_end_matches('=').is_empty() + { + return false; + } + } + } else { + headers.remove(header::AUTHORIZATION); + } + let connection_headers: Vec = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(|value| value.trim().to_ascii_lowercase()) + .collect(); + let remove: Vec<_> = headers + .keys() + .filter(|name| { + gateway_header(name.as_str()) + || ((hop_header(name.as_str()) + || connection_headers + .iter() + .any(|value| value == name.as_str())) + && !(websocket && matches!(name.as_str(), "connection" | "upgrade"))) + }) + .cloned() + .collect(); + for name in remove { + headers.remove(name); + } + let cookies: Vec<_> = headers + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(';')) + .filter(|cookie| { + cookie.split_once('=').is_some_and(|(name, _)| { + !name.trim().eq_ignore_ascii_case("CF_Authorization") + && !name.trim().eq_ignore_ascii_case("cf-authorization") + }) + }) + .map(str::trim) + .collect(); + let cookie = cookies.join("; "); + headers.remove(header::COOKIE); + if !cookie.is_empty() + && let Ok(cookie) = cookie.parse() + { + headers.insert(header::COOKIE, cookie); + } + true +} + +async fn proxy( + mut req: Request, + sni: String, + route: SandboxServiceRoute, + routes: Routes, + connector: Arc, + permit: Arc, + token: RouteToken, +) -> std::result::Result, std::convert::Infallible> { + if !routes.valid(&route, token) { + return Ok(error(StatusCode::SERVICE_UNAVAILABLE)); + } + if !authority_matches(&req, &sni) { + return Ok(error(StatusCode::MISDIRECTED_REQUEST)); + } + // Keep the public authority after validating it against SNI. The dial target + // is loopback, but rewriting Host would break application same-origin checks. + let authority = req.headers()[header::HOST].clone(); + let websocket = req + .headers() + .get(header::UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")); + if !sanitize_headers(req.headers_mut(), route.authorization_mode == 2, websocket) { + return Ok(error(StatusCode::BAD_REQUEST)); + } + let downstream = websocket.then(|| hyper::upgrade::on(&mut req)); + let target = LoopbackTarget::new( + "127.0.0.1".parse().expect("loopback"), + u16::try_from(route.target_port).expect("validated route port"), + ) + .expect("validated route port"); + let Ok(Ok(stream)) = tokio::time::timeout(CONNECT_TIMEOUT, connector.connect(target)).await + else { + return Ok(error(StatusCode::BAD_GATEWAY)); + }; + let Ok((mut sender, connection)) = + hyper::client::conn::http1::handshake(TokioIo::new(stream)).await + else { + return Ok(error(StatusCode::BAD_GATEWAY)); + }; + let upstream_task = tokio::spawn(async move { + let _ = connection.with_upgrades().await; + }); + let (mut parts, body) = req.into_parts(); + parts.uri = parts + .uri + .path_and_query() + .map_or("/", |path| path.as_str()) + .parse() + .expect("valid path"); + parts.headers.insert(header::HOST, authority); + let response = sender.send_request(Request::from_parts(parts, body)).await; + let Ok(mut response) = response else { + upstream_task.abort(); + return Ok(error(StatusCode::BAD_GATEWAY)); + }; + if response.status() == StatusCode::SWITCHING_PROTOCOLS && websocket { + let upstream = hyper::upgrade::on(&mut response); + let downstream = downstream.expect("websocket upgrade"); + tokio::spawn(async move { + let _permit = permit; + tokio::select! { + () = async { + if let (Ok(upstream), Ok(downstream)) = tokio::join!(upstream, downstream) { + let _ = tokio::io::copy_bidirectional(&mut TokioIo::new(upstream), &mut TokioIo::new(downstream)).await; + } + } => {}, + () = routes.revoked(&route, token) => {}, + } + upstream_task.abort(); + }); + let (parts, _) = response.into_parts(); + return Ok(Response::from_parts(parts, Body::empty())); + } + let (parts, body) = response.into_parts(); + Ok(Response::from_parts(parts, Body::new(body))) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::{SinkExt as _, StreamExt as _}; + use openshell_isolation_interface::contract::{BackendError, BoundaryDuplexStream}; + use openshell_sandbox_backend::test_support as rpc_test; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio_tungstenite::tungstenite::client::IntoClientRequest as _; + + #[test] + fn listener_requires_explicit_enablement_and_complete_configuration() { + let complete = BTreeMap::from([ + ("OPENSHELL_APP_LISTEN_ADDRESS", "0.0.0.0:9443"), + ("OPENSHELL_APP_SERVICE_DOMAIN", "services.example.com"), + ("OPENSHELL_APP_TLS_CERT", "/operator/cert.pem"), + ("OPENSHELL_APP_TLS_KEY", "/operator/key.pem"), + ]); + assert!( + Config::from_values(|name| complete.get(name).map(ToString::to_string)) + .unwrap() + .is_none() + ); + let mut values = complete.clone(); + values.insert("OPENSHELL_APP_LISTENER_ENABLED", "false"); + assert!( + Config::from_values(|name| values.get(name).map(ToString::to_string)) + .unwrap() + .is_none() + ); + values.insert("OPENSHELL_APP_LISTENER_ENABLED", "true"); + assert_eq!( + Config::from_values(|name| values.get(name).map(ToString::to_string)) + .unwrap() + .unwrap() + .address + .port(), + 9443 + ); + values.remove("OPENSHELL_APP_TLS_KEY"); + assert!(Config::from_values(|name| values.get(name).map(ToString::to_string)).is_err()); + assert!( + Config::from_values( + |name| (name == "OPENSHELL_APP_LISTENER_ENABLED").then(|| "true".into()) + ) + .is_err() + ); + } + + const HOST: &str = "default--sandbox-a--web.svc.openshell.internal"; + + fn installed(routes: Vec) -> SettingsPollResult { + SettingsPollResult { + service_routes: routes, + settings_revision: 1, + configuration_instance_id: "runtime-a".into(), + configuration_admitted: true, + configuration_error: String::new(), + policy: None, + version: 1, + policy_hash: String::new(), + config_revision: 1, + policy_source: openshell_core::proto::PolicySource::Unspecified, + settings: std::collections::HashMap::default(), + global_policy_version: 0, + provider_env_revision: 0, + provider_attachment_epoch: String::new(), + supervisor_middleware_services: Vec::new(), + workspace: "default".into(), + policy_validation_failure_mode: openshell_core::PolicyValidationFailureMode::default(), + extension_authentication_enabled: false, + } + } + + struct Connector { + calls: AtomicUsize, + } + + #[tonic::async_trait] + impl rpc_test::BoundaryLoopbackConnector for Connector { + async fn connect( + &self, + target: LoopbackTarget, + ) -> std::result::Result { + assert_eq!(target.port(), 8080); + let (client, server) = tokio::io::duplex(8192); + self.calls.fetch_add(1, Ordering::SeqCst); + tokio::spawn(async move { + let service = hyper::service::service_fn(|mut req: Request| async move { + if let Some(key) = req.headers().get("sec-websocket-key") { + let accept = tokio_tungstenite::tungstenite::handshake::derive_accept_key( + key.as_bytes(), + ); + let upgraded = hyper::upgrade::on(&mut req); + tokio::spawn(async move { + if let Ok(stream) = upgraded.await { + let mut ws = tokio_tungstenite::WebSocketStream::from_raw_socket( + TokioIo::new(stream), + tokio_tungstenite::tungstenite::protocol::Role::Server, + None, + ) + .await; + while let Some(Ok(message)) = ws.next().await { + if ws.send(message).await.is_err() { + break; + } + } + } + }); + return Ok::<_, std::convert::Infallible>( + Response::builder() + .status(101) + .header("connection", "upgrade") + .header("upgrade", "websocket") + .header("sec-websocket-accept", accept) + .header("x-test-upstream-host", req.headers()[header::HOST].clone()) + .header( + "x-test-upstream-origin", + req.headers() + .get(header::ORIGIN) + .cloned() + .unwrap_or_else(|| "absent".parse().unwrap()), + ) + .body(Body::empty()) + .unwrap(), + ); + } + let body = format!( + "{} {} {} {}", + req.uri(), + req.headers()[header::HOST].to_str().unwrap(), + req.headers() + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .unwrap_or("stripped"), + req.headers() + .get(header::ORIGIN) + .and_then(|v| v.to_str().ok()) + .unwrap_or("absent") + ); + Ok::<_, std::convert::Infallible>(Response::new(Body::from(body))) + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(server), service) + .with_upgrades() + .await; + }); + Ok(Box::new(client)) + } + } + + struct Fixture { + _directory: tempfile::TempDir, + handle: Handle, + address: SocketAddr, + tls: Arc, + routes: Routes, + connector: Arc, + route: SandboxServiceRoute, + host: HostHandle, + } + + fn fixture() -> Fixture { + openshell_crypto::tls::ensure_default_provider(); + let directory = tempfile::tempdir().unwrap(); + let cert = openshell_crypto::pki::generate_simple_self_signed(vec![ + "*.svc.openshell.internal".into(), + ]) + .unwrap(); + let cert_path = directory.path().join("cert.pem"); + let key_path = directory.path().join("key.pem"); + std::fs::write(&cert_path, cert.cert.pem()).unwrap(); + std::fs::write(&key_path, cert.key_pair.serialize_pem().unwrap()).unwrap(); + let config = Config { + address: "127.0.0.1:0".parse().unwrap(), + domain: "svc.openshell.internal".into(), + certificate: cert_path, + key: key_path, + }; + let bound = config.bind().unwrap(); + let address = bound.listener.local_addr().unwrap(); + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.cert.der().clone()).unwrap(); + let tls = Arc::new( + openshell_crypto::tls::client_builder() + .with_root_certificates(roots) + .with_no_client_auth(), + ); + let host = bound.start(); + let instance = host.router().instance("sandbox-a").unwrap(); + let routes = instance.routes.clone(); + let route = SandboxServiceRoute { + endpoint_id: "endpoint-a".into(), + name: "web".into(), + target_port: 8080, + authorization_mode: 1, + resource_version: 1, + }; + routes.install(&installed(vec![route.clone()])); + let connector = Arc::new(Connector { + calls: AtomicUsize::new(0), + }); + let handle = instance.start(rpc_test::loopback(connector.clone())); + Fixture { + _directory: directory, + handle, + address, + tls, + routes, + connector, + route, + host, + } + } + + async fn request(f: &Fixture, sni: &str, host: &str, extra: &str) -> String { + request_method(f, "GET", sni, host, extra).await + } + + async fn request_method( + f: &Fixture, + method: &str, + sni: &str, + host: &str, + extra: &str, + ) -> String { + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[f.address]) + .await + .unwrap(); + let server_name = rustls::pki_types::ServerName::try_from(sni.to_string()).unwrap(); + let mut tls = tokio_rustls::TlsConnector::from(f.tls.clone()) + .connect(server_name, stream) + .await + .unwrap(); + let _ = tls + .write_all( + format!( + "{method} /hello?x=1 HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n{extra}\r\n" + ) + .as_bytes(), + ) + .await; + let mut response = Vec::new(); + // Unknown SNI closes the TLS stream without an HTTP response. + let _ = tokio::time::timeout(Duration::from_secs(3), tls.read_to_end(&mut response)) + .await + .unwrap(); + String::from_utf8(response).unwrap() + } + + #[tokio::test] + async fn browser_post_preserves_public_authority_and_origin() { + let f = fixture(); + let authority = format!("{HOST}:18443"); + let origin = format!("https://{authority}"); + let response = request_method( + &f, + "POST", + HOST, + &authority, + &format!("Origin: {origin}\r\nContent-Length: 0\r\n"), + ) + .await; + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!( + response.contains(&format!("/hello?x=1 {authority} stripped {origin}")), + "{response}" + ); + } + + #[tokio::test] + async fn shared_listener_keeps_instance_connectors_and_lifetimes_separate() { + let f = fixture(); + let second = f.host.router().instance("sandbox-b").unwrap(); + let second_routes = second.routes.clone(); + let second_connector = Arc::new(Connector { + calls: AtomicUsize::new(0), + }); + let second_handle = second.start(rpc_test::loopback(second_connector.clone())); + second_routes.install(&installed(vec![SandboxServiceRoute { + endpoint_id: "endpoint-b".into(), + ..f.route.clone() + }])); + let host_b = "default--sandbox-b--web.svc.openshell.internal"; + assert!( + request(&f, host_b, host_b, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert_eq!(second_connector.calls.load(Ordering::SeqCst), 1); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 0); + + drop(second_handle); + assert!( + !request(&f, host_b, host_b, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert!( + request(&f, HOST, HOST, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn stale_instance_cleanup_cannot_remove_its_replacement() { + let f = fixture(); + let replacement = f.host.router().instance("sandbox-a").unwrap(); + replacement + .routes + .install(&installed(vec![f.route.clone()])); + assert!( + replacement.routes.sender.borrow().routes.is_empty(), + "duplicate instance must not claim a live registration" + ); + let old_routes = f.routes.clone(); + drop(f.handle); + replacement + .routes + .install(&installed(vec![f.route.clone()])); + let prefix = "default--sandbox-a"; + assert!( + f.host + .router() + .registrations + .lock() + .unwrap() + .contains_key(prefix) + ); + // Simulate delayed cleanup from the old runtime after replacement. + old_routes.registration.as_ref().unwrap().remove(); + assert!( + f.host + .router() + .registrations + .lock() + .unwrap() + .contains_key(prefix) + ); + } + + #[tokio::test(start_paused = true)] + async fn push_freshness_renews_live_routes_but_cannot_revive_rejected_routes() { + let routes = Routes::default(); + let route = SandboxServiceRoute { + endpoint_id: "ep".into(), + name: "web".into(), + target_port: 8080, + authorization_mode: 1, + resource_version: 1, + }; + routes.install(&installed(vec![route.clone()])); + for _ in 0..10 { + tokio::time::advance(Duration::from_secs(5)).await; + routes.renew(); + } + assert!( + routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .is_some() + ); + let mut rejected = installed(vec![route]); + rejected.configuration_admitted = false; + routes.install(&rejected); + routes.renew(); + assert!( + routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .is_none() + ); + } + + #[tokio::test] + async fn tls_routes_only_matching_sni_and_authority() { + let f = fixture(); + let response = request( + &f, + HOST, + &format!("{HOST}:18443"), + "Authorization: Bearer app-token\r\n", + ) + .await; + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!( + response.contains(&format!("/hello?x=1 {HOST}:18443 stripped")), + "{response}" + ); + let mismatch = request( + &f, + HOST, + "default--sandbox-b--web.svc.openshell.internal", + "", + ) + .await; + assert!(mismatch.starts_with("HTTP/1.1 421"), "{mismatch}"); + let unknown = request( + &f, + "default--sandbox-b--web.svc.openshell.internal", + "default--sandbox-b--web.svc.openshell.internal", + "", + ) + .await; + assert!(unknown.is_empty()); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn duplicate_host_and_malformed_bearer_never_dial_boundary() { + let f = fixture(); + let duplicate = request(&f, HOST, HOST, &format!("Host: {HOST}\r\n")).await; + assert!(!duplicate.starts_with("HTTP/1.1 200")); + f.routes + .sender + .send_modify(|snapshot| snapshot.routes.get_mut("web").unwrap().authorization_mode = 2); + let malformed = request(&f, HOST, HOST, "Authorization: Basic credential\r\n").await; + assert!(malformed.starts_with("HTTP/1.1 400")); + let valid = request(&f, HOST, HOST, "Authorization: bEaReR app-token\r\n").await; + assert!(valid.contains("bEaReR app-token"), "{valid}"); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn withdrawn_and_expired_routes_never_dial_boundary() { + let f = fixture(); + let (_, old_token) = f + .routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .unwrap(); + f.routes.suspend(); + f.routes.install(&installed(vec![f.route.clone()])); + // Even a quickly restored identical route cannot revive an old stream. + tokio::time::timeout( + Duration::from_secs(1), + f.routes.revoked(&f.route, old_token), + ) + .await + .unwrap(); + f.routes.sender.send_modify(|snapshot| { + snapshot.expires = Some(Instant::now() - Duration::from_secs(1)); + }); + assert!(request(&f, HOST, HOST, "").await.is_empty()); + f.routes.suspend(); + assert!(request(&f, HOST, HOST, "").await.is_empty()); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 0); + } + + #[test] + fn headers_strip_edge_identity_and_connection_named_fields() { + let mut headers = http::HeaderMap::new(); + headers.insert( + header::COOKIE, + "app=session; CF_Authorization=secret".parse().unwrap(), + ); + headers.insert("x-forwarded-client-cert", "secret".parse().unwrap()); + headers.insert(header::CONNECTION, "x-internal".parse().unwrap()); + headers.insert("x-internal", "secret".parse().unwrap()); + assert!(sanitize_headers(&mut headers, false, false)); + assert_eq!(headers[header::COOKIE], "app=session"); + assert!(!headers.contains_key("x-forwarded-client-cert")); + assert!(!headers.contains_key("x-internal")); + } + #[tokio::test] + async fn websocket_echo_closes_on_route_withdrawal() { + let f = fixture(); + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[f.address]) + .await + .unwrap(); + let name = rustls::pki_types::ServerName::try_from(HOST.to_string()).unwrap(); + let tls = tokio_rustls::TlsConnector::from(f.tls.clone()) + .connect(name, stream) + .await + .unwrap(); + let mut request = format!("wss://{HOST}/echo").into_client_request().unwrap(); + request + .headers_mut() + .insert(header::ORIGIN, format!("https://{HOST}").parse().unwrap()); + let (mut ws, response) = tokio_tungstenite::client_async(request, tls).await.unwrap(); + assert_eq!(response.headers()["x-test-upstream-host"], HOST); + assert_eq!( + response.headers()["x-test-upstream-origin"], + format!("https://{HOST}") + ); + ws.send(tokio_tungstenite::tungstenite::Message::Text( + "hello".into(), + )) + .await + .unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .unwrap() + .unwrap() + .unwrap() + .to_text() + .unwrap(), + "hello" + ); + f.routes.suspend(); + let closed = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .unwrap(); + assert!(closed.is_none() || closed.unwrap().is_err()); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } + + type TestWebSocket = + tokio_tungstenite::WebSocketStream>; + + async fn websocket(f: &Fixture, service: &str) -> TestWebSocket { + let host = format!("default--sandbox-a--{service}.svc.openshell.internal"); + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[f.address]) + .await + .unwrap(); + let name = rustls::pki_types::ServerName::try_from(host.clone()).unwrap(); + let tls = tokio_rustls::TlsConnector::from(f.tls.clone()) + .connect(name, stream) + .await + .unwrap(); + tokio_tungstenite::client_async(format!("wss://{host}/echo"), tls) + .await + .unwrap() + .0 + } + + async fn echo(ws: &mut TestWebSocket, value: &str) { + ws.send(tokio_tungstenite::tungstenite::Message::Text(value.into())) + .await + .unwrap(); + let echoed = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(echoed.to_text().unwrap(), value); + } + + async fn closed(ws: &mut TestWebSocket) { + let next = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .expect("revoked WebSocket must close"); + assert!(next.is_none() || next.unwrap().is_err()); + } + + #[tokio::test] + async fn service_edits_close_only_affected_websockets() { + let f = fixture(); + let mut web = websocket(&f, "web").await; + let mut portal = SandboxServiceRoute { + endpoint_id: "portal-id".into(), + name: "portal".into(), + ..f.route.clone() + }; + f.routes + .install(&installed(vec![f.route.clone(), portal.clone()])); + echo(&mut web, "web survives portal addition").await; + let mut portal_ws = websocket(&f, "portal").await; + echo(&mut portal_ws, "portal active").await; + + // Renewing identical declarations does not revoke either stream. + f.routes + .install(&installed(vec![f.route.clone(), portal.clone()])); + echo(&mut web, "web renewed").await; + echo(&mut portal_ws, "portal renewed").await; + + portal.authorization_mode = 2; + portal.resource_version += 1; + f.routes + .install(&installed(vec![f.route.clone(), portal.clone()])); + closed(&mut portal_ws).await; + echo(&mut web, "web survives portal edit").await; + + let mut portal_ws = websocket(&f, "portal").await; + f.routes.install(&installed(vec![f.route.clone()])); + closed(&mut portal_ws).await; + echo(&mut web, "web survives portal deletion").await; + + f.routes.install(&installed(vec![f.route.clone(), portal])); + let mut portal_ws = websocket(&f, "portal").await; + f.routes.suspend(); + closed(&mut web).await; + closed(&mut portal_ws).await; + } + + #[tokio::test] + async fn coalesced_recreation_and_lease_lapse_cannot_revive_old_streams() { + let f = fixture(); + let (_, original) = f + .routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .unwrap(); + // Both updates can occur before a watch receiver observes either one. + f.routes.install(&installed(Vec::new())); + f.routes.install(&installed(vec![f.route.clone()])); + tokio::time::timeout(Duration::from_secs(1), f.routes.revoked(&f.route, original)) + .await + .unwrap(); + + let (_, renewed) = f + .routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .unwrap(); + f.routes.sender.send_modify(|snapshot| { + snapshot.expires = Some(Instant::now() - Duration::from_secs(1)); + }); + f.routes.install(&installed(vec![f.route.clone()])); + tokio::time::timeout(Duration::from_secs(1), f.routes.revoked(&f.route, renewed)) + .await + .unwrap(); + } +} diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index bdf4035c61..49d343b24e 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -825,6 +825,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .store(true, Ordering::SeqCst); let startup = |await_main_process_attachment| { let config = crate::SandboxInstanceConfig { + app_router: None, command: vec!["test-agent".into(), "test-argument".into()], workdir: Some("/test-workspace".into()), timeout_secs: 15, @@ -1029,6 +1030,7 @@ async fn suspend_signal_exits_without_terminating_boundary() { let setup = TestSetup::new(); let startup = || { let config = crate::SandboxInstanceConfig { + app_router: None, command: vec!["test-agent".into(), "test-argument".into()], workdir: Some("/test-workspace".into()), timeout_secs: 15, diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index 1a83fd0887..44a372670e 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -904,6 +904,9 @@ impl ConfigRuntime { let requested_revision = sandbox_config_revision(&snapshot); let desired_environment = EnvironmentIdentity::from_settings(&snapshot); let provider = if !snapshot.configuration_admitted { + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } if !self.reloads_gateway_policy { return DeliveredResult { outcome: ConfigApplyOutcome::FailedRetainedLastKnownGood, @@ -938,6 +941,9 @@ impl ConfigRuntime { { Ok(prepared) => Some(prepared), Err(failure) => { + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } // Static credentials were revoked, so this configuration // is not applied until its environment installs. let outcome = if self.applied_revision.is_some() { @@ -968,6 +974,9 @@ impl ConfigRuntime { outcome, policy_status, }) => { + if let Some(routes) = &ctx.app_routes { + routes.install(&snapshot); + } self.stream.rejected_sandbox = None; let _ = ctx.workspace_tx.send(snapshot.workspace.clone()); let outcome = if !self.reloads_gateway_policy { @@ -987,6 +996,9 @@ impl ConfigRuntime { }; } Err(error) => { + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } // Polling ends its loop on this error; session delivery // reports it instead, so record it here. ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) @@ -1013,6 +1025,9 @@ impl ConfigRuntime { } }; + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } // The previous revision is retained only while its policy is enforced. let retained_revision = self .applied_revision @@ -1214,6 +1229,10 @@ async fn run_policy_poll_loop_with_client( client: C, ) -> Result<()> { let mut config_apply_rx = ctx.config_apply_rx.take(); + let _app_configuration = ctx + .app_routes + .clone() + .map(crate::app_listener::Routes::configuration_guard); let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(run_policy_status_reporter( client.clone(), @@ -1296,6 +1315,9 @@ async fn run_policy_poll_loop_with_client( ctx.provider_credentials.snapshot().revision, ) .await; + if let Some(routes) = &ctx.app_routes { + routes.install(&result); + } runtime.middleware_services = result.supervisor_middleware_services; runtime.extension_authentication_enabled = result.extension_authentication_enabled; @@ -1351,7 +1373,20 @@ async fn run_policy_poll_loop_with_client( // The session's bootstrap supersedes a polled result that was // not reconciled yet. pending_result = None; + // A quiet push stream still has authenticated session freshness. + // Renew only installed, unexpired routes. Poll mode renews by + // successfully applying a complete configuration instead. + if ctx.supervisor_session_id.borrow().is_some() + && let Some(routes) = &ctx.app_routes + { + routes.renew(); + } let delay = next_poll_delay(&ctx.extension_credentials, interval); + let delay = if ctx.app_routes.is_some() { + delay.min(Duration::from_secs(5)) + } else { + delay + }; tokio::select! { changed = async { match config_push_updates.as_mut() { @@ -1413,9 +1448,25 @@ async fn run_policy_poll_loop_with_client( } }; - let reconciled = runtime + let reconciled = match runtime .reconcile(&ctx, &client, &result, ProviderSource::Fetch) - .await?; + .await + { + Ok(reconciled) => reconciled, + Err(error) => { + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } + return Err(error); + } + }; + if let Some(routes) = &ctx.app_routes { + if matches!(reconciled.outcome, ReconcileOutcome::Rejected(_)) { + routes.suspend(); + } else { + routes.install(&result); + } + } if let Some(status) = reconciled.policy_status { enqueue_policy_status(&status_sender, status); } @@ -2710,6 +2761,7 @@ mod tests { ); } PolicyPollLoopContext { + app_routes: None, gateway: test_gateway(), sandbox_id: "sandbox-test".to_string(), sandbox: "sandbox-test-name".to_string(), diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index 2d3894d563..992bbbde18 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -278,6 +278,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl probe: probe.clone(), }; let config = SandboxInstanceConfig { + app_router: None, command: vec!["true".into()], workdir: None, timeout_secs: 0, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index dddf8990c0..0d19e26e7f 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -18,6 +18,7 @@ compile_error!( ); mod activity_aggregator; +pub mod app_listener; pub mod backend_setup; mod config_runtime; mod denial_aggregator; @@ -886,6 +887,7 @@ pub fn trace_context_from_environment() -> Option { /// from its [`SandboxInstanceBindings`], so one process can host several /// instances. pub struct SandboxInstanceConfig { + pub app_router: Option, pub command: Vec, pub workdir: Option, pub timeout_secs: u64, @@ -1051,6 +1053,7 @@ pub async fn run_sandbox_instance( controls: SandboxInstanceControls, ) -> Result { let SandboxInstanceConfig { + app_router, command, workdir, timeout_secs, @@ -1578,6 +1581,26 @@ pub async fn run_sandbox_instance( }); } + let app_instance = + match app_router { + Some(router) => { + if !loaded_policy_origin.allows_gateway_policy_reload() { + return Err(miette::miette!( + "app ingress requires gateway-managed configuration" + )); + } + Some( + router.instance(sandbox.as_deref().ok_or_else(|| { + miette::miette!("app ingress requires sandbox identity") + })?)?, + ) + } + None => None, + }; + let app_routes = app_instance + .as_ref() + .map(|instance| instance.routes.clone()); + // Spawn background policy poll task (gRPC mode only). if let (Some(id), Some(sandbox), Some(gateway), Some(engine)) = ( sandbox_id.as_deref(), @@ -1603,6 +1626,7 @@ pub async fn run_sandbox_instance( .and_then(|v| v.parse().ok()) .unwrap_or(10); let poll_ctx = PolicyPollLoopContext { + app_routes: app_routes.clone(), gateway: poll_gateway, sandbox_id: poll_id, sandbox: poll_sandbox, @@ -1688,6 +1712,8 @@ pub async fn run_sandbox_instance( } else { info!(backend = %backend_name, "Isolation boundary agent started"); } + let _app_listener = + app_instance.map(|instance| instance.start(running.loopback_connector())); let agent = running.agent(); let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( sandbox_id.as_deref(), @@ -4193,6 +4219,7 @@ async fn report_initial_policy_failure( /// When the entrypoint PID is available, policy reloads include symlink /// resolution for binary paths via the container filesystem. struct PolicyPollLoopContext { + app_routes: Option, gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, /// Canonical sandbox reference used by name-scoped configuration APIs. @@ -4618,6 +4645,7 @@ mod tests { ocsf_otlp_headers: None, }; let config = SandboxInstanceConfig { + app_router: None, command: vec!["true".to_string()], workdir: None, timeout_secs: 0, @@ -5090,6 +5118,7 @@ network_policies: source: openshell_core::proto::PolicySource, ) -> openshell_core::grpc_client::SettingsPollResult { openshell_core::grpc_client::SettingsPollResult { + service_routes: Vec::new(), policy, version, policy_hash: format!("hash-v{version}"), @@ -7168,6 +7197,7 @@ network_policies: let canonical = settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); let canonical = openshell_core::grpc_client::SettingsPollResult { + service_routes: Vec::new(), policy_hash: "hash-provider-change".to_string(), config_revision: loaded.config_revision + 1, ..canonical @@ -7222,6 +7252,7 @@ network_policies: #[test] fn unchanged_sandbox_policy_revision_candidate_is_strictly_scoped() { let sandbox_result = openshell_core::grpc_client::SettingsPollResult { + service_routes: Vec::new(), policy_hash: "same-policy".to_string(), ..settings_poll_result( Some(proto_policy_fixture()), @@ -7262,6 +7293,7 @@ network_policies: ); let global_result = openshell_core::grpc_client::SettingsPollResult { + service_routes: Vec::new(), policy_hash: "same-policy".to_string(), ..settings_poll_result( Some(proto_policy_fixture()), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index be91a6455b..c81961636f 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -604,7 +604,14 @@ fn main() -> Result<()> { "isolation-backend role started without validated runtime inputs" )); }; + let app_host = openshell_supervisor::app_listener::Config::from_env()? + .map(openshell_supervisor::app_listener::Config::bind) + .transpose()? + .map(openshell_supervisor::app_listener::BoundListener::start); let config = openshell_supervisor::SandboxInstanceConfig { + app_router: app_host + .as_ref() + .map(openshell_supervisor::app_listener::HostHandle::router), command, workdir, timeout_secs: args.timeout, diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index 28f49b426b..305ebb3ca2 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -182,6 +182,7 @@ pub struct SharedHostConfig { /// The sandbox instance registry and the routed runtime listener its /// instances claim routes on, reconciled to the gateway's assignments. pub struct SharedHost { + app_listener: Option, host: SandboxInstanceHost, router: Arc, gateway_endpoint: String, @@ -287,10 +288,15 @@ impl From for ControlError { impl SharedHost { /// Bind the routed runtime listener and prepare the instance root. pub fn start(config: SharedHostConfig) -> Result> { + let app_listener = crate::app_listener::Config::from_env()? + .map(crate::app_listener::Config::bind) + .transpose()? + .map(crate::app_listener::BoundListener::start); let host = SandboxInstanceHost::new(config.instance_root)?; let router = RuntimeRouter::bind(config.runtime_listen) .map_err(|error| miette::miette!("{error}"))?; Ok(Arc::new(Self { + app_listener, host, router, gateway_endpoint: config.gateway_endpoint, @@ -776,6 +782,10 @@ impl SharedHost { crate::sandbox_instance_host::prepare_private_dir(&trust_dir).map_err(failure)?; let ssh_socket = dir.join("ssh.sock"); let config = crate::SandboxInstanceConfig { + app_router: self + .app_listener + .as_ref() + .map(crate::app_listener::HostHandle::router), command: main_process.command, workdir: registration.workdir, timeout_secs: 0, diff --git a/deploy/helm/openshell/ci/values-app-listener.yaml b/deploy/helm/openshell/ci/values-app-listener.yaml new file mode 100644 index 0000000000..f5f568e145 --- /dev/null +++ b/deploy/helm/openshell/ci/values-app-listener.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Opt-in fixture. Provision openshell-app-tls and an app-ingress NetworkPolicy +# before sending traffic. This does not deploy discovery or a proxy. +gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.supervisor.app_listener: + enabled: true + port: 9443 + domain: services.example.com + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + tls_secret_name: openshell-app-tls + create_service: true diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 48be86591d..73e4797b95 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -269,6 +269,11 @@ shared workspace mode. {{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- $names := list -}} +{{- $app := include "openshell.appListenerConfig" . | fromYaml -}} +{{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml -}} +{{- if and (get $app "enabled") (get $appDeployment "tls_secret_name") (not (include "openshell.supervisorEnabled" .)) -}} +{{- $names = append $names (get $appDeployment "tls_secret_name") -}} +{{- end -}} {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) -}} {{- $names = append $names .Values.server.tls.clientTlsSecretName -}} {{- end -}} @@ -631,6 +636,13 @@ absent fields so every chart consumer observes the same configuration. {{- end -}} {{- $legacyRuntime := .Values.supervisor.sandboxRuntime | default dict -}} {{- $runtimeConfig := get $kubernetes "sandbox_runtime" | default dict -}} +{{- if hasKey $gatewayConfig "openshell.drivers.kubernetes.sandbox_runtime" -}} +{{- $runtimeConfig = deepCopy (get $gatewayConfig "openshell.drivers.kubernetes.sandbox_runtime") -}} +{{- end -}} +{{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml -}} +{{- if $appDeployment -}} +{{- $_ := set $runtimeConfig "app_listener" $appDeployment -}} +{{- end -}} {{- if and (ne (int (get $legacyRuntime "boundaryPort" | default 5500)) 5500) (not (hasKey $runtimeConfig "boundary_port")) -}} {{- $_ := set $runtimeConfig "boundary_port" (int (get $legacyRuntime "boundaryPort")) -}} {{- end -}} @@ -807,3 +819,28 @@ Validate chart values that Helm would otherwise accept silently. {{- fail "server.dataStorageSize must be a Kubernetes quantity with a unit, such as 20Gi" -}} {{- end -}} {{- end }} + +{{/* Common app settings are separate from driver-owned TLS/Service details. */}} +{{- define "openshell.appListenerConfig" -}} +{{- $config := .Values.gatewayConfig | default dict -}} +{{- $supervisor := get $config "openshell.supervisor" | default dict -}} +{{- $app := get $supervisor "app_listener" | default dict -}} +{{- if hasKey $config "openshell.supervisor.app_listener" -}} +{{- $app = get $config "openshell.supervisor.app_listener" -}} +{{- end -}} +{{- toYaml $app -}} +{{- end -}} + +{{- define "openshell.appListenerDeployment" -}} +{{- $config := .Values.gatewayConfig | default dict -}} +{{- $kube := get $config "openshell.drivers.kubernetes" | default dict -}} +{{- $runtime := get $kube "sandbox_runtime" | default dict -}} +{{- if hasKey $config "openshell.drivers.kubernetes.sandbox_runtime" -}} +{{- $runtime = get $config "openshell.drivers.kubernetes.sandbox_runtime" -}} +{{- end -}} +{{- $app := get $runtime "app_listener" | default dict -}} +{{- if hasKey $config "openshell.drivers.kubernetes.sandbox_runtime.app_listener" -}} +{{- $app = get $config "openshell.drivers.kubernetes.sandbox_runtime.app_listener" -}} +{{- end -}} +{{- toYaml $app -}} +{{- end -}} diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index cf8bf61202..9f65498986 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -132,6 +132,9 @@ Top-level lists represent TOML arrays of tables and preserve their YAML order. * {{- $_ := set $config "openshell.gateway.gateway_jwt" $gatewayJwt -}} {{- $kubernetesCompat := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $_ := set $config "openshell.drivers.kubernetes" $kubernetesCompat -}} +{{/* Runtime children are merged above: do not extend a sealed inline table. */}} +{{- $_ := unset $config "openshell.drivers.kubernetes.sandbox_runtime" -}} +{{- $_ := unset $config "openshell.drivers.kubernetes.sandbox_runtime.app_listener" -}} {{- $legacyDrivers := get $legacyServer "drivers" | default dict -}} {{- $legacyKubernetes := get $legacyDrivers "kubernetes" | default dict -}} {{- if hasKey $kubernetesCompat "resource_admission" -}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index 8af91e1901..5cf51ca971 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -119,7 +119,7 @@ rules: {{- if ne $workspaceMode "shared" }} - apiGroups: [""] resources: ["services"] - verbs: ["create", "get"] + verbs: ["create", "get"{{ if and (get (include "openshell.appListenerDeployment" . | fromYaml) "create_service") (not (include "openshell.supervisorEnabled" .)) }}, "patch", "delete"{{ end }}] - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 85d2e954d4..08ee2de873 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -61,7 +61,7 @@ rules: - watch - apiGroups: [""] resources: ["services"] - verbs: ["create", "get"] + verbs: ["create", "get"{{ if and (get (include "openshell.appListenerDeployment" . | fromYaml) "create_service") (not (include "openshell.supervisorEnabled" .)) }}, "patch", "delete"{{ end }}] - apiGroups: [""] resources: ["secrets"] # Bootstrap Secrets are generation-scoped. Recovery deletes them by exact diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index 205803481d..513f49d283 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -9,6 +9,16 @@ # SPDX-License-Identifier: Apache-2.0 {{- $shared := get $kubernetes "shared_supervisor" }} {{- $name := include "openshell.supervisorName" . }} +{{- $app := include "openshell.appListenerConfig" . | fromYaml }} +{{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml }} +{{- $appPort := get $app "port" | default 8443 }} +{{- if get $app "enabled" }} +{{- $_ := required "supervisor app listener requires domain" (get $app "domain") }} +{{- $_ := required "supervisor app listener requires tls_secret_name" (get $appDeployment "tls_secret_name") }} +{{- if or (lt (int $appPort) 1024) (gt (int $appPort) 65535) (eq (int $appPort) 5501) (eq (int $appPort) (int (get $shared "runtime_port"))) }} +{{- fail "supervisor app listener port must be in 1024..65535 and distinct from runtime and health ports" }} +{{- end }} +{{- end }} {{- $endpoint := get $kubernetes "grpc_endpoint" }} {{- $endpointUrl := urlParse $endpoint }} {{- $gatewayTls := and (not .Values.server.disableTls) .Values.server.tls.clientTlsSecretName }} @@ -86,6 +96,18 @@ spec: - --session-root - /run/openshell/sessions env: + {{- if get $app "enabled" }} + - name: OPENSHELL_APP_LISTENER_ENABLED + value: "true" + - name: OPENSHELL_APP_LISTEN_ADDRESS + value: {{ printf "0.0.0.0:%v" $appPort | quote }} + - name: OPENSHELL_APP_SERVICE_DOMAIN + value: {{ get $app "domain" | quote }} + - name: OPENSHELL_APP_TLS_CERT + value: /etc/openshell/app-tls/tls.crt + - name: OPENSHELL_APP_TLS_KEY + value: /etc/openshell/app-tls/tls.key + {{- end }} - name: OPENSHELL_ENDPOINT value: {{ $endpoint | quote }} - name: OPENSHELL_LOG_LEVEL @@ -110,6 +132,11 @@ spec: fieldRef: fieldPath: status.podIP ports: + {{- if get $app "enabled" }} + - name: app + containerPort: {{ $appPort }} + protocol: TCP + {{- end }} - name: runtime containerPort: {{ get $shared "runtime_port" }} protocol: TCP @@ -137,6 +164,11 @@ spec: tcpSocket: port: health volumeMounts: + {{- if get $app "enabled" }} + - name: app-tls + mountPath: /etc/openshell/app-tls + readOnly: true + {{- end }} - name: run mountPath: /run/openshell - name: logs @@ -150,6 +182,11 @@ spec: readOnly: true {{- end }} volumes: + {{- if get $app "enabled" }} + - name: app-tls + secret: + secretName: {{ get $appDeployment "tls_secret_name" | quote }} + {{- end }} - name: run emptyDir: {} - name: logs diff --git a/deploy/helm/openshell/tests/app_listener_rbac_test.yaml b/deploy/helm/openshell/tests/app_listener_rbac_test.yaml new file mode 100644 index 0000000000..ac6916a72e --- /dev/null +++ b/deploy/helm/openshell/tests/app_listener_rbac_test.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: app endpoint reconciliation permissions +templates: + - templates/role.yaml + - templates/clusterrole.yaml +tests: + - it: keeps default Service permissions unchanged + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["services"] + verbs: ["create", "get"] + template: templates/role.yaml + - it: permits dedicated endpoint reconciliation and cleanup even when disabled + set: + gatewayConfig: + openshell.supervisor.app_listener: + enabled: false + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + create_service: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["services"] + verbs: ["create", "get", "patch", "delete"] + template: templates/role.yaml + - it: grants managed workspace endpoint reconciliation permissions + set: + gatewayConfig: + openshell.drivers.kubernetes: + workspace_mode: managed + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + create_service: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["services"] + verbs: ["create", "get", "patch", "delete"] + template: templates/clusterrole.yaml + - it: does not grant dedicated endpoint mutation permissions to a shared supervisor deployment + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + create_service: true + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["services"] + verbs: ["create", "get"] + template: templates/role.yaml diff --git a/deploy/helm/openshell/tests/app_listener_test.yaml b/deploy/helm/openshell/tests/app_listener_test.yaml new file mode 100644 index 0000000000..a6d9e759b0 --- /dev/null +++ b/deploy/helm/openshell/tests/app_listener_test.yaml @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: supervisor app listener +templates: + - templates/supervisor.yaml +release: + name: openshell + namespace: gateway +tests: + - it: disabled configuration adds no socket port TLS mount or app Service + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.supervisor.app_listener: + enabled: false + port: 9443 + domain: services.example.com + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + tls_secret_name: app-tls + create_service: true + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: spec.template.spec.containers[0].ports + count: 2 + documentIndex: 0 + - notContains: + path: spec.template.spec.volumes + content: + name: app-tls + any: true + documentIndex: 0 + - it: enables the common nondefault app port without creating a shared app Service + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.supervisor.app_listener: + enabled: true + port: 9443 + domain: services.example.com + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + tls_secret_name: app-tls + create_service: true + asserts: + - hasDocuments: + count: 1 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_APP_LISTEN_ADDRESS + value: 0.0.0.0:9443 + documentIndex: 0 + - contains: + path: spec.template.spec.volumes + content: + name: app-tls + secret: + secretName: app-tls + documentIndex: 0 + - it: refuses app ports that collide with the runtime listener + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.supervisor.app_listener: + enabled: true + port: 5500 + domain: services.example.com + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + tls_secret_name: app-tls + asserts: + - failedTemplate: + errorMessage: supervisor app listener port must be in 1024..65535 and distinct from runtime and health ports diff --git a/deploy/helm/test-gateway-config-parser.sh b/deploy/helm/test-gateway-config-parser.sh index 664af825df..6aeca1f9f0 100755 --- a/deploy/helm/test-gateway-config-parser.sh +++ b/deploy/helm/test-gateway-config-parser.sh @@ -43,6 +43,13 @@ render "${work_dir}/default.yaml" extract_toml "${work_dir}/default.yaml" "${work_dir}/default.toml" preflight "${work_dir}/default.toml" +# Explicit app ingress must survive both template rendering and driver +# preflight; child options cannot extend a sealed TOML inline table. +render "${work_dir}/app-listener.yaml" \ + --values "${chart}/ci/values-app-listener.yaml" +extract_toml "${work_dir}/app-listener.yaml" "${work_dir}/app-listener.toml" +preflight "${work_dir}/app-listener.toml" + # Exercise all serializer shapes through a raw driver table and prove that # literal strings, TOML escaping, and null omission survive parsing. render "${work_dir}/shapes.yaml" --values "${fixture}" diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index ef49f8ec86..1075614cee 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1711,3 +1711,51 @@ Replace the executable and configuration paths with your installation's paths, a The command reports each selected executable path and version. Each tool receives only `-V`, with a five-second deadline and an 8 KiB output limit per stream. Missing, non-executable, outdated, or failing tools return a nonzero exit status with corrective guidance and the selected tool's error. The checks create no gateway state or images and do not start a driver or VM. Version checks confirm the installation; they do not test image creation or filesystem recovery. Other drivers do not require these VM tools. A VM configuration table alone does not select the VM driver. When a remote driver socket is configured, preflight reports that host tool checks were not performed; run the check on that host in the driver service's environment with a local VM configuration. + +## Supervisor application listener + +Enable a separate application TLS listener for declared sandbox HTTP and +WebSocket services. Application traffic reaches the supervisor directly and uses +authentication supplied by your application or ingress edge. Gateway service +ingress remains available independently. + +```toml +[openshell.supervisor.app_listener] +enabled = true +port = 9443 +domain = "services.example.com" + +[openshell.drivers.kubernetes.sandbox_runtime.app_listener] +tls_secret_name = "openshell-app-tls" +create_service = true +``` + +The listener defaults to disabled, with port `8443` and no domain. Port, domain, +or driver options alone do not enable it. Use an unprivileged port distinct from +the supervisor runtime and health ports. The common `port` controls both the +listener and its Kubernetes endpoint. + +On Kubernetes, provide a TLS Secret with `tls.crt` and `tls.key` matching +`*.services.example.com` in each supervisor namespace, and allow your ingress +clients to reach the app port through NetworkPolicy. The driver mounts the Secret +only in supervisor containers. Restart supervisors after changing certificates. +With `create_service = true`, dedicated supervisors get a sandbox-owned headless +`os-app-` Service. This option applies only to dedicated supervisors. +Shared supervisors expose the app listener directly on each member's Pod IP; +no shared app Service is created. Select the supervisor owning the sandbox +when routing application traffic. + +The Helm chart grants dedicated endpoint management the Service `patch` and +`delete` permissions when `create_service` is set. If you supply RBAC separately, +include those permissions alongside `create` and `get`. To remove existing +dedicated endpoints, keep `create_service = true` while disabling the listener +and recreating the sandbox runtime. Existing supervisor processes retain their +startup listener configuration until restarted. + +Expose services with the existing commands. For direct listener testing, an +unnamed service on sandbox `demo` in workspace `default` uses SNI and Host +`default--demo.services.example.com`; a service named `web` uses +`default--demo--web.services.example.com`. Configure DNS or your ingress to reach +the owning supervisor while preserving that hostname. Automatic discovery and +application URL publication are separate work; current CLI URLs still identify +the existing gateway path. diff --git a/e2e/kubernetes/app-listener/README.md b/e2e/kubernetes/app-listener/README.md new file mode 100644 index 0000000000..3e26d81db6 --- /dev/null +++ b/e2e/kubernetes/app-listener/README.md @@ -0,0 +1,77 @@ +# Supervisor application listener verification + +Use a Kubernetes development cluster with the current gateway and supervisor +images. This fixture exercises the listener without discovery or the reference +proxy. The opt-in Helm values are +`deploy/helm/openshell/ci/values-app-listener.yaml`. + +Before deployment, provide an operator-owned `openshell-app-tls` Secret in the +supervisor namespace with `tls.crt` and `tls.key` for +`*.services.example.com`. Keep private keys outside the checkout. Allow a labeled +test-client Pod to reach supervisor TCP port 9443 through a separate +NetworkPolicy. The normal runtime/control port rules do not grant app access. +Deploy through the repository's Helm development workflow with these values. + +Create two sandboxes running an HTTP/WebSocket test application bound to loopback +port 8080. Expose named and unnamed services using the existing CLI, including +`openshell service expose 8080 web`. CLI URLs still use gateway mode in +this stage. Use these direct-listener hostnames instead: + +- `default--.services.example.com` for the unnamed service. +- `default----web.services.example.com` for the named service. + +Inspect supervisor Pods. Dedicated supervisors optionally expose +`os-app-`; inspect that Service's EndpointSlices. Shared supervisors +have no app Service: select the Pod IP actually owning the sandbox and use its +configured app port. Do not select an arbitrary member when more than one is +present. Check ownership through supervisor admission/release logs in this +stage; automated owner selection belongs to the later discovery API. + +From a permitted client Pod, mount only the public app certificate/CA and run: + +```shell +curl --cacert /test/app-ca.crt \ + --connect-to default--demo--web.services.example.com:443:SUPERVISOR_POD_IP:9443 \ + https://default--demo--web.services.example.com/ +``` + +For a dedicated supervisor, repeat using its headless Service DNS name instead +of the Pod IP. Verify both paths through ordinary cluster networking. A +port-forward alone does not verify Service selection or NetworkPolicy. + +Verify HTTP and WebSockets, public Host/Origin, application session cookies, +unknown SNI, mismatched Host, undeclared services, bearer stripping/passthrough, +and lack of access to another sandbox's ports. Change or remove one service +while its WebSocket is open: its connection closes while an unrelated service's +connection survives. Recreate the sandbox and confirm old streams cannot revive. + +Exercise configuration polling and push modes. In push mode, leave configuration +unchanged for longer than 30 seconds and confirm ingress remains usable while +the session is healthy. Reject configuration and confirm ingress closes. Stop +configuration/session refresh and require routes and streams to expire after +the remaining freshness lease once session loss is detected. + +Run with listener configuration omitted, explicitly disabled despite complete +TLS/port options, and explicitly enabled. Disabled deployments have no app port, +TLS mount, or app Service. Retain the dedicated driver `create_service` option +while disabling and recreating a sandbox to verify cleanup of its previously +owned endpoint. Confirm gateway ingress still works in each mode. + +Dedicated endpoint reconciliation runs more than once during creation. Verify +that it accepts its own non-controller Sandbox owner reference and has Service +`patch`/`delete` permissions when `create_service` is configured. Foreign-owned +Services must be left untouched, and cleanup uses UID/resource-version +preconditions. + +For HA pool validation after integration with #4864, use at least two supervisor +members with distinct sandbox owners. Move one owner through graceful downscale +and abrupt loss. Existing streams may disconnect; after selecting the new owner, +new HTTP/WebSocket connections must work without restarting the workload. +Validate unchanged workload generation/process memory, rejection of old-owner +connectors, and continued service for the unaffected sandbox. A static ingress +must be updated manually in this stage; automated owner tracking belongs to +subsequent discovery work. + +The local Rust listener tests use a real TLS client and backend RPC connector. +They supplement this live cluster check; they do not establish HA pool recovery +or real Kubernetes reachability. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 80344bd4a1..b57c5d4cda 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -427,6 +427,8 @@ message SandboxConfigSnapshot { string configuration_instance_id = 15; string configuration_error = 16; uint64 settings_revision = 17; + // Explicit application ingress allowlist. Absent means no app routes. + repeated SandboxServiceRoute service_routes = 18; } // Response containing effective sandbox settings and policy, projected from @@ -477,6 +479,17 @@ message GetSandboxConfigResponse { string configuration_instance_id = 15; // Monotonic revision of the sandbox-scoped settings row. uint64 settings_revision = 17; + repeated SandboxServiceRoute service_routes = 18; +} + +// Gateway-declared service installed by a supervisor; this does not grant +// access to arbitrary workload ports or assert application health. +message SandboxServiceRoute { + string endpoint_id = 1; + string name = 2; + uint32 target_port = 3; + int32 authorization_mode = 4; + uint64 resource_version = 5; } // Connection details for one operator-registered supervisor middleware service. diff --git a/sdk/go/proto/sandboxv1/sandbox.pb.go b/sdk/go/proto/sandboxv1/sandbox.pb.go index 4b09b2016c..045e15fac5 100644 --- a/sdk/go/proto/sandboxv1/sandbox.pb.go +++ b/sdk/go/proto/sandboxv1/sandbox.pb.go @@ -2116,8 +2116,10 @@ type SandboxConfigSnapshot struct { ConfigurationInstanceId string `protobuf:"bytes,15,opt,name=configuration_instance_id,json=configurationInstanceId,proto3" json:"configuration_instance_id,omitempty"` ConfigurationError string `protobuf:"bytes,16,opt,name=configuration_error,json=configurationError,proto3" json:"configuration_error,omitempty"` SettingsRevision uint64 `protobuf:"varint,17,opt,name=settings_revision,json=settingsRevision,proto3" json:"settings_revision,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Explicit application ingress allowlist. Absent means no app routes. + ServiceRoutes []*SandboxServiceRoute `protobuf:"bytes,18,rep,name=service_routes,json=serviceRoutes,proto3" json:"service_routes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxConfigSnapshot) Reset() { @@ -2269,6 +2271,13 @@ func (x *SandboxConfigSnapshot) GetSettingsRevision() uint64 { return 0 } +func (x *SandboxConfigSnapshot) GetServiceRoutes() []*SandboxServiceRoute { + if x != nil { + return x.ServiceRoutes + } + return nil +} + // Response containing effective sandbox settings and policy, projected from // the SandboxConfigSnapshot that push supervisors receive. type GetSandboxConfigResponse struct { @@ -2317,7 +2326,8 @@ type GetSandboxConfigResponse struct { // Registration fence for a new supervisor; capture once and retain on retry. ConfigurationInstanceId string `protobuf:"bytes,15,opt,name=configuration_instance_id,json=configurationInstanceId,proto3" json:"configuration_instance_id,omitempty"` // Monotonic revision of the sandbox-scoped settings row. - SettingsRevision uint64 `protobuf:"varint,17,opt,name=settings_revision,json=settingsRevision,proto3" json:"settings_revision,omitempty"` + SettingsRevision uint64 `protobuf:"varint,17,opt,name=settings_revision,json=settingsRevision,proto3" json:"settings_revision,omitempty"` + ServiceRoutes []*SandboxServiceRoute `protobuf:"bytes,18,rep,name=service_routes,json=serviceRoutes,proto3" json:"service_routes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2471,6 +2481,91 @@ func (x *GetSandboxConfigResponse) GetSettingsRevision() uint64 { return 0 } +func (x *GetSandboxConfigResponse) GetServiceRoutes() []*SandboxServiceRoute { + if x != nil { + return x.ServiceRoutes + } + return nil +} + +// Gateway-declared service installed by a supervisor; this does not grant +// access to arbitrary workload ports or assert application health. +type SandboxServiceRoute struct { + state protoimpl.MessageState `protogen:"open.v1"` + EndpointId string `protobuf:"bytes,1,opt,name=endpoint_id,json=endpointId,proto3" json:"endpoint_id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + TargetPort uint32 `protobuf:"varint,3,opt,name=target_port,json=targetPort,proto3" json:"target_port,omitempty"` + AuthorizationMode int32 `protobuf:"varint,4,opt,name=authorization_mode,json=authorizationMode,proto3" json:"authorization_mode,omitempty"` + ResourceVersion uint64 `protobuf:"varint,5,opt,name=resource_version,json=resourceVersion,proto3" json:"resource_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SandboxServiceRoute) Reset() { + *x = SandboxServiceRoute{} + mi := &file_sandbox_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SandboxServiceRoute) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SandboxServiceRoute) ProtoMessage() {} + +func (x *SandboxServiceRoute) ProtoReflect() protoreflect.Message { + mi := &file_sandbox_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SandboxServiceRoute.ProtoReflect.Descriptor instead. +func (*SandboxServiceRoute) Descriptor() ([]byte, []int) { + return file_sandbox_proto_rawDescGZIP(), []int{23} +} + +func (x *SandboxServiceRoute) GetEndpointId() string { + if x != nil { + return x.EndpointId + } + return "" +} + +func (x *SandboxServiceRoute) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SandboxServiceRoute) GetTargetPort() uint32 { + if x != nil { + return x.TargetPort + } + return 0 +} + +func (x *SandboxServiceRoute) GetAuthorizationMode() int32 { + if x != nil { + return x.AuthorizationMode + } + return 0 +} + +func (x *SandboxServiceRoute) GetResourceVersion() uint64 { + if x != nil { + return x.ResourceVersion + } + return 0 +} + // Connection details for one operator-registered supervisor middleware service. // V1 supports plaintext and server-authenticated TLS gRPC. type SupervisorMiddlewareService struct { @@ -2510,7 +2605,7 @@ type SupervisorMiddlewareService struct { func (x *SupervisorMiddlewareService) Reset() { *x = SupervisorMiddlewareService{} - mi := &file_sandbox_proto_msgTypes[23] + mi := &file_sandbox_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2522,7 +2617,7 @@ func (x *SupervisorMiddlewareService) String() string { func (*SupervisorMiddlewareService) ProtoMessage() {} func (x *SupervisorMiddlewareService) ProtoReflect() protoreflect.Message { - mi := &file_sandbox_proto_msgTypes[23] + mi := &file_sandbox_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2535,7 +2630,7 @@ func (x *SupervisorMiddlewareService) ProtoReflect() protoreflect.Message { // Deprecated: Use SupervisorMiddlewareService.ProtoReflect.Descriptor instead. func (*SupervisorMiddlewareService) Descriptor() ([]byte, []int) { - return file_sandbox_proto_rawDescGZIP(), []int{23} + return file_sandbox_proto_rawDescGZIP(), []int{24} } func (x *SupervisorMiddlewareService) GetName() string { @@ -2755,7 +2850,7 @@ const file_sandbox_proto_rawDesc = "" + "\x05value\"\x86\x01\n" + "\x10EffectiveSetting\x128\n" + "\x05value\x18\x01 \x01(\v2\".openshell.sandbox.v1.SettingValueR\x05value\x128\n" + - "\x05scope\x18\x02 \x01(\x0e2\".openshell.sandbox.v1.SettingScopeR\x05scope\"\xd8\b\n" + + "\x05scope\x18\x02 \x01(\x0e2\".openshell.sandbox.v1.SettingScopeR\x05scope\"\xaa\t\n" + "\x15SandboxConfigSnapshot\x12;\n" + "\x06policy\x18\x01 \x01(\v2#.openshell.sandbox.v1.SandboxPolicyR\x06policy\x12\x18\n" + "\aversion\x18\x02 \x01(\rR\aversion\x12\x1f\n" + @@ -2775,10 +2870,11 @@ const file_sandbox_proto_rawDesc = "" + "\x19provider_attachment_epoch\x18\x0e \x01(\tR\x17providerAttachmentEpoch\x12:\n" + "\x19configuration_instance_id\x18\x0f \x01(\tR\x17configurationInstanceId\x12/\n" + "\x13configuration_error\x18\x10 \x01(\tR\x12configurationError\x12+\n" + - "\x11settings_revision\x18\x11 \x01(\x04R\x10settingsRevision\x1ac\n" + + "\x11settings_revision\x18\x11 \x01(\x04R\x10settingsRevision\x12P\n" + + "\x0eservice_routes\x18\x12 \x03(\v2).openshell.sandbox.v1.SandboxServiceRouteR\rserviceRoutes\x1ac\n" + "\rSettingsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12<\n" + - "\x05value\x18\x02 \x01(\v2&.openshell.sandbox.v1.EffectiveSettingR\x05value:\x028\x01\"\xde\b\n" + + "\x05value\x18\x02 \x01(\v2&.openshell.sandbox.v1.EffectiveSettingR\x05value:\x028\x01\"\xb0\t\n" + "\x18GetSandboxConfigResponse\x12;\n" + "\x06policy\x18\x01 \x01(\v2#.openshell.sandbox.v1.SandboxPolicyR\x06policy\x12\x18\n" + "\aversion\x18\x02 \x01(\rR\aversion\x12\x1f\n" + @@ -2798,10 +2894,19 @@ const file_sandbox_proto_rawDesc = "" + "\x16configuration_admitted\x18\r \x01(\bR\x15configurationAdmitted\x12/\n" + "\x13configuration_error\x18\x10 \x01(\tR\x12configurationError\x12:\n" + "\x19configuration_instance_id\x18\x0f \x01(\tR\x17configurationInstanceId\x12+\n" + - "\x11settings_revision\x18\x11 \x01(\x04R\x10settingsRevision\x1ac\n" + + "\x11settings_revision\x18\x11 \x01(\x04R\x10settingsRevision\x12P\n" + + "\x0eservice_routes\x18\x12 \x03(\v2).openshell.sandbox.v1.SandboxServiceRouteR\rserviceRoutes\x1ac\n" + "\rSettingsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12<\n" + - "\x05value\x18\x02 \x01(\v2&.openshell.sandbox.v1.EffectiveSettingR\x05value:\x028\x01\"\x81\x04\n" + + "\x05value\x18\x02 \x01(\v2&.openshell.sandbox.v1.EffectiveSettingR\x05value:\x028\x01\"\xc5\x01\n" + + "\x13SandboxServiceRoute\x12\x1f\n" + + "\vendpoint_id\x18\x01 \x01(\tR\n" + + "endpointId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n" + + "\vtarget_port\x18\x03 \x01(\rR\n" + + "targetPort\x12-\n" + + "\x12authorization_mode\x18\x04 \x01(\x05R\x11authorizationMode\x12)\n" + + "\x10resource_version\x18\x05 \x01(\x04R\x0fresourceVersion\"\x81\x04\n" + "\x1bSupervisorMiddlewareService\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12#\n" + "\rgrpc_endpoint\x18\x02 \x01(\tR\fgrpcEndpoint\x12*\n" + @@ -2858,7 +2963,7 @@ func file_sandbox_proto_rawDescGZIP() []byte { } var file_sandbox_proto_enumTypes = make([]protoimpl.EnumInfo, 7) -var file_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 34) +var file_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 35) var file_sandbox_proto_goTypes = []any{ (NetworkTlsMode)(0), // 0: openshell.sandbox.v1.NetworkTlsMode (NetworkEnforcementMode)(0), // 1: openshell.sandbox.v1.NetworkEnforcementMode @@ -2890,74 +2995,77 @@ var file_sandbox_proto_goTypes = []any{ (*EffectiveSetting)(nil), // 27: openshell.sandbox.v1.EffectiveSetting (*SandboxConfigSnapshot)(nil), // 28: openshell.sandbox.v1.SandboxConfigSnapshot (*GetSandboxConfigResponse)(nil), // 29: openshell.sandbox.v1.GetSandboxConfigResponse - (*SupervisorMiddlewareService)(nil), // 30: openshell.sandbox.v1.SupervisorMiddlewareService - nil, // 31: openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry - nil, // 32: openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry - nil, // 33: openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry - nil, // 34: openshell.sandbox.v1.L7DenyRule.QueryEntry - nil, // 35: openshell.sandbox.v1.L7DenyRule.ParamsEntry - nil, // 36: openshell.sandbox.v1.L7Allow.QueryEntry - nil, // 37: openshell.sandbox.v1.L7Allow.ParamsEntry - nil, // 38: openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry - nil, // 39: openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry - nil, // 40: openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry - (*structpb.Struct)(nil), // 41: google.protobuf.Struct - (*datamodelv1.WorkspaceSelector)(nil), // 42: openshell.datamodel.v1.WorkspaceSelector - (*durationpb.Duration)(nil), // 43: google.protobuf.Duration + (*SandboxServiceRoute)(nil), // 30: openshell.sandbox.v1.SandboxServiceRoute + (*SupervisorMiddlewareService)(nil), // 31: openshell.sandbox.v1.SupervisorMiddlewareService + nil, // 32: openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry + nil, // 33: openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry + nil, // 34: openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry + nil, // 35: openshell.sandbox.v1.L7DenyRule.QueryEntry + nil, // 36: openshell.sandbox.v1.L7DenyRule.ParamsEntry + nil, // 37: openshell.sandbox.v1.L7Allow.QueryEntry + nil, // 38: openshell.sandbox.v1.L7Allow.ParamsEntry + nil, // 39: openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry + nil, // 40: openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry + nil, // 41: openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry + (*structpb.Struct)(nil), // 42: google.protobuf.Struct + (*datamodelv1.WorkspaceSelector)(nil), // 43: openshell.datamodel.v1.WorkspaceSelector + (*durationpb.Duration)(nil), // 44: google.protobuf.Duration } var file_sandbox_proto_depIdxs = []int32{ 8, // 0: openshell.sandbox.v1.SandboxPolicy.filesystem:type_name -> openshell.sandbox.v1.FilesystemPolicy 9, // 1: openshell.sandbox.v1.SandboxPolicy.landlock:type_name -> openshell.sandbox.v1.LandlockPolicy 10, // 2: openshell.sandbox.v1.SandboxPolicy.process:type_name -> openshell.sandbox.v1.ProcessPolicy - 31, // 3: openshell.sandbox.v1.SandboxPolicy.network_policies:type_name -> openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry - 32, // 4: openshell.sandbox.v1.SandboxPolicy.network_middlewares:type_name -> openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry + 32, // 3: openshell.sandbox.v1.SandboxPolicy.network_policies:type_name -> openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry + 33, // 4: openshell.sandbox.v1.SandboxPolicy.network_middlewares:type_name -> openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry 15, // 5: openshell.sandbox.v1.NetworkPolicyRule.endpoints:type_name -> openshell.sandbox.v1.NetworkEndpoint 22, // 6: openshell.sandbox.v1.NetworkPolicyRule.binaries:type_name -> openshell.sandbox.v1.NetworkBinary - 41, // 7: openshell.sandbox.v1.NetworkMiddlewareConfig.config:type_name -> google.protobuf.Struct + 42, // 7: openshell.sandbox.v1.NetworkMiddlewareConfig.config:type_name -> google.protobuf.Struct 13, // 8: openshell.sandbox.v1.NetworkMiddlewareConfig.endpoints:type_name -> openshell.sandbox.v1.MiddlewareEndpointSelector 0, // 9: openshell.sandbox.v1.NetworkEndpoint.tls:type_name -> openshell.sandbox.v1.NetworkTlsMode 1, // 10: openshell.sandbox.v1.NetworkEndpoint.enforcement:type_name -> openshell.sandbox.v1.NetworkEnforcementMode 2, // 11: openshell.sandbox.v1.NetworkEndpoint.access:type_name -> openshell.sandbox.v1.NetworkAccessPreset 19, // 12: openshell.sandbox.v1.NetworkEndpoint.rules:type_name -> openshell.sandbox.v1.L7Rule 18, // 13: openshell.sandbox.v1.NetworkEndpoint.deny_rules:type_name -> openshell.sandbox.v1.L7DenyRule - 33, // 14: openshell.sandbox.v1.NetworkEndpoint.graphql_persisted_queries:type_name -> openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry + 34, // 14: openshell.sandbox.v1.NetworkEndpoint.graphql_persisted_queries:type_name -> openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry 16, // 15: openshell.sandbox.v1.NetworkEndpoint.mcp:type_name -> openshell.sandbox.v1.McpOptions 14, // 16: openshell.sandbox.v1.NetworkEndpoint.credential_binding:type_name -> openshell.sandbox.v1.NetworkCredentialBinding - 34, // 17: openshell.sandbox.v1.L7DenyRule.query:type_name -> openshell.sandbox.v1.L7DenyRule.QueryEntry - 35, // 18: openshell.sandbox.v1.L7DenyRule.params:type_name -> openshell.sandbox.v1.L7DenyRule.ParamsEntry + 35, // 17: openshell.sandbox.v1.L7DenyRule.query:type_name -> openshell.sandbox.v1.L7DenyRule.QueryEntry + 36, // 18: openshell.sandbox.v1.L7DenyRule.params:type_name -> openshell.sandbox.v1.L7DenyRule.ParamsEntry 20, // 19: openshell.sandbox.v1.L7Rule.allow:type_name -> openshell.sandbox.v1.L7Allow - 36, // 20: openshell.sandbox.v1.L7Allow.query:type_name -> openshell.sandbox.v1.L7Allow.QueryEntry - 37, // 21: openshell.sandbox.v1.L7Allow.params:type_name -> openshell.sandbox.v1.L7Allow.ParamsEntry - 42, // 22: openshell.sandbox.v1.GetSandboxConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 38, // 23: openshell.sandbox.v1.GetGatewayConfigResponse.settings:type_name -> openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry + 37, // 20: openshell.sandbox.v1.L7Allow.query:type_name -> openshell.sandbox.v1.L7Allow.QueryEntry + 38, // 21: openshell.sandbox.v1.L7Allow.params:type_name -> openshell.sandbox.v1.L7Allow.ParamsEntry + 43, // 22: openshell.sandbox.v1.GetSandboxConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 39, // 23: openshell.sandbox.v1.GetGatewayConfigResponse.settings:type_name -> openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry 26, // 24: openshell.sandbox.v1.EffectiveSetting.value:type_name -> openshell.sandbox.v1.SettingValue 3, // 25: openshell.sandbox.v1.EffectiveSetting.scope:type_name -> openshell.sandbox.v1.SettingScope 7, // 26: openshell.sandbox.v1.SandboxConfigSnapshot.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 39, // 27: openshell.sandbox.v1.SandboxConfigSnapshot.settings:type_name -> openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry + 40, // 27: openshell.sandbox.v1.SandboxConfigSnapshot.settings:type_name -> openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry 4, // 28: openshell.sandbox.v1.SandboxConfigSnapshot.policy_source:type_name -> openshell.sandbox.v1.PolicySource - 30, // 29: openshell.sandbox.v1.SandboxConfigSnapshot.supervisor_middleware_services:type_name -> openshell.sandbox.v1.SupervisorMiddlewareService - 7, // 30: openshell.sandbox.v1.GetSandboxConfigResponse.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 40, // 31: openshell.sandbox.v1.GetSandboxConfigResponse.settings:type_name -> openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry - 4, // 32: openshell.sandbox.v1.GetSandboxConfigResponse.policy_source:type_name -> openshell.sandbox.v1.PolicySource - 30, // 33: openshell.sandbox.v1.GetSandboxConfigResponse.supervisor_middleware_services:type_name -> openshell.sandbox.v1.SupervisorMiddlewareService - 43, // 34: openshell.sandbox.v1.SupervisorMiddlewareService.request_timeout:type_name -> google.protobuf.Duration - 5, // 35: openshell.sandbox.v1.SupervisorMiddlewareService.binding_policy:type_name -> openshell.sandbox.v1.SupervisorMiddlewareBindingPolicy - 6, // 36: openshell.sandbox.v1.SupervisorMiddlewareService.bindings:type_name -> openshell.sandbox.v1.SupervisorMiddlewareBinding - 11, // 37: openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry.value:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 12, // 38: openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry.value:type_name -> openshell.sandbox.v1.NetworkMiddlewareConfig - 17, // 39: openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry.value:type_name -> openshell.sandbox.v1.GraphqlOperation - 21, // 40: openshell.sandbox.v1.L7DenyRule.QueryEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher - 21, // 41: openshell.sandbox.v1.L7DenyRule.ParamsEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher - 21, // 42: openshell.sandbox.v1.L7Allow.QueryEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher - 21, // 43: openshell.sandbox.v1.L7Allow.ParamsEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher - 26, // 44: openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry.value:type_name -> openshell.sandbox.v1.SettingValue - 27, // 45: openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry.value:type_name -> openshell.sandbox.v1.EffectiveSetting - 27, // 46: openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry.value:type_name -> openshell.sandbox.v1.EffectiveSetting - 47, // [47:47] is the sub-list for method output_type - 47, // [47:47] is the sub-list for method input_type - 47, // [47:47] is the sub-list for extension type_name - 47, // [47:47] is the sub-list for extension extendee - 0, // [0:47] is the sub-list for field type_name + 31, // 29: openshell.sandbox.v1.SandboxConfigSnapshot.supervisor_middleware_services:type_name -> openshell.sandbox.v1.SupervisorMiddlewareService + 30, // 30: openshell.sandbox.v1.SandboxConfigSnapshot.service_routes:type_name -> openshell.sandbox.v1.SandboxServiceRoute + 7, // 31: openshell.sandbox.v1.GetSandboxConfigResponse.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 41, // 32: openshell.sandbox.v1.GetSandboxConfigResponse.settings:type_name -> openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry + 4, // 33: openshell.sandbox.v1.GetSandboxConfigResponse.policy_source:type_name -> openshell.sandbox.v1.PolicySource + 31, // 34: openshell.sandbox.v1.GetSandboxConfigResponse.supervisor_middleware_services:type_name -> openshell.sandbox.v1.SupervisorMiddlewareService + 30, // 35: openshell.sandbox.v1.GetSandboxConfigResponse.service_routes:type_name -> openshell.sandbox.v1.SandboxServiceRoute + 44, // 36: openshell.sandbox.v1.SupervisorMiddlewareService.request_timeout:type_name -> google.protobuf.Duration + 5, // 37: openshell.sandbox.v1.SupervisorMiddlewareService.binding_policy:type_name -> openshell.sandbox.v1.SupervisorMiddlewareBindingPolicy + 6, // 38: openshell.sandbox.v1.SupervisorMiddlewareService.bindings:type_name -> openshell.sandbox.v1.SupervisorMiddlewareBinding + 11, // 39: openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry.value:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 12, // 40: openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry.value:type_name -> openshell.sandbox.v1.NetworkMiddlewareConfig + 17, // 41: openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry.value:type_name -> openshell.sandbox.v1.GraphqlOperation + 21, // 42: openshell.sandbox.v1.L7DenyRule.QueryEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher + 21, // 43: openshell.sandbox.v1.L7DenyRule.ParamsEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher + 21, // 44: openshell.sandbox.v1.L7Allow.QueryEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher + 21, // 45: openshell.sandbox.v1.L7Allow.ParamsEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher + 26, // 46: openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry.value:type_name -> openshell.sandbox.v1.SettingValue + 27, // 47: openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry.value:type_name -> openshell.sandbox.v1.EffectiveSetting + 27, // 48: openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry.value:type_name -> openshell.sandbox.v1.EffectiveSetting + 49, // [49:49] is the sub-list for method output_type + 49, // [49:49] is the sub-list for method input_type + 49, // [49:49] is the sub-list for extension type_name + 49, // [49:49] is the sub-list for extension extendee + 0, // [0:49] is the sub-list for field type_name } func init() { file_sandbox_proto_init() } @@ -2978,7 +3086,7 @@ func file_sandbox_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_sandbox_proto_rawDesc), len(file_sandbox_proto_rawDesc)), NumEnums: 7, - NumMessages: 34, + NumMessages: 35, NumExtensions: 0, NumServices: 0, }, diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 29b4407563..d06e9e5551 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1576,3 +1576,35 @@ probes. Arguments after `--` validate the effective daemon invocation, including its command-line overrides. Preflight preserves every failed file. Do not advise users to delete or rewrite it automatically; back it up and follow the manual schema-v2 migration in the Gateway Configuration reference. + +## Supervisor application ingress + +When `[openshell.supervisor.app_listener]` is explicitly enabled, check its +`port` and `domain` separately from the gateway API and supervisor runtime ports. +Kubernetes TLS and Service options belong to +`[openshell.drivers.kubernetes.sandbox_runtime.app_listener]`: +`tls_secret_name` and `create_service`. Disabled listeners create no app TLS +mount or app port. TLS material is mounted only in supervisors and reloads on +supervisor restart. + +Inspect supervisor Pods, app Services, and EndpointSlices with `kubectl`. A +dedicated endpoint is `os-app-`. Shared supervisors have no app +Service: connect to the current sandbox owner's Pod IP and configured app port. +Preserve service SNI and Host +when using curl's `--connect-to`, and verify the application certificate. A +port-forward bypasses Service and NetworkPolicy forwarding; also test from a +permitted in-cluster client. Operators provide ingress network rules. + +Dedicated endpoint management needs Service `create`, `get`, `patch`, and +`delete` permissions. Helm adds the mutation verbs only when dedicated endpoint +management is configured. A repeated reconcile must accept the sandbox's +non-controller owner reference and refuse a foreign owner. When disabling, +retain `create_service` through runtime recreation to clean up owned app +Services; running supervisors retain their startup listener configuration. + +Service changes arrive through polling or push configuration. Only admitted +service declarations are installed. Polling and accepted live push sessions +renew a 30-second lease; rejected configuration or expired freshness closes +routes. App traffic uses application/edge authentication independently of gateway +user authentication. This listener stage does not add service discovery or +change the public URLs printed by the CLI.