diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 1289968d14..11dcb071f4 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -114,6 +114,14 @@ jobs: binary: openshell-sandbox triple: aarch64-unknown-linux-musl runner: linux-arm64-cpu8 + - package: openshell-sandbox-proxy + binary: openshell-sandbox-proxy + triple: x86_64-unknown-linux-gnu + runner: linux-amd64-cpu8 + - package: openshell-sandbox-proxy + binary: openshell-sandbox-proxy + triple: aarch64-unknown-linux-gnu + runner: linux-arm64-cpu8 - package: openshell-supervisor binary: openshell-supervisor triple: x86_64-unknown-linux-gnu diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index e43b5bf62e..799fb2df98 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -64,3 +64,12 @@ jobs: runs-on: linux-amd64-cpu8 timeout-minutes: 20 steps: *build_steps + + sandbox-proxy: + env: + COMPONENT: sandbox-proxy + BINARY: openshell-sandbox-proxy + TARGET_SUFFIX: unknown-linux-gnu + runs-on: linux-amd64-cpu8 + timeout-minutes: 20 + steps: *build_steps diff --git a/.github/workflows/package-release-binaries.yml b/.github/workflows/package-release-binaries.yml index fb8df4ada2..9124ecb228 100644 --- a/.github/workflows/package-release-binaries.yml +++ b/.github/workflows/package-release-binaries.yml @@ -54,6 +54,12 @@ jobs: - artifact: openshell-supervisor-aarch64-unknown-linux-gnu binary: openshell-supervisor package: supervisor-binary-linux-arm64 + - artifact: openshell-sandbox-proxy-x86_64-unknown-linux-gnu + binary: openshell-sandbox-proxy + package: sandbox-proxy-binary-linux-amd64 + - artifact: openshell-sandbox-proxy-aarch64-unknown-linux-gnu + binary: openshell-sandbox-proxy + package: sandbox-proxy-binary-linux-arm64 - artifact: openshell-driver-vm-x86_64-unknown-linux-gnu binary: openshell-driver-vm package: driver-vm-linux-amd64 diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index aec77972ee..543d934dfd 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -189,7 +189,7 @@ jobs: run: | set -euo pipefail REGISTRY="ghcr.io/nvidia/openshell" - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do echo "Tagging ${REGISTRY}/${component}:${{ github.sha }} as dev..." docker buildx imagetools create \ --prefer-index=false \ @@ -321,6 +321,13 @@ jobs: path: release/ merge-multiple: true + - name: Download sandbox proxy binary artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: sandbox-proxy-binary-* + path: release/ + merge-multiple: true + - name: Download sandbox binary artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -441,6 +448,10 @@ jobs: openshell-supervisor-x86_64-unknown-linux-gnu.tar.gz \ openshell-supervisor-aarch64-unknown-linux-gnu.tar.gz > openshell-supervisor-checksums-sha256.txt cat openshell-supervisor-checksums-sha256.txt + sha256sum -- \ + openshell-sandbox-proxy-x86_64-unknown-linux-gnu.tar.gz \ + openshell-sandbox-proxy-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-proxy-checksums-sha256.txt + cat openshell-sandbox-proxy-checksums-sha256.txt - name: Generate Homebrew formula run: | @@ -547,7 +558,9 @@ jobs: release/openshell-sandbox-x86_64-unknown-linux-musl.tar.gz release/openshell-sandbox-aarch64-unknown-linux-musl.tar.gz release/openshell-supervisor-x86_64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-proxy-x86_64-unknown-linux-gnu.tar.gz release/openshell-supervisor-aarch64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-proxy-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-apple-darwin.tar.gz @@ -560,6 +573,7 @@ jobs: release/openshell-gateway-checksums-sha256.txt release/openshell-sandbox-checksums-sha256.txt release/openshell-supervisor-checksums-sha256.txt + release/openshell-sandbox-proxy-checksums-sha256.txt release/openshell-prover-checksums-sha256.txt release-helm: diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index ba8493f510..207c33fdad 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -103,6 +103,13 @@ jobs: path: release/ merge-multiple: true + - name: Download sandbox proxy binary artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: sandbox-proxy-binary-* + path: release/ + merge-multiple: true + - name: Download sandbox binary artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -198,6 +205,10 @@ jobs: openshell-supervisor-x86_64-unknown-linux-gnu.tar.gz \ openshell-supervisor-aarch64-unknown-linux-gnu.tar.gz > openshell-supervisor-checksums-sha256.txt cat openshell-supervisor-checksums-sha256.txt + sha256sum -- \ + openshell-sandbox-proxy-x86_64-unknown-linux-gnu.tar.gz \ + openshell-sandbox-proxy-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-proxy-checksums-sha256.txt + cat openshell-sandbox-proxy-checksums-sha256.txt - name: Generate Homebrew formula run: | @@ -262,7 +273,7 @@ jobs: VERSION: ${{ inputs.semver }} run: | set -euo pipefail - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do image="ghcr.io/nvidia/openshell/${component}:${VERSION}" # Archives are build artifacts from this run, not externally signed images. nix shell --inputs-from . nixpkgs#skopeo -c \ @@ -296,7 +307,7 @@ jobs: run: | set -euo pipefail REGISTRY="ghcr.io/nvidia/openshell" - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do if [[ "${IS_PRERELEASE}" == "true" ]]; then echo "Tagging ${REGISTRY}/${component}:${SOURCE_TAG} as ${VERSION}..." docker buildx imagetools create \ @@ -429,7 +440,9 @@ jobs: release/openshell-sandbox-x86_64-unknown-linux-musl.tar.gz release/openshell-sandbox-aarch64-unknown-linux-musl.tar.gz release/openshell-supervisor-x86_64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-proxy-x86_64-unknown-linux-gnu.tar.gz release/openshell-supervisor-aarch64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-proxy-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz release/openshell-driver-vm-aarch64-apple-darwin.tar.gz @@ -442,6 +455,7 @@ jobs: release/openshell-gateway-checksums-sha256.txt release/openshell-sandbox-checksums-sha256.txt release/openshell-supervisor-checksums-sha256.txt + release/openshell-sandbox-proxy-checksums-sha256.txt release/openshell-prover-checksums-sha256.txt - name: Upload amd64 Debian prerelease package diff --git a/AGENTS.md b/AGENTS.md index 6323591822..d114f3dc3d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,7 @@ Do not rely on this file for a full inventory. The detailed public and contribut | `crates/openshell-conformance/` | CLI conformance library | Reusable driver-agnostic scenarios and command runner | | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | | `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O | +| `crates/openshell-sandbox-proxy/` | Optional application proxy | Discovery-driven TLS passthrough directly to supervisor app listeners | | `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking | | `crates/openshell-supervisor-control/` | Shared supervisor control | Gateway-owned assignment controller, launch registrations, recovery coordination, and host observations for a supervisor shared by many sandboxes | | `crates/openshell-binary-identity/` | Binary identity | Shared trusted procfs executable identity resolution for isolation backends | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b1034dd23f..d149f29443 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -344,6 +344,8 @@ still miss across worktrees. ## Container Image Base Overrides +The optional application proxy image can be built with `mise run docker:build:sandbox-proxy`. Its standalone deployment chart is in `deploy/helm/openshell-sandbox-proxy`. + The gateway and supervisor Dockerfiles accept `GATEWAY_BASE_IMAGE` and `SUPERVISOR_BASE_IMAGE` build arguments. Omitting them keeps the pinned distroless defaults. To build with your own bases, pass the argument through diff --git a/Cargo.lock b/Cargo.lock index c165c8e418..038be08553 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4963,6 +4963,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "openshell-sandbox-proxy" +version = "0.0.0" +dependencies = [ + "axum", + "clap", + "openshell-core", + "openshell-crypto", + "openshell-sdk", + "rustls", + "tokio", + "tonic", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" diff --git a/crates/openshell-sandbox-proxy/Cargo.toml b/crates/openshell-sandbox-proxy/Cargo.toml new file mode 100644 index 0000000000..1951a176b0 --- /dev/null +++ b/crates/openshell-sandbox-proxy/Cargo.toml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-sandbox-proxy" +description = "Optional discovery-driven TLS passthrough proxy for sandbox applications" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core" } +openshell-sdk = { path = "../openshell-sdk" } +axum = { workspace = true } +clap = { workspace = true } +rustls = { workspace = true } +tokio = { workspace = true } +tonic = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } +openshell-crypto = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-sandbox-proxy/README.md b/crates/openshell-sandbox-proxy/README.md new file mode 100644 index 0000000000..a14b7683a8 --- /dev/null +++ b/crates/openshell-sandbox-proxy/README.md @@ -0,0 +1,94 @@ + + +# Sandbox application proxy + +`openshell-sandbox-proxy` is an optional consumer of the gateway service discovery +API. It selects the exact discovered hostname from TLS SNI and forwards TLS bytes +to the owning supervisor's app listener. Application certificates and private +keys remain at supervisors. HTTP, WebSockets, application authentication, and +sandbox policy remain enforced by the existing listener and application. + +```shell +openshell-sandbox-proxy \ + --gateway-endpoint https://gateway.example.com \ + --workspace default \ + --gateway-ca /run/discovery/ca.crt \ + --discovery-token-file /run/discovery/token \ + --listen-address 0.0.0.0:8443 +``` + +Repeat `--workspace` for a finite set (up to 64 distinct names), or use +`--all-workspaces` to include every active workspace, including future ones. +These modes are mutually exclusive. For example: + +```shell +openshell-sandbox-proxy --gateway-endpoint https://gateway.example.com \ + --workspace team-a --workspace team-b \ + --discovery-client-cert /run/discovery/tls.crt \ + --discovery-client-key /run/discovery/tls.key --gateway-ca /run/discovery/ca.crt +``` + +Use a discovery-only `OU=discovery` certificate with an administrator-managed +fingerprint grant, or an existing identity authorized for the selection. Client mTLS can +use `--discovery-client-cert` and `--discovery-client-key`. Credentials and CA +files reload on reconnect. The gateway must authorize every selected workspace; +a finite grant cannot request platform scope. Unauthorized selections fail as a +whole. All-workspaces output is bounded by the gateway's 64-workspace scan and +shared route/byte limits. The proxy does not require operator credentials. Plaintext +credential-free loopback development requires `--allow-insecure-gateway`. + +A snapshot initializes the cache and bounded watches replace it. Reconnects +relist; transient errors retain routes only until their published expiry. +Authorization/configuration/data-integrity failures clear the cache. Monotonic +local deadlines are derived from absolute expiries and capped at fifteen seconds; +operators must synchronize gateway and proxy clocks. Snapshot revisions are +opaque and installation versions identify replacement independently of renewal. + +Only exact, current routes can dial a backend. A route withdrawal, installation +replacement, or expiry closes affected TCP streams; renewal preserves them. +There is no gateway payload fallback, wildcard route fallback, or caller-selected +backend. Limits are 4,096 cached routes, 1,024 concurrent connections, 64 per +hostname, a 64 KiB ClientHello, and five-second hello/connect/setup deadlines. +Each relay uses Tokio's bounded bidirectional copy buffers. ClientHello parsing +is dependency-owned rustls protocol parsing, not a first-party cryptographic +operation. Authenticated discovery transport uses the shared SDK/crypto facade. + +The health listener defaults to port 9091: `/healthz` is process liveness, +`/readyz` reflects discovery availability, and `/metrics` exposes connection +counts. During transient discovery failures readiness fails while existing +unexpired connections can continue. Expiry still closes streams independently +of the discovery connection. Multiple replicas keep independent caches and need +no Kubernetes ownership or leader election. + +Build with `mise run docker:build:sandbox-proxy`. The standalone Helm chart at +`deploy/helm/openshell-sandbox-proxy` mounts only discovery identity/trust Secrets; +it needs no Kubernetes API permissions. Configure NetworkPolicy to allow the +proxy to reach the gateway API and discovered supervisor app endpoints, and +restrict supervisor ingress where edge authentication is required. The proxy +never authenticates application users using its discovery identity. + +For Gateway API TLS passthrough, attach a TLSRoute to a Gateway listener in +`Passthrough` mode and target the proxy Service on port 443. Application TLS +terminates at the supervisor. An HTTPRoute requires a separate edge termination +and backend-TLS contract. A controller can instead consume the same discovery +API and reconcile exact-host routes directly to supervisor endpoints; it does +not need this proxy. + +## Verification + +```shell +cargo test -p openshell-sandbox-proxy +cargo clippy -p openshell-sandbox-proxy --all-targets -- -D warnings +helm lint deploy/helm/openshell-sandbox-proxy \ + --set gatewayEndpoint=https://gateway.example.com --set workspace=default +``` + +Kubernetes discovery resolves dedicated supervisors and acknowledged shared-pool +owners. During HA takeover the proxy withdraws the old installation and connects +to the new owner after it advertises an accepted installation. Existing streams +may close; clients reconnect without restarting the sandbox. Hashed multi-gateway hostnames require +the common listener/discovery hostname integration; the proxy consumes returned +hostnames and never reconstructs them. diff --git a/crates/openshell-sandbox-proxy/src/cache.rs b/crates/openshell-sandbox-proxy/src/cache.rs new file mode 100644 index 0000000000..78bfc2f406 --- /dev/null +++ b/crates/openshell-sandbox-proxy/src/cache.rs @@ -0,0 +1,205 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use openshell_core::proto::ServiceRouteSnapshot; +use tokio::sync::{Semaphore, watch}; +use tokio::time::Instant; + +const MAX_ROUTES: usize = 4096; +const MAX_METADATA_BYTES: usize = 4 * 1024 * 1024; +const MAX_LEASE: Duration = Duration::from_secs(15); + +#[derive(Clone)] +pub struct Route { + pub endpoint: SocketAddr, + pub version: String, + pub deadline: Instant, + pub connections: Arc, +} + +pub type Routes = BTreeMap; +pub type RouteCache = watch::Sender; + +pub fn valid_hostname(host: &str) -> bool { + !host.is_empty() + && host.len() <= 253 + && host.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + }) +} + +pub fn valid_workspace(name: &str) -> bool { + valid_hostname(name) && name.len() <= 19 && !name.contains('.') +} + +#[cfg(test)] +fn replace( + cache: &RouteCache, + snapshot: ServiceRouteSnapshot, + workspace: &str, +) -> Result<(), &'static str> { + replace_selected(cache, snapshot, &[workspace.to_string()], false) +} + +/// A complete view is validated before replacement. Receipt never extends a +/// gateway's absolute expiry; a monotonic deadline prevents local clock rollback. +pub fn replace_selected( + cache: &RouteCache, + snapshot: ServiceRouteSnapshot, + workspaces: &[String], + all_workspaces: bool, +) -> Result<(), &'static str> { + if snapshot.routes.len() > MAX_ROUTES || snapshot.revision.is_empty() { + return Err("invalid snapshot bounds or revision"); + } + let now_ms = i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| "invalid system clock")? + .as_millis(), + ) + .map_err(|_| "invalid system clock")?; + let now = Instant::now(); + let old = cache.borrow(); + let mut next = Routes::new(); + let mut metadata_bytes = snapshot.revision.len(); + for route in snapshot.routes { + metadata_bytes = metadata_bytes + .saturating_add(route.hostname.len()) + .saturating_add(route.installation_version.len()) + .saturating_add(route.workspace.len()) + .saturating_add(route.sandbox.len()) + .saturating_add(route.name.len()); + if metadata_bytes > MAX_METADATA_BYTES + || !valid_workspace(&route.workspace) + || (!all_workspaces && !workspaces.contains(&route.workspace)) + || !valid_hostname(&route.hostname) + || route.installation_version.is_empty() + { + return Err("invalid discovered route identity"); + } + let endpoint = route.endpoint.ok_or("missing supervisor endpoint")?; + let host: IpAddr = endpoint.host.parse().map_err(|_| "invalid supervisor IP")?; + let port = u16::try_from(endpoint.port) + .ok() + .filter(|port| *port != 0) + .ok_or("invalid supervisor port")?; + let expiry = route + .expires_at + .as_ref() + .and_then(|value| openshell_core::time::timestamp_to_millis(value).ok()) + .ok_or("invalid installation expiry")?; + let remaining = expiry.saturating_sub(now_ms); + if remaining <= 0 { + continue; + } + let deadline = now + Duration::from_millis(remaining.unsigned_abs()).min(MAX_LEASE); + let connections = old.get(&route.hostname).map_or_else( + || Arc::new(Semaphore::new(64)), + |previous| previous.connections.clone(), + ); + if next + .insert( + route.hostname, + Route { + endpoint: SocketAddr::new(host, port), + version: route.installation_version, + deadline, + connections, + }, + ) + .is_some() + { + return Err("duplicate discovered hostname"); + } + } + drop(old); + cache.send_replace(next); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::{DiscoveredServiceRoute, ServiceRouteEndpoint}; + + fn snapshot() -> ServiceRouteSnapshot { + let now = openshell_core::time::timestamp_from_millis(crate::now_ms() + 10_000).unwrap(); + ServiceRouteSnapshot { + revision: "revision".into(), + routes: vec![DiscoveredServiceRoute { + workspace: "default".into(), + sandbox: "demo".into(), + name: "web".into(), + hostname: "demo.example.com".into(), + installation_version: "owner-1".into(), + endpoint: Some(ServiceRouteEndpoint { + host: "127.0.0.1".into(), + port: 9443, + }), + expires_at: Some(now), + ..Default::default() + }], + } + } + + #[test] + fn renewal_preserves_capacity_and_errors_do_not_publish_partial_views() { + let (cache, _) = watch::channel(Routes::new()); + replace(&cache, snapshot(), "default").unwrap(); + let connections = cache.borrow()["demo.example.com"].connections.clone(); + replace(&cache, snapshot(), "default").unwrap(); + assert!(Arc::ptr_eq( + &connections, + &cache.borrow()["demo.example.com"].connections + )); + let mut duplicate = snapshot(); + duplicate.routes.push(duplicate.routes[0].clone()); + assert!(replace(&cache, duplicate, "default").is_err()); + assert_eq!(cache.borrow().len(), 1); + assert!(replace(&cache, snapshot(), "other").is_err()); + let mut expired = snapshot(); + expired.routes[0].expires_at = + openshell_core::time::timestamp_from_millis(crate::now_ms() - 1).ok(); + replace(&cache, expired, "default").unwrap(); + assert!(cache.borrow().is_empty()); + } + #[test] + fn combined_views_validate_every_workspace_before_atomic_replacement() { + let (cache, _) = watch::channel(Routes::new()); + let mut view = snapshot(); + let mut other = view.routes[0].clone(); + other.workspace = "team-b".into(); + other.hostname = "team-b.example.com".into(); + view.routes.push(other); + let scopes = vec!["default".to_string(), "team-b".to_string()]; + replace_selected(&cache, view.clone(), &scopes, false).unwrap(); + assert_eq!(cache.borrow().len(), 2); + let previous = cache.borrow()["team-b.example.com"].connections.clone(); + replace_selected(&cache, view.clone(), &[], true).unwrap(); + assert!(Arc::ptr_eq( + &previous, + &cache.borrow()["team-b.example.com"].connections + )); + view.routes[1].workspace = "outside".into(); + assert!(replace_selected(&cache, view, &scopes, false).is_err()); + assert_eq!(cache.borrow().len(), 2); + let empty = ServiceRouteSnapshot { + revision: "empty".into(), + routes: vec![], + }; + replace_selected(&cache, empty, &[], true).unwrap(); + assert!(cache.borrow().is_empty()); + } +} diff --git a/crates/openshell-sandbox-proxy/src/discovery.rs b/crates/openshell-sandbox-proxy/src/discovery.rs new file mode 100644 index 0000000000..54709b6274 --- /dev/null +++ b/crates/openshell-sandbox-proxy/src/discovery.rs @@ -0,0 +1,306 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, atomic::Ordering}; +use std::time::Duration; + +use clap::Parser; +use openshell_core::proto::{ + GetServiceRouteSnapshotRequest, WatchServiceRoutesRequest, workspace_selector, +}; +use openshell_sdk::{AuthConfig, ClientConfig, ClientIdentity, OpenShellClient}; +use tonic::{Code, Status}; + +use crate::{Health, cache}; + +#[derive(Clone, Parser)] +#[command( + version, + about = "Route application TLS directly to discovered supervisor listeners" +)] +pub struct Args { + #[arg(long)] + pub gateway_endpoint: String, + /// Repeat for a finite set. Exclusive with --all-workspaces. + #[arg( + long, + required_unless_present = "all_workspaces", + conflicts_with = "all_workspaces" + )] + pub workspace: Vec, + /// Discover all active workspaces, including newly created ones. + #[arg(long, conflicts_with = "workspace")] + pub all_workspaces: bool, + #[arg(long, default_value = "0.0.0.0:8443")] + pub listen_address: std::net::SocketAddr, + #[arg(long, default_value = "0.0.0.0:9091")] + pub health_address: std::net::SocketAddr, + #[arg(long)] + pub gateway_ca: Option, + #[arg(long, requires = "discovery_client_key")] + pub discovery_client_cert: Option, + #[arg(long, requires = "discovery_client_cert")] + pub discovery_client_key: Option, + #[arg(long)] + pub discovery_token_file: Option, + /// Permit a plaintext loopback development gateway without credentials. + #[arg(long)] + pub allow_insecure_gateway: bool, +} + +async fn read_file(path: &Path) -> Result, Status> { + let metadata = tokio::fs::metadata(path) + .await + .map_err(|_| Status::unavailable("credential file unavailable"))?; + if metadata.len() > 1024 * 1024 { + return Err(Status::invalid_argument("credential file too large")); + } + tokio::fs::read(path) + .await + .map_err(|_| Status::unavailable("credential file unavailable")) +} + +impl Args { + pub fn validate(&self) -> Result<(), &'static str> { + let gateway = url::Url::parse(&self.gateway_endpoint).map_err(|_| "invalid gateway URL")?; + if !gateway.username().is_empty() + || gateway.password().is_some() + || gateway.query().is_some() + || gateway.fragment().is_some() + || gateway.path() != "/" + || self.all_workspaces != self.workspace.is_empty() + || self.workspace.len() > 64 + || self + .workspace + .iter() + .any(|name| !cache::valid_workspace(name)) + || self + .workspace + .iter() + .collect::>() + .len() + != self.workspace.len() + { + return Err("invalid gateway origin or workspace"); + } + if gateway.scheme() != "https" { + let loopback = gateway.host_str().is_some_and(|host| { + host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + }); + if gateway.scheme() != "http" + || !loopback + || !self.allow_insecure_gateway + || self.discovery_token_file.is_some() + || self.discovery_client_cert.is_some() + || self.discovery_client_key.is_some() + || self.gateway_ca.is_some() + { + return Err( + "discovery requires HTTPS; plaintext is limited to credential-free loopback development", + ); + } + } + if self.discovery_token_file.is_some() + && (self.discovery_client_cert.is_some() || self.discovery_client_key.is_some()) + { + return Err("choose bearer credentials or a discovery client certificate"); + } + if self.discovery_client_cert.is_some() != self.discovery_client_key.is_some() { + return Err("client certificate and key must be configured together"); + } + Ok(()) + } + + fn selector(&self) -> Option { + if self.all_workspaces { + Some(openshell_core::proto::all_workspaces_selector()) + } else if self.workspace.len() == 1 { + Some(workspace_selector(&self.workspace[0])) + } else { + None + } + } + + fn selected_workspaces(&self) -> Vec { + if self.workspace.len() > 1 { + self.workspace.clone() + } else { + Vec::new() + } + } + + /// Files are read on every reconnect so mounted credentials can rotate. + async fn client(&self) -> Result { + let mut config = ClientConfig::new(&self.gateway_endpoint); + if let Some(path) = &self.gateway_ca { + config.ca_cert = Some(read_file(path).await?); + } + if let (Some(cert), Some(key)) = (&self.discovery_client_cert, &self.discovery_client_key) { + config.client_identity = Some(ClientIdentity::new( + read_file(cert).await?, + read_file(key).await?, + )); + } + if let Some(path) = &self.discovery_token_file { + let bytes = read_file(path).await?; + let token = std::str::from_utf8(&bytes) + .map_err(|_| Status::invalid_argument("invalid token encoding"))? + .trim(); + if token.is_empty() { + return Err(Status::invalid_argument("empty discovery token")); + } + config.auth = Some(AuthConfig::oidc(token)); + } + OpenShellClient::connect(config) + .await + .map_err(|_| Status::unavailable("gateway connection failed")) + } +} + +async fn consume(args: &Args, routes: &cache::RouteCache, health: &Health) -> Result<(), Status> { + let client = args.client().await?; + let mut grpc = client.raw_grpc(); + let request = GetServiceRouteSnapshotRequest { + workspace_scope: args.selector(), + workspaces: args.selected_workspaces(), + }; + let initial = tokio::time::timeout( + Duration::from_secs(10), + grpc.get_service_route_snapshot(request), + ) + .await + .map_err(|_| Status::deadline_exceeded("snapshot timeout"))?? + .into_inner() + .snapshot + .ok_or_else(|| Status::data_loss("snapshot missing"))?; + cache::replace_selected(routes, initial, &args.workspace, args.all_workspaces) + .map_err(Status::data_loss)?; + health.discovery_ready.store(true, Ordering::Release); + let mut stream = grpc + .watch_service_routes(WatchServiceRoutesRequest { + workspace_scope: args.selector(), + workspaces: args.selected_workspaces(), + }) + .await? + .into_inner(); + loop { + // The gateway ends bounded watches after one minute. A deadline also + // protects against a stalled intermediary that never closes the stream. + let message = tokio::time::timeout(Duration::from_secs(70), stream.message()) + .await + .map_err(|_| Status::deadline_exceeded("watch stalled"))??; + let Some(message) = message else { + return Ok(()); + }; + cache::replace_selected( + routes, + message + .snapshot + .ok_or_else(|| Status::data_loss("snapshot missing"))?, + &args.workspace, + args.all_workspaces, + ) + .map_err(Status::data_loss)?; + health.discovery_ready.store(true, Ordering::Release); + } +} + +pub async fn run(args: Args, routes: cache::RouteCache, health: Arc) { + let mut backoff = Duration::from_secs(1); + loop { + let result = consume(&args, &routes, &health).await; + health.discovery_ready.store(false, Ordering::Release); + match result { + Ok(()) => { + backoff = Duration::from_secs(1); + tokio::time::sleep(backoff).await; + } + Err(error) => { + if matches!( + error.code(), + Code::PermissionDenied + | Code::Unauthenticated + | Code::InvalidArgument + | Code::FailedPrecondition + | Code::DataLoss + ) { + routes.send_replace(cache::Routes::new()); + } + tracing::warn!(code = ?error.code(), "discovery reconnect required"); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(Duration::from_secs(30)); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn credentials_require_tls_and_workspace_selection_is_explicit() { + let args = |endpoint: &str| { + Args::try_parse_from([ + "proxy", + "--gateway-endpoint", + endpoint, + "--workspace", + "default", + "--allow-insecure-gateway", + ]) + .unwrap() + }; + assert!(args("https://gateway.example.com").validate().is_ok()); + assert!(args("http://localhost:8080").validate().is_ok()); + assert!(args("http://gateway.example.com").validate().is_err()); + let mut plaintext = args("http://localhost:8080"); + plaintext.discovery_token_file = Some("/run/token".into()); + assert!(plaintext.validate().is_err()); + assert!( + args("https://user:secret@gateway.example.com") + .validate() + .is_err() + ); + assert!( + Args::try_parse_from(["proxy", "--gateway-endpoint", "https://gateway.example.com"]) + .is_err() + ); + } + #[test] + fn workspace_modes_are_exclusive_bounded_and_preserve_single_workspace_wire_calls() { + let parse = |flags: &[&str]| { + let mut input = vec!["proxy", "--gateway-endpoint", "https://gateway.example.com"]; + input.extend_from_slice(flags); + Args::try_parse_from(input) + }; + let single = parse(&["--workspace", "team-a"]).unwrap(); + assert!(single.validate().is_ok()); + assert_eq!( + single.selector().unwrap().selection, + Some(workspace_selector::Selection::Workspace("team-a".into())) + ); + let set = parse(&["--workspace", "team-a", "--workspace", "team-b"]).unwrap(); + assert!(set.validate().is_ok()); + assert!(set.selector().is_none()); + assert_eq!(set.selected_workspaces(), vec!["team-a", "team-b"]); + let all = parse(&["--all-workspaces"]).unwrap(); + assert!(all.validate().is_ok()); + assert!(all.workspace.is_empty()); + assert!(all.selector().is_some()); + assert!(parse(&["--workspace", "team-a", "--all-workspaces"]).is_err()); + assert!( + parse(&["--workspace", "team-a", "--workspace", "team-a"]) + .unwrap() + .validate() + .is_err() + ); + let mut too_many = set; + too_many.workspace = (0..65).map(|i| format!("team-{i}")).collect(); + assert!(too_many.validate().is_err()); + } +} diff --git a/crates/openshell-sandbox-proxy/src/main.rs b/crates/openshell-sandbox-proxy/src/main.rs new file mode 100644 index 0000000000..1e22e07c76 --- /dev/null +++ b/crates/openshell-sandbox-proxy/src/main.rs @@ -0,0 +1,376 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! TLS passthrough: discovery credentials never enter application streams. + +mod cache; +mod discovery; + +use std::io::{self, Cursor}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, +}; +use std::time::Duration; + +use axum::{Router, extract::State, http::StatusCode, routing::get}; +use clap::Parser as _; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::{Semaphore, watch}; +use tokio::task::JoinSet; +use tokio::time::Instant; + +const SETUP_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_CLIENT_HELLO: usize = 64 * 1024; + +#[derive(Default)] +pub struct Health { + discovery_ready: AtomicBool, + accepted: AtomicU64, + rejected: AtomicU64, + active: AtomicU64, +} + +#[cfg(test)] +fn now_ms() -> i64 { + i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_millis(), + ) + .expect("bounded timestamp") +} + +fn denied(message: &'static str) -> io::Error { + io::Error::new(io::ErrorKind::PermissionDenied, message) +} + +/// rustls owns `ClientHello` parsing. No cryptographic handshake or TLS termination +/// happens here; every received byte is forwarded unchanged after SNI selection. +async fn client_hello(stream: &mut TcpStream) -> io::Result<(String, Vec)> { + let mut acceptor = rustls::server::Acceptor::default(); + let mut hello = Vec::new(); + loop { + let mut bytes = [0; 4096]; + let count = stream.read(&mut bytes).await?; + if count == 0 { + return Err(denied("incomplete ClientHello")); + } + if hello.len() + count > MAX_CLIENT_HELLO { + return Err(denied("ClientHello too large")); + } + hello.extend_from_slice(&bytes[..count]); + acceptor.read_tls(&mut Cursor::new(&bytes[..count]))?; + if let Some(accepted) = acceptor + .accept() + .map_err(|_| denied("invalid ClientHello"))? + { + let hostname = accepted + .client_hello() + .server_name() + .filter(|name| cache::valid_hostname(name)) + .ok_or_else(|| denied("service SNI required"))? + .to_string(); + return Ok((hostname, hello)); + } + } +} + +async fn connection( + mut incoming: TcpStream, + mut updates: watch::Receiver, +) -> io::Result<()> { + openshell_core::net::set_tcp_nodelay_best_effort(&incoming); + let (host, hello) = tokio::time::timeout(SETUP_TIMEOUT, client_hello(&mut incoming)) + .await + .map_err(|_| denied("ClientHello timeout"))??; + let route = updates + .borrow() + .get(&host) + .filter(|route| route.deadline > Instant::now()) + .cloned() + .ok_or_else(|| denied("unknown or expired service"))?; + let _permit = route + .connections + .clone() + .try_acquire_owned() + .map_err(|_| denied("service connection limit"))?; + let mut outgoing = tokio::time::timeout( + SETUP_TIMEOUT, + openshell_core::net::connect_tcp_nodelay_best_effort(&[route.endpoint]), + ) + .await + .map_err(|_| denied("backend connect timeout"))??; + let still_current = updates.borrow().get(&host).is_some_and(|current| { + current.version == route.version + && current.endpoint == route.endpoint + && current.deadline > Instant::now() + }); + if !still_current || route.deadline <= Instant::now() { + return Err(denied("installation changed during setup")); + } + tokio::time::timeout_at( + route.deadline.min(Instant::now() + SETUP_TIMEOUT), + outgoing.write_all(&hello), + ) + .await + .map_err(|_| denied("backend setup timeout"))??; + let transfer = tokio::io::copy_bidirectional(&mut incoming, &mut outgoing); + tokio::pin!(transfer); + let mut deadline = route.deadline; + loop { + // Even a queued renewal cannot resurrect a connection after its + // previously observed lease has lapsed. + if Instant::now() >= deadline { + return Ok(()); + } + let current = updates.borrow().get(&host).cloned(); + let Some(current) = current.filter(|current| { + current.version == route.version + && current.endpoint == route.endpoint + && current.deadline > Instant::now() + }) else { + return Ok(()); + }; + deadline = current.deadline; + tokio::select! { + result = &mut transfer => return result.map(|_| ()), + () = tokio::time::sleep_until(deadline) => return Ok(()), + result = updates.changed() => { if result.is_err() { return Ok(()); } } + } + } +} + +async fn serve( + listener: TcpListener, + routes: cache::RouteCache, + health: Arc, +) -> io::Result<()> { + let capacity = Arc::new(Semaphore::new(1024)); + let mut connections = JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let (stream, _) = accepted?; + let Ok(permit) = capacity.clone().try_acquire_owned() else { + health.rejected.fetch_add(1, Ordering::Relaxed); + continue; + }; + let updates = routes.subscribe(); + let health = health.clone(); + connections.spawn(async move { + let _permit = permit; + health.accepted.fetch_add(1, Ordering::Relaxed); + health.active.fetch_add(1, Ordering::Relaxed); + let result = connection(stream, updates).await; + health.active.fetch_sub(1, Ordering::Relaxed); + if result.is_err() { health.rejected.fetch_add(1, Ordering::Relaxed); } + }); + } + _ = connections.join_next(), if !connections.is_empty() => {} + } + } +} + +async fn readiness(State(health): State>) -> StatusCode { + if health.discovery_ready.load(Ordering::Acquire) { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } +} + +async fn metrics(State(health): State>) -> String { + format!( + "# TYPE openshell_proxy_connections_total counter\nopenshell_proxy_connections_total {}\n# TYPE openshell_proxy_rejected_total counter\nopenshell_proxy_rejected_total {}\n# TYPE openshell_proxy_active gauge\nopenshell_proxy_active {}\n", + health.accepted.load(Ordering::Relaxed), + health.rejected.load(Ordering::Relaxed), + health.active.load(Ordering::Relaxed) + ) +} + +async fn shutdown() -> io::Result<()> { + #[cfg(unix)] + { + let mut terminate = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + tokio::select! { result = tokio::signal::ctrl_c() => result, _ = terminate.recv() => Ok(()) } + } + #[cfg(not(unix))] + { + tokio::signal::ctrl_c().await + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + let args = discovery::Args::parse(); + args.validate()?; + let listener = TcpListener::bind(args.listen_address).await?; + let health_listener = TcpListener::bind(args.health_address).await?; + let health = Arc::new(Health::default()); + let (routes, _) = watch::channel(cache::Routes::new()); + let discovery = tokio::spawn(discovery::run(args, routes.clone(), health.clone())); + let diagnostics = Router::new() + .route("/healthz", get(|| async { StatusCode::OK })) + .route("/readyz", get(readiness)) + .route("/metrics", get(metrics)) + .with_state(health.clone()); + tokio::select! { + result = serve(listener, routes, health) => result?, + result = axum::serve(health_listener, diagnostics) => result?, + result = shutdown() => result?, + } + discovery.abort(); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hello(host: &str) -> Vec { + let config = openshell_crypto::tls::client_builder() + .with_root_certificates(rustls::RootCertStore::empty()) + .with_no_client_auth(); + let mut client = + rustls::ClientConnection::new(Arc::new(config), host.to_owned().try_into().unwrap()) + .unwrap(); + let mut bytes = Vec::new(); + client.write_tls(&mut bytes).unwrap(); + bytes + } + + #[tokio::test] + async fn passthrough_preserves_bytes_and_revokes_replaced_installations() { + let backend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let frontend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = frontend.local_addr().unwrap(); + let (routes, _) = watch::channel(cache::Routes::from([( + "demo.example.com".into(), + cache::Route { + endpoint: backend.local_addr().unwrap(), + version: "owner-1".into(), + deadline: Instant::now() + Duration::from_secs(10), + connections: Arc::new(Semaphore::new(64)), + }, + )])); + let updates = routes.subscribe(); + let server = tokio::spawn(async move { + let (stream, _) = frontend.accept().await.unwrap(); + connection(stream, updates).await.unwrap(); + }); + let mut client = TcpStream::connect(address).await.unwrap(); + let bytes = hello("demo.example.com"); + client.write_all(&bytes).await.unwrap(); + let (mut peer, _) = backend.accept().await.unwrap(); + let mut received = vec![0; bytes.len()]; + peer.read_exact(&mut received).await.unwrap(); + assert_eq!(received, bytes); + peer.write_all(b"tls-stream").await.unwrap(); + let mut response = [0; 10]; + client.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"tls-stream"); + routes.send_modify(|view| { + view.get_mut("demo.example.com").unwrap().deadline = + Instant::now() + Duration::from_secs(14); + }); + peer.write_all(b"renewed").await.unwrap(); + let mut renewed = [0; 7]; + client.read_exact(&mut renewed).await.unwrap(); + assert_eq!(&renewed, b"renewed"); + routes.send_modify(|view| { + view.get_mut("demo.example.com").unwrap().version = "owner-2".into(); + }); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), client.read(&mut response)) + .await + .unwrap() + .unwrap(), + 0 + ); + server.await.unwrap(); + } + + #[tokio::test] + async fn unknown_sni_never_dials() { + let frontend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = frontend.local_addr().unwrap(); + let (routes, _) = watch::channel(cache::Routes::new()); + let server = tokio::spawn(async move { + let (stream, _) = frontend.accept().await.unwrap(); + connection(stream, routes.subscribe()).await.unwrap_err() + }); + let mut client = TcpStream::connect(address).await.unwrap(); + client + .write_all(&hello("unknown.example.com")) + .await + .unwrap(); + assert_eq!( + server.await.unwrap().kind(), + io::ErrorKind::PermissionDenied + ); + } + #[tokio::test] + async fn incomplete_hello_has_a_deadline() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let _client = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let (routes, _) = watch::channel(cache::Routes::new()); + tokio::time::pause(); + let result = tokio::time::timeout( + SETUP_TIMEOUT + Duration::from_secs(1), + connection(stream, routes.subscribe()), + ) + .await + .unwrap(); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::PermissionDenied); + } + #[tokio::test] + async fn cached_lease_expiry_closes_a_live_backend_stream() { + let backend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let frontend = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = frontend.local_addr().unwrap(); + let (routes, _) = watch::channel(cache::Routes::from([( + "demo.example.com".into(), + cache::Route { + endpoint: backend.local_addr().unwrap(), + version: "owner-1".into(), + deadline: Instant::now() + Duration::from_secs(10), + connections: Arc::new(Semaphore::new(64)), + }, + )])); + let updates = routes.subscribe(); + let server = tokio::spawn(async move { + let (stream, _) = frontend.accept().await.unwrap(); + connection(stream, updates).await.unwrap(); + }); + let mut client = TcpStream::connect(address).await.unwrap(); + let bytes = hello("demo.example.com"); + client.write_all(&bytes).await.unwrap(); + let (mut peer, _) = backend.accept().await.unwrap(); + let mut received = vec![0; bytes.len()]; + peer.read_exact(&mut received).await.unwrap(); + routes.send_modify(|view| { + view.get_mut("demo.example.com").unwrap().deadline = + Instant::now() + Duration::from_millis(100); + }); + let mut buffer = [0; 1]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), client.read(&mut buffer)) + .await + .unwrap() + .unwrap(), + 0 + ); + server.await.unwrap(); + drop(peer); + } +} diff --git a/deploy/docker/Dockerfile.sandbox-proxy b/deploy/docker/Dockerfile.sandbox-proxy new file mode 100644 index 0000000000..f2ffde7aba --- /dev/null +++ b/deploy/docker/Dockerfile.sandbox-proxy @@ -0,0 +1,8 @@ +# syntax=docker/dockerfile:1.4 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +FROM gcr.io/distroless/base-nossl-debian13@sha256:af5cb8dd589b8520b8c06bebb9efb73d7e16406cab58e85c51761fff49d370a0 AS sandbox-proxy +ARG TARGETARCH +COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox-proxy /openshell-sandbox-proxy +USER 65532:65532 +ENTRYPOINT ["/openshell-sandbox-proxy"] diff --git a/deploy/helm/openshell-sandbox-proxy/Chart.yaml b/deploy/helm/openshell-sandbox-proxy/Chart.yaml new file mode 100644 index 0000000000..13148e19f4 --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/Chart.yaml @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: v2 +name: openshell-sandbox-proxy +description: Optional discovery-driven application TLS passthrough proxy +type: application +version: 0.1.0 +appVersion: dev diff --git a/deploy/helm/openshell-sandbox-proxy/README.md b/deploy/helm/openshell-sandbox-proxy/README.md new file mode 100644 index 0000000000..03104efe0a --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/README.md @@ -0,0 +1,45 @@ + + +# Optional sandbox application proxy + +This standalone chart deploys the discovery-driven TLS passthrough proxy. It +requires an enabled supervisor app listener, accepted advertisements, the service +discovery API, and an existing identity authorized for one explicit workspace. + +```shell +helm upgrade --install app-proxy ./deploy/helm/openshell-sandbox-proxy \ + --namespace openshell \ + --set gatewayEndpoint=https://openshell.openshell.svc:8080 \ + --set workspace=default \ + --set gatewayCASecret=discovery-ca \ + --set tokenSecret=discovery-token \ + --set image.tag=YOUR_VERSION +``` + +The CA Secret uses `ca.crt`; the token Secret uses `token`. Set +`clientIdentitySecret` for a Secret containing discovery `tls.crt` and `tls.key`. +These are discovery credentials, not the application TLS private key. Secret +volumes use directory mounts so rotations are visible on reconnect. + +The Service exposes port 443 and forwards to the nonroot proxy's port 8443. +Attach your TLS-passthrough ingress and configure application DNS/public origin +separately. `/readyz` on port 9091 checks discovery availability; `/healthz` checks +process liveness, and `/metrics` reports connection counters. Two replicas are +deployed by default, each with its own cache. No Kubernetes API access is needed. + +Provide NetworkPolicy permitting discovery and supervisor app connections and +restrict direct supervisor ingress if your edge applies application access +controls. Configure discovery-only credentials through the gateway's certificate +grant mapping. Shared-supervisor publication still requires the HA integration. See the +[proxy README](../../../crates/openshell-sandbox-proxy/README.md) for limits and +cache/expiry behavior. + +Select exactly one workspace mode: existing `workspace: team-a`, +`workspaces: [team-a, team-b]`, or `allWorkspaces: true`. Match the mounted +client identity to an administrator-managed discovery grant for that scope. +A platform grant can still be used by a proxy configured for a narrower set. +The gateway rejects partially authorized selections; cache/connection limits +apply to the combined view, not independently to each workspace. diff --git a/deploy/helm/openshell-sandbox-proxy/templates/proxy.yaml b/deploy/helm/openshell-sandbox-proxy/templates/proxy.yaml new file mode 100644 index 0000000000..d3a407c820 --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/templates/proxy.yaml @@ -0,0 +1,145 @@ +{{- $modes := 0 -}} +{{- if .Values.workspace }}{{ $modes = add $modes 1 }}{{ end -}} +{{- if .Values.workspaces }}{{ $modes = add $modes 1 }}{{ end -}} +{{- if .Values.allWorkspaces }}{{ $modes = add $modes 1 }}{{ end -}} +{{- if ne (int $modes) 1 }}{{ fail "select exactly one of workspace, workspaces, or allWorkspaces" }}{{ end -}} +{{- if gt (len .Values.workspaces) 64 }}{{ fail "select at most 64 workspaces" }}{{ end -}} +{{- if ne (len .Values.workspaces) (len (uniq .Values.workspaces)) }}{{ fail "duplicate workspaces are not allowed" }}{{ end -}} +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: openshell-sandbox-proxy + template: + metadata: + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: openshell-sandbox-proxy + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: proxy + image: {{ printf "%s:%s" .Values.image.repository .Values.image.tag | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --gateway-endpoint + - {{ required "gatewayEndpoint is required" .Values.gatewayEndpoint | quote }} + {{- if .Values.allWorkspaces }} + - --all-workspaces + {{- else if .Values.workspace }} + - --workspace + - {{ .Values.workspace | quote }} + {{- else }} + {{- range .Values.workspaces }} + - --workspace + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .Values.gatewayCASecret }} + - --gateway-ca + - /run/discovery/ca/ca.crt + {{- end }} + {{- if .Values.clientIdentitySecret }} + - --discovery-client-cert + - /run/discovery/client/tls.crt + - --discovery-client-key + - /run/discovery/client/tls.key + {{- end }} + {{- if .Values.tokenSecret }} + - --discovery-token-file + - /run/discovery/token/token + {{- end }} + ports: + - name: app + containerPort: 8443 + - name: health + containerPort: 9091 + readinessProbe: + httpGet: + path: /readyz + port: health + livenessProbe: + httpGet: + path: /healthz + port: health + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + {{- if .Values.gatewayCASecret }} + - name: ca + mountPath: /run/discovery/ca + readOnly: true + {{- end }} + {{- if .Values.clientIdentitySecret }} + - name: client + mountPath: /run/discovery/client + readOnly: true + {{- end }} + {{- if .Values.tokenSecret }} + - name: token + mountPath: /run/discovery/token + readOnly: true + {{- end }} + volumes: + {{- if .Values.gatewayCASecret }} + - name: ca + secret: + secretName: {{ .Values.gatewayCASecret }} + items: + - key: ca.crt + path: ca.crt + defaultMode: 0440 + {{- end }} + {{- if .Values.clientIdentitySecret }} + - name: client + secret: + secretName: {{ .Values.clientIdentitySecret }} + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + defaultMode: 0440 + {{- end }} + {{- if .Values.tokenSecret }} + - name: token + secret: + secretName: {{ .Values.tokenSecret }} + items: + - key: token + path: token + defaultMode: 0440 + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} +spec: + type: {{ .Values.service.type }} + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: openshell-sandbox-proxy + ports: + - name: app + port: {{ .Values.service.port }} + targetPort: app diff --git a/deploy/helm/openshell-sandbox-proxy/tests/proxy_test.yaml b/deploy/helm/openshell-sandbox-proxy/tests/proxy_test.yaml new file mode 100644 index 0000000000..95637fa8ac --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/tests/proxy_test.yaml @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: discovery proxy deployment +templates: + - templates/proxy.yaml +set: + gatewayEndpoint: https://gateway.example.com + clientIdentitySecret: discovery-reader +tests: + - it: renders a Kubernetes Deployment for one workspace + set: + workspace: default + asserts: + - equal: + path: apiVersion + value: apps/v1 + documentIndex: 0 + - isKind: + of: Deployment + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].args + content: default + documentIndex: 0 + - it: renders a Deployment for a finite workspace set + set: + workspaces: [team-a, team-b] + asserts: + - equal: + path: apiVersion + value: apps/v1 + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].args + content: team-b + documentIndex: 0 + - it: renders a Deployment for all workspaces + set: + allWorkspaces: true + asserts: + - equal: + path: apiVersion + value: apps/v1 + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].args + content: --all-workspaces + documentIndex: 0 + - it: rejects ambiguous scope + set: + workspace: default + allWorkspaces: true + asserts: + - failedTemplate: + errorMessage: select exactly one of workspace, workspaces, or allWorkspaces + - it: rejects missing scope + asserts: + - failedTemplate: + errorMessage: select exactly one of workspace, workspaces, or allWorkspaces + - it: rejects duplicate workspaces + set: + workspaces: [team-a, team-a] + asserts: + - failedTemplate: + errorMessage: duplicate workspaces are not allowed diff --git a/deploy/helm/openshell-sandbox-proxy/values.yaml b/deploy/helm/openshell-sandbox-proxy/values.yaml new file mode 100644 index 0000000000..f4834722c8 --- /dev/null +++ b/deploy/helm/openshell-sandbox-proxy/values.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +replicaCount: 2 +image: + repository: ghcr.io/nvidia/openshell/sandbox-proxy + tag: dev + pullPolicy: IfNotPresent +gatewayEndpoint: "" +# Exactly one of workspace, workspaces, or allWorkspaces must be configured. +workspace: "" +workspaces: [] +allWorkspaces: false +# Secrets are mounted as directories so projected credential rotation is visible. +gatewayCASecret: "" +clientIdentitySecret: "" +tokenSecret: "" +service: + type: ClusterIP + port: 443 +resources: + requests: + cpu: 100m + memory: 64Mi + limits: + memory: 256Mi diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index b5ff063426..ab7b569636 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -314,3 +314,28 @@ for the routing-name field, its validation, and the effects of changing it. ## Next Steps Return to [Setup](/kubernetes/setup) to complete the installation. + +## Independent application proxy + +With supervisor application listeners and service discovery enabled, the optional +`openshell-sandbox-proxy` forwards TLS directly to the discovered supervisor. +Application payloads bypass the gateway. Configure the application public origin +with `supervisor.app_listener.public_url`; the gateway API endpoint and internal +supervisor listener port can differ from that origin. + +Deploy the standalone chart in `deploy/helm/openshell-sandbox-proxy` with a gateway +HTTPS endpoint, a workspace selection, and discovery trust/identity Secrets. +Select `workspace`, a `workspaces` list, or `allWorkspaces: true`. Use a +discovery-only certificate whose administrator-managed grant covers the selection; +see [gateway configuration](../how-it-works/gateways/configuration#discovery-reader-grants). +The proxy does not need Kubernetes API +permissions or application private keys; application TLS terminates at supervisors. + +Attach TLSRoute passthrough to the proxy Service's port 443 and configure DNS +for the application domain. HTTPRoute requires an explicit edge-termination and +backend-TLS design. Allow proxy egress to the gateway API and supervisor app +ports, and restrict direct supervisor ingress when edge authentication is required. +Discovery failures retain routes only through their expiry; withdrawals, +replacement, and expiry close affected streams. Multiple proxy replicas maintain +independent caches. A separate ingress controller can consume the same API and +route directly to supervisors without deploying this proxy. diff --git a/e2e/kubernetes/app-listener/README.md b/e2e/kubernetes/app-listener/README.md index 55909b336e..cbdd01188f 100644 --- a/e2e/kubernetes/app-listener/README.md +++ b/e2e/kubernetes/app-listener/README.md @@ -76,6 +76,33 @@ The local Rust listener tests use a real TLS client and backend RPC connector. They supplement this live cluster check; they do not establish HA pool recovery or real Kubernetes reachability. +## Discovery-driven proxy path + +Build the gateway, CLI, and optional proxy from the same stack. Deploy dedicated +supervisors with the listener enabled and set `public_url` to an HTTPS origin +whose port differs from both the gateway port-forward and listener port. Deploy +`deploy/helm/openshell-sandbox-proxy` with two replicas, the gateway CA, and one +explicit workspace. Provide a workspace-authorized discovery identity when the +gateway requires authentication. Permit the proxy Pods to reach supervisor app +ports through NetworkPolicy; the proxy needs no Kubernetes API access. + +Create a sandbox with an unnamed service and expose a second named service. +Verify `service expose`, API get/list, create-time response maps, and SDK results +retain the public origin. The gateway alternative must remain independently +available. From a permitted client Pod, use the returned hostname as SNI and +HTTP authority while connecting to each proxy Pod and then its Service. Verify +the supervisor certificate using only the public application CA. No application +private key is mounted in the proxy. + +Hold named and unnamed WebSockets through the proxy beyond the thirty-second +listener freshness interval. Delete the named declaration and require that +stream to close while the unnamed stream survives. Verify both proxy replicas +observe the same accepted installation. Repeat reconnect/expiry checks during +discovery interruption: unexpired cached routes may be used, authorization loss +clears them, and expired routes never open or retain a connection. Test malformed +or missing SNI without backend fallback. Inspect `/readyz`, `/healthz`, and +`/metrics` separately from application health. + ## Shared-supervisor HA and discovery Use the shared-supervisor HA overlay with the app-listener values and two gateway diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 9dbba8dafe..d9c978b1f7 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1657,6 +1657,17 @@ stripped; application cookies and explicit bearer passthrough remain available. Only explicit local-development unauthenticated access bypasses gateway auth. Direct listener authentication belongs to the ingress provider. +For `openshell-sandbox-proxy`, check `/readyz` on port 9091 and discovery access +before inspecting application TLS. `/healthz` is liveness, while `/metrics` +reports accepted, rejected, and active connections. The proxy Service's port 443 +maps to a nonroot TLS-passthrough listener on 8443. It reads one explicit +workspace through the same snapshot/watch API; discovery identity files reload +on reconnect. Never mount application TLS private keys or grant the proxy +Kubernetes API access. Missing SNI, unknown/expired routes, and exhausted setup +capacity are rejected. Discovery authorization loss clears the cache; temporary +errors retain entries only until expiry. Preserve application SNI to the owning +supervisor and inspect NetworkPolicy on both proxy and supervisor paths. + For discovery-only readers, verify the TLS client CA and an `OU=discovery` certificate whose lowercase DER SHA-256 fingerprint appears in the administrator-managed `openshell.gateway.auth.discovery_grants_file` JSON. @@ -1666,3 +1677,9 @@ propagation precedes the gateway's one-second refresh and bounded collection rechecks. Do not replace a discovery identity with an operator certificate. Single, finite-set, and all-workspaces discovery require explicit selectors; workspace users retain membership checks and cannot request platform scope. + +For proxy workspace selection, configure exactly one of Helm `workspace`, +`workspaces`, or `allWorkspaces`. The CLI accepts repeatable `--workspace` flags +or `--all-workspaces`. Check the entire selected set against the certificate's +grant; rejection is never a partially authorized successful snapshot. Limits +apply to the combined view. Discovery credentials and bearer mode are exclusive. diff --git a/tasks/docker.toml b/tasks/docker.toml index ec585c2de9..b4476dbdff 100644 --- a/tasks/docker.toml +++ b/tasks/docker.toml @@ -69,3 +69,7 @@ run = "scripts/docker-cleanup.sh --force" ["docker:cleanup:dry-run"] description = "Preview what docker:cleanup would remove without deleting anything" run = "scripts/docker-cleanup.sh --dry-run" + +["docker:build:sandbox-proxy"] +description = "Build the optional sandbox TLS proxy image" +run = "tasks/scripts/docker-build-image.sh sandbox-proxy" diff --git a/tasks/helm.toml b/tasks/helm.toml index 047cdad602..d978a23e84 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -55,7 +55,7 @@ run = """ run_windows = "echo Skipping helm:lint: Helm validation is not part of the native Windows lane." ["helm:test"] -description = "Run gateway and workspace Helm chart unit tests" +description = "Run gateway, workspace, and sandbox proxy Helm chart unit tests" run = """ set -e if ! helm plugin list | grep -q unittest; then @@ -64,6 +64,7 @@ run = """ helm dependency build deploy/helm/openshell helm unittest deploy/helm/openshell helm unittest deploy/helm/openshell-workspace + helm unittest deploy/helm/openshell-sandbox-proxy deploy/helm/test-split-ownership.sh deploy/helm/test-gateway-config-parser.sh deploy/helm/test-gateway-resource-coherence.sh diff --git a/tasks/scripts/docker-build-image.sh b/tasks/scripts/docker-build-image.sh index 06102fc649..040ab5e2ad 100755 --- a/tasks/scripts/docker-build-image.sh +++ b/tasks/scripts/docker-build-image.sh @@ -40,6 +40,9 @@ prebuilt_arches() { required_prebuilt_binaries() { case "$1" in + sandbox-proxy) + echo "openshell-sandbox-proxy" + ;; gateway) echo "openshell-gateway" ;; @@ -95,7 +98,7 @@ ensure_prebuilt_binaries() { fi } -TARGET=${1:?"Usage: docker-build-image.sh [extra-args...]"} +TARGET=${1:?"Usage: docker-build-image.sh [extra-args...]"} shift IS_FINAL_IMAGE=0 @@ -103,6 +106,12 @@ IMAGE_NAME="" DOCKER_TARGET="" DOCKERFILE="" case "${TARGET}" in + sandbox-proxy) + IS_FINAL_IMAGE=1 + IMAGE_NAME="openshell/sandbox-proxy" + DOCKER_TARGET="sandbox-proxy" + DOCKERFILE="deploy/docker/Dockerfile.sandbox-proxy" + ;; gateway) IS_FINAL_IMAGE=1 IMAGE_NAME="openshell/gateway" diff --git a/tasks/scripts/docker-publish-multiarch.sh b/tasks/scripts/docker-publish-multiarch.sh index 430bfa2b7d..d101dfb9e4 100755 --- a/tasks/scripts/docker-publish-multiarch.sh +++ b/tasks/scripts/docker-publish-multiarch.sh @@ -54,13 +54,16 @@ _publish_multiarch_docker() { echo "Building multi-arch supervisor image..." tasks/scripts/docker-build-image.sh supervisor + echo "Building multi-arch sandbox proxy image..." + tasks/scripts/docker-build-image.sh sandbox-proxy + TAGS_TO_APPLY=("${EXTRA_TAGS[@]}") if [[ "${TAG_LATEST}" == "true" ]]; then TAGS_TO_APPLY+=("latest") fi if [[ ${#TAGS_TO_APPLY[@]} -gt 0 ]]; then - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do full_image="${REGISTRY}/${component}" for tag in "${TAGS_TO_APPLY[@]}"; do [[ "${tag}" == "${IMAGE_TAG}" ]] && continue @@ -83,7 +86,7 @@ _publish_multiarch_podman() { # Split comma-separated platforms into an array. IFS=',' read -ra PLATFORM_LIST <<< "${PLATFORMS}" - for component in gateway sandbox supervisor; do + for component in gateway sandbox supervisor sandbox-proxy; do local full_image="${REGISTRY}/${component}" local manifest_name="${full_image}:${IMAGE_TAG}" @@ -151,6 +154,7 @@ echo "Done! Multi-arch images pushed to ${REGISTRY}:" echo " ${REGISTRY}/gateway:${IMAGE_TAG}" echo " ${REGISTRY}/sandbox:${IMAGE_TAG}" echo " ${REGISTRY}/supervisor:${IMAGE_TAG}" +echo " ${REGISTRY}/sandbox-proxy:${IMAGE_TAG}" if [[ "${TAG_LATEST}" == "true" ]]; then echo " (all also tagged :latest)" fi diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index 01d2bff12f..2a9284cf6a 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -12,7 +12,7 @@ ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" source "${SCRIPT_DIR}/build-env.sh" usage() { - echo "Usage: stage-prebuilt-binaries.sh " >&2 + echo "Usage: stage-prebuilt-binaries.sh " >&2 } normalize_arch() { @@ -84,8 +84,8 @@ detect_arches() { components_for_target() { case "$1" in - gateway) - echo "gateway" + gateway|sandbox-proxy) + echo "$1" ;; sandbox) echo "sandbox" @@ -94,7 +94,7 @@ components_for_target() { echo "supervisor" ;; all) - echo "gateway sandbox supervisor" + echo "gateway sandbox supervisor sandbox-proxy" ;; *) usage @@ -105,6 +105,11 @@ components_for_target() { resolve_component() { case "$1" in + sandbox-proxy) + crate=openshell-sandbox-proxy + binary=openshell-sandbox-proxy + target_libc=gnu + ;; gateway) crate=openshell-gateway binary=openshell-gateway