From 28551bd87798b5d5b2efc928696af17e06879148 Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Thu, 22 Oct 2026 15:22:34 -0400 Subject: [PATCH] feat(gateway): add hashed service-routing hostnames for multi-gateway clusters Closes #4840 Add an optional service_routing_gateway_name gateway setting (TOML, CLI, env, Helm) that disambiguates exposed service URLs when multiple gateways share a cluster, ingress, and wildcard certificate. When configured, exposed-service hostnames become a deterministic 16-character lowercase base32 SHA-256 label of (gateway name, workspace, sandbox, service) under the same base domain, keeping every hostname exactly one label deep for wildcard-certificate compatibility. Unset gateways preserve the existing readable hostname format byte-for-byte, and configured gateways still accept legacy hosts during migration. Hashed hosts resolve through an in-memory route index hydrated from persisted ServiceEndpoint records at startup, refreshed on miss for multi-replica deployments, maintained across expose, delete, and sandbox cleanup, and enforced for collisions before endpoint persistence. Signed-off-by: Jesse Jaggars --- crates/openshell-core/src/config.rs | 118 +++++ crates/openshell-server/src/cli.rs | 150 +++++++ crates/openshell-server/src/compute/mod.rs | 17 +- crates/openshell-server/src/config_file.rs | 40 ++ crates/openshell-server/src/grpc/service.rs | 75 +++- crates/openshell-server/src/http.rs | 6 +- crates/openshell-server/src/lib.rs | 13 + .../openshell-server/src/service_routing.rs | 423 +++++++++++++++++- deploy/helm/openshell/README.md | 1 + deploy/helm/openshell/templates/_toml.tpl | 4 + .../openshell/tests/gateway_config_test.yaml | 28 ++ deploy/helm/openshell/values.yaml | 8 + docs/how-it-works/gateways/configuration.mdx | 46 ++ docs/how-it-works/gateways/overview.mdx | 4 +- docs/how-it-works/sandboxes/overview.mdx | 6 + docs/kubernetes/ingress.mdx | 43 ++ docs/kubernetes/setup.mdx | 1 + 17 files changed, 962 insertions(+), 21 deletions(-) diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 7f5d4d1ec0..dfdf9bc789 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -346,6 +346,51 @@ pub struct ServiceRoutingConfig { /// Enable TLS-enabled loopback gateway listeners to also accept plaintext /// HTTP for sandbox service hostnames. pub enable_loopback_service_http: bool, + + /// Optional gateway name used to derive opaque hashed service route + /// labels. + /// + /// When `None` (the default), the gateway keeps generating readable + /// `--[--]` service hostnames. When set, + /// newly generated service hostnames use a stable, one-label hash derived + /// from this name plus the workspace, sandbox, and service instead. + /// + /// This value is unrelated to [`Config::name`] and + /// [`GatewayJwtConfig::gateway_id`]; it is never derived from them. + pub service_routing_gateway_name: Option, +} + +impl ServiceRoutingConfig { + /// Validate the service-routing configuration. + /// + /// Currently validates [`Self::service_routing_gateway_name`]: `None` is + /// always valid, while `Some(name)` must be a DNS-1123 label — 1–63 ASCII + /// bytes of lowercase `a-z`, `0-9`, or `-`, with no leading or trailing + /// `-`. An empty string is rejected; the setting is represented as + /// disabled only by `None`. + pub fn validate(&self) -> crate::error::Result<()> { + Self::validate_gateway_name(self.service_routing_gateway_name.as_deref()) + } + + /// Validate an optional service-routing gateway name. + /// + /// `None` is always valid. `Some(name)` must be a DNS-1123 label — 1–63 + /// ASCII bytes of lowercase `a-z`, `0-9`, or `-`, with no leading or + /// trailing `-`. An empty string is rejected; the setting is represented + /// as disabled only by `None`. + pub fn validate_gateway_name(name: Option<&str>) -> crate::error::Result<()> { + if let Some(name) = name + && !is_dns_label(name) + { + return Err(crate::error::Error::Config { + message: format!( + "service_routing_gateway_name must be a DNS label (1-63 lowercase \ + ASCII alphanumeric or '-' bytes, no leading/trailing '-'): {name:?}" + ), + }); + } + Ok(()) + } } /// TLS configuration. @@ -1125,6 +1170,17 @@ impl Config { self } + /// Set the service-routing gateway name used to derive hashed service + /// route labels. + /// + /// `None` preserves readable service hostnames. `Some(name)` must pass + /// [`ServiceRoutingConfig::validate`]. + #[must_use] + pub fn with_service_routing_gateway_name(mut self, name: Option>) -> Self { + self.service_routing.service_routing_gateway_name = name.map(Into::into); + self + } + /// Enable or disable plaintext HTTP routing for loopback sandbox service /// hostnames on TLS-enabled gateway listeners. #[must_use] @@ -1146,6 +1202,7 @@ impl Default for ServiceRoutingConfig { Self { base_domains: default_service_routing_domains(), enable_loopback_service_http: default_enable_loopback_service_http(), + service_routing_gateway_name: None, } } } @@ -1700,6 +1757,67 @@ mod tests { assert!(!cfg.service_routing.enable_loopback_service_http); } + #[test] + fn service_routing_gateway_name_defaults_to_none() { + let cfg = Config::new(None); + assert!(cfg.service_routing.service_routing_gateway_name.is_none()); + cfg.service_routing.validate().expect("unset is valid"); + } + + #[test] + fn service_routing_gateway_name_accepts_dns_label() { + for name in ["gateway-west", "g", &"g".repeat(63), "gateway-1-west"] { + let cfg = Config::new(None).with_service_routing_gateway_name(Some(name)); + assert_eq!( + cfg.service_routing.service_routing_gateway_name.as_deref(), + Some(name) + ); + cfg.service_routing + .validate() + .unwrap_or_else(|e| panic!("{name:?} must be valid: {e}")); + } + } + + #[test] + fn service_routing_gateway_name_rejects_empty_uppercase_invalid_hyphen_and_overlong_values() { + for name in [ + "", + "Gateway-West", + "-gateway", + "gateway-", + "gateway_west", + "gateway west", + &"g".repeat(64), + ] { + let cfg = Config::new(None).with_service_routing_gateway_name(Some(name)); + let error = cfg + .service_routing + .validate() + .expect_err(&format!("{name:?} must be rejected")); + let message = error.to_string(); + assert!(message.contains("service_routing_gateway_name"), "{name:?}"); + } + } + + #[test] + fn server_sans_update_preserves_service_routing_gateway_name() { + let cfg = Config::new(None) + .with_service_routing_gateway_name(Some("gateway-west")) + .with_server_sans(["*.apps.example.com"]); + + assert_eq!( + cfg.service_routing.service_routing_gateway_name.as_deref(), + Some("gateway-west") + ); + assert_eq!( + cfg.service_routing.base_domains, + vec![ + "apps.example.com".to_string(), + DEFAULT_SERVICE_ROUTING_DOMAIN.to_string() + ] + ); + } + #[test] fn service_routing_domains_are_derived_from_wildcard_server_sans() { let cfg = Config::new(None).with_server_sans([ diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 5b2ce01c13..126f164d77 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -286,6 +286,12 @@ struct RunArgs { )] server_sans: Vec, + /// Gateway name used to derive opaque hashed sandbox service route + /// labels. Unset keeps readable `--[--]` + /// service hostnames. + #[arg(long, env = "OPENSHELL_SERVICE_ROUTING_GATEWAY_NAME")] + service_routing_gateway_name: Option, + /// Enable plaintext HTTP routing for loopback sandbox service URLs. #[arg( long, @@ -568,8 +574,13 @@ fn prepare_server_config_with_drivers( .unwrap_or_default(), ) .with_server_sans(args.server_sans.clone()) + .with_service_routing_gateway_name(args.service_routing_gateway_name.clone()) .with_loopback_service_http(args.enable_loopback_service_http) .with_websocket_tunnel(args.enable_websocket_tunnel); + config + .service_routing + .validate() + .map_err(|error| miette::miette!("invalid service routing configuration: {error}"))?; if let Some(sources) = file .as_ref() .and_then(|file| file.openshell.gateway.provider_profile_sources.clone()) @@ -1044,6 +1055,10 @@ fn validate_preflight_semantics( args.grpc_rate_limit_requests, args.grpc_rate_limit_window_seconds, )?; + openshell_core::config::ServiceRoutingConfig::validate_gateway_name( + args.service_routing_gateway_name.as_deref(), + ) + .map_err(|error| miette::miette!("invalid service routing configuration: {error}"))?; GuestTlsPaths::validate_configuration(Some(gateway), args.disable_tls) .map_err(|error| miette::miette!("invalid gateway guest TLS configuration: {error}"))?; @@ -1252,6 +1267,12 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: { args.server_sans.clone_from(sans); } + if let Some(name) = &file.service_routing_gateway_name + && args.service_routing_gateway_name.is_none() + && arg_defaulted(matches, "service_routing_gateway_name") + { + args.service_routing_gateway_name = Some(name.clone()); + } if let Some(enabled) = file.enable_loopback_service_http && arg_defaulted(matches, "enable_loopback_service_http") { @@ -2509,6 +2530,10 @@ mod tests { "rate-limit", "[openshell]\nversion = 2\n[openshell.gateway]\nname = 'secret-semantic-marker'\ngrpc_rate_limit_requests = 10\n", ), + ( + "service-routing", + "[openshell]\nversion = 2\n[openshell.gateway]\nservice_routing_gateway_name = 'Not A Label'\n", + ), ( "guest-tls", "[openshell]\nversion = 2\n[openshell.gateway]\nguest_tls_ca = '/tls/ca.pem'\ndisable_tls = true\n", @@ -3657,6 +3682,131 @@ enable_loopback_service_http = false }); } + #[test] + fn file_populates_service_routing_gateway_name() { + Environment::new() + .remove("OPENSHELL_SERVICE_ROUTING_GATEWAY_NAME") + .run(|| { + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let file = config_file_from_toml( + "[openshell.gateway]\nservice_routing_gateway_name = \"gateway-west\"\n", + ); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + + assert_eq!( + args.service_routing_gateway_name.as_deref(), + Some("gateway-west") + ); + }); + } + + #[test] + fn service_routing_gateway_name_cli_and_env_override_file_value() { + // CLI flag wins over the file value. + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--service-routing-gateway-name", + "gateway-cli", + ]); + let file = config_file_from_toml( + "[openshell.gateway]\nservice_routing_gateway_name = \"gateway-west\"\n", + ); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert_eq!( + args.service_routing_gateway_name.as_deref(), + Some("gateway-cli"), + "CLI flag must win over file value" + ); + + // Env var wins over the file value. + Environment::new() + .set("OPENSHELL_SERVICE_ROUTING_GATEWAY_NAME", "gateway-env") + .run(|| { + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + let file = config_file_from_toml( + "[openshell.gateway]\nservice_routing_gateway_name = \"gateway-west\"\n", + ); + merge_file_into_args(&mut args, &file.openshell.gateway, &matches); + assert_eq!( + args.service_routing_gateway_name.as_deref(), + Some("gateway-env"), + "env var must win over file value" + ); + }); + } + + #[test] + fn server_config_preparation_applies_and_validates_service_routing_gateway_name() { + let state = tempfile::tempdir().unwrap(); + let local_tls = tempfile::tempdir().unwrap(); + + Environment::new() + .set("XDG_STATE_HOME", state.path().to_str().unwrap()) + .set( + "OPENSHELL_LOCAL_TLS_DIR", + local_tls.path().to_str().unwrap(), + ) + .remove("OPENSHELL_SERVICE_ROUTING_GATEWAY_NAME") + .run(|| { + let config_path = state.path().join("gateway.toml"); + std::fs::write( + &config_path, + "[openshell]\nversion = 2\n\n[openshell.gateway]\nservice_routing_gateway_name = \"gateway-west\"\n", + ) + .unwrap(); + + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--config", + config_path.to_str().unwrap(), + "--db-url", + "sqlite::memory:", + "--compute-driver", + "local", + "--disable-tls", + ]); + + let prepared = super::prepare_server_config(&mut args, &matches) + .expect("server config is prepared"); + assert_eq!( + prepared + .config + .service_routing + .service_routing_gateway_name + .as_deref(), + Some("gateway-west") + ); + + let invalid_path = state.path().join("invalid-gateway-name.toml"); + std::fs::write( + &invalid_path, + "[openshell]\nversion = 2\n\n[openshell.gateway]\nservice_routing_gateway_name = \"Gateway West\"\n", + ) + .unwrap(); + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--config", + invalid_path.to_str().unwrap(), + "--db-url", + "sqlite::memory:", + "--compute-driver", + "local", + "--disable-tls", + ]); + let error = super::prepare_server_config(&mut args, &matches) + .err() + .expect("invalid routing gateway name must fail preparation"); + assert!( + error.to_string().contains("service_routing_gateway_name"), + "unexpected error: {error}" + ); + }); + } + #[test] fn canonical_file_driver_selector_populates_cli_args() { let (mut args, matches) = diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 1ec4cfd522..32373e5d63 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -766,6 +766,7 @@ pub struct ComputeRuntime { /// Desired assignments of the shared supervisor host, when the gateway /// coordinates one for the in-process driver. supervisor_host: Option, + service_route_index: Arc>>, } impl fmt::Debug for ComputeRuntime { @@ -875,6 +876,7 @@ impl ComputeRuntime { ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), supervisor_host: None, + service_route_index: Arc::new(OnceLock::new()), }) } @@ -910,6 +912,13 @@ impl ComputeRuntime { /// Collector headers that launch bundles carry to supervisors for OCSF /// export. + pub(crate) fn configure_service_route_index( + &self, + index: Arc, + ) { + let _ = self.service_route_index.set(index); + } + pub(crate) fn configure_supervisor_ocsf_headers( &self, headers: Option, @@ -5889,12 +5898,16 @@ impl ComputeRuntime { if let Ok(endpoint) = ServiceEndpoint::decode(record.payload.as_slice()) && endpoint.sandbox_id == sandbox_id { - self.store + let deleted = self + .store .delete(ServiceEndpoint::object_type(), endpoint.object_id()) .await .map_err(|e| { format!("delete service endpoint {}: {e}", endpoint.object_id()) })?; + if deleted && let Some(index) = self.service_route_index.get() { + index.remove(workspace, &endpoint.sandbox, &endpoint.name); + } } } @@ -7873,6 +7886,7 @@ pub fn new_test_runtime_with_driver( ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), supervisor_host: None, + service_route_index: Arc::new(OnceLock::new()), } } @@ -8994,6 +9008,7 @@ mod tests { ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), supervisor_host: None, + service_route_index: Arc::new(OnceLock::new()), } } diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 817a6a02ad..ba3c20bd5e 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -159,6 +159,11 @@ pub struct GatewayFileSection { /// Enable plaintext HTTP routing for loopback sandbox service URLs. #[serde(default)] pub enable_loopback_service_http: Option, + /// Gateway name used to derive opaque hashed sandbox service route + /// labels. Unset keeps readable `--[--]` + /// service hostnames. + #[serde(default)] + pub service_routing_gateway_name: Option, /// Enable the WebSocket tunnel for an authenticated edge proxy. #[serde(default)] pub enable_websocket_tunnel: Option, @@ -1094,6 +1099,41 @@ compute_driver = 42 assert!(serialized.contains("compute_driver = \"docker\"")); } + #[test] + fn service_routing_gateway_name_parses_and_serializes() { + let file = toml::from_str::( + r#" +[openshell] +version = 2 + +[openshell.gateway] +service_routing_gateway_name = "gateway-west" +"#, + ) + .expect("service routing gateway name parses"); + + assert_eq!( + file.openshell + .gateway + .service_routing_gateway_name + .as_deref(), + Some("gateway-west") + ); + + let serialized = toml::to_string(&file).expect("config serializes"); + assert!(serialized.contains("service_routing_gateway_name = \"gateway-west\"")); + + let unset = toml::from_str::("[openshell]\nversion = 2\n") + .expect("file without the field parses"); + assert!( + unset + .openshell + .gateway + .service_routing_gateway_name + .is_none() + ); + } + #[test] fn parses_full_example() { let toml = r#" diff --git a/crates/openshell-server/src/grpc/service.rs b/crates/openshell-server/src/grpc/service.rs index bb44f395b8..57738c1df0 100644 --- a/crates/openshell-server/src/grpc/service.rs +++ b/crates/openshell-server/src/grpc/service.rs @@ -155,8 +155,14 @@ pub(super) async fn expose_service_endpoint( authorization_mode: authorization_mode as i32, }; - // Single-attempt CAS write: fails with ABORTED on concurrent modification - let result = state + let route_reserved = state + .service_route_index + .reserve(workspace, sandbox_name, service) + .map_err(|_| Status::already_exists("service routing hostname hash collision"))?; + + // Single-attempt CAS write: fails with ABORTED on concurrent modification. + // Release a newly reserved route if the durable write did not succeed. + let result = match state .store .put_if( ServiceEndpoint::object_type(), @@ -168,7 +174,17 @@ pub(super) async fn expose_service_endpoint( condition, ) .await - .map_err(|e| super::persistence_error_to_status(e, "expose service"))?; + { + Ok(result) => result, + Err(error) => { + if route_reserved { + state + .service_route_index + .remove(workspace, sandbox_name, service); + } + return Err(super::persistence_error_to_status(error, "expose service")); + } + }; let mut endpoint = endpoint; if let Some(ref mut meta) = endpoint.metadata { @@ -341,6 +357,9 @@ pub(super) async fn handle_delete_service( .map_err(|e| super::persistence_error_to_status(e, "delete endpoint"))?; if deleted { + state + .service_route_index + .remove(workspace, sandbox_name, &req.name); service_routing::emit_service_endpoint_delete_event(&endpoint); } @@ -508,6 +527,56 @@ mod tests { ); } + #[tokio::test] + async fn expose_service_rejects_route_hash_collision_before_persistence() { + let mut state = test_server_state().await; + let state_mut = Arc::get_mut(&mut state).unwrap(); + state_mut.config = state_mut + .config + .clone() + .with_service_routing_gateway_name(Some("gateway-a")); + state_mut.service_route_index = Arc::new(service_routing::ServiceRouteIndex::new(Some( + "gateway-a".to_string(), + ))); + seed_sandbox(&state, "my-sandbox").await; + + let label = service_routing::service_route_hash_label( + "gateway-a", + "default", + "my-sandbox", + Some("web"), + ); + state.service_route_index.seed_for_test( + label, + service_routing::ServiceRouteTarget { + workspace: "other".to_string(), + sandbox: "sandbox".to_string(), + service: Some("web".to_string()), + }, + ); + + let error = handle_expose_service( + &state, + authed_request(ExposeServiceRequest { + sandbox: "my-sandbox".to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + name: "web".to_string(), + target_port: 8080, + ..Default::default() + }), + ) + .await + .unwrap_err(); + + assert_eq!(error.code(), tonic::Code::AlreadyExists); + assert!( + get_service_endpoint(&state, "default", "my-sandbox", "web") + .await + .unwrap() + .is_none() + ); + } + #[tokio::test] async fn unknown_authorization_mode_is_rejected_before_persistence() { let state = test_server_state().await; diff --git a/crates/openshell-server/src/http.rs b/crates/openshell-server/src/http.rs index cf37aa18b9..af837d3ff4 100644 --- a/crates/openshell-server/src/http.rs +++ b/crates/openshell-server/src/http.rs @@ -204,7 +204,8 @@ async fn sandbox_service_routing_first( req: Request, next: Next, ) -> impl IntoResponse { - if crate::service_routing::is_sandbox_service_request(&req, &state.config.service_routing) { + let host = crate::service_routing::service_request_host(&req); + if crate::service_routing::is_sandbox_service_request_with_state(host, &state).await { return crate::service_routing::proxy_sandbox_service_request(state, req) .await .into_response(); @@ -216,7 +217,8 @@ async fn sandbox_service_routing_only( State(state): State>, req: Request, ) -> impl IntoResponse { - if !crate::service_routing::is_sandbox_service_request(&req, &state.config.service_routing) { + let host = crate::service_routing::service_request_host(&req); + if !crate::service_routing::is_sandbox_service_request_with_state(host, &state).await { return StatusCode::NOT_FOUND.into_response(); } if !browser_context_allows_plaintext_service_request(&req) { diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index f1f991e4ae..518fc3653e 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -364,6 +364,9 @@ pub struct ServerState { /// relay instead of opening one per request. pub service_upstreams: Arc, + /// In-memory hostname-to-service index for opaque service routing labels. + pub service_route_index: Arc, + /// Latest placement ring, refreshed from live gateway membership. Decides /// which replica *should* own a sandbox; the owner record in the store /// remains authoritative for which one does. @@ -477,6 +480,10 @@ impl ServerState { ) -> Self { let replica_id = compute::lease::replica_id(); compute.configure_ssh_identities(credentials.clone()); + let service_route_index = Arc::new(service_routing::ServiceRouteIndex::new( + config.service_routing.service_routing_gateway_name.clone(), + )); + compute.configure_service_route_index(Arc::clone(&service_route_index)); let peer_endpoint = derive_peer_endpoint(&config); let grpc_rate_limiter = multiplex::GrpcRateLimiter::from_config(&config); let reflection_service = reflection::build_gateway_reflection_service(&config) @@ -505,6 +512,7 @@ impl ServerState { peer_endpoint, peer_routes: Arc::new(supervisor_session::PeerRouteCache::default()), service_upstreams: Arc::new(service_routing::ServiceUpstreamPool::default()), + service_route_index, gateway_ring: Arc::new(std::sync::RwLock::new(gateway_ring::GatewayRing::default())), gateway_peers: Arc::new(std::sync::RwLock::new(HashMap::new())), extension_mint_limiter: auth::extension_mint_limit::ExtensionMintLimiter::default(), @@ -945,6 +953,11 @@ pub(crate) async fn run_server( // first snapshots observe the post-start backend state. Explicitly stopped // sandboxes remain stopped. ensure_default_workspace(&store).await?; + state + .service_route_index + .rebuild(&state.store) + .await + .map_err(|error| Error::config(format!("service routing index rebuild failed: {error}")))?; grpc::policy::validate_provider_composition_startup_preflight(&state) .await .map_err(|error| { diff --git a/crates/openshell-server/src/service_routing.rs b/crates/openshell-server/src/service_routing.rs index a84245d2cc..7d6b44f486 100644 --- a/crates/openshell-server/src/service_routing.rs +++ b/crates/openshell-server/src/service_routing.rs @@ -9,24 +9,25 @@ use axum::{ }; use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, header}; use hyper_util::rt::TokioIo; -use openshell_core::ObjectId; use openshell_core::config::ServiceRoutingConfig; use openshell_core::proto::{ Sandbox, SandboxPhase, ServiceAuthorizationMode, ServiceEndpoint, TcpRelayTarget, relay_open, }; +use openshell_core::{ObjectId, ObjectWorkspace}; use openshell_ocsf::{ ActionId, ActivityId, ConfigStateChangeBuilder, DispositionId, Endpoint, EventContext, HttpActivityBuilder, HttpRequest, HttpResponse as OcsfHttpResponse, NetworkActivityBuilder, OcsfEvent, SeverityId, StateId, StatusId, Url as OcsfUrl, }; +use sha2::{Digest, Sha256}; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant}; use tokio::io::AsyncWriteExt; use tracing::warn; use crate::ServerState; -use crate::persistence::{ObjectType, Store}; +use crate::persistence::{ObjectListQuery, ObjectType, Store}; const ENDPOINT_OBJECT_TYPE: &str = "service_endpoint"; const ROUTING_RULE_NAME: &str = "sandbox_service_routing"; @@ -190,11 +191,221 @@ fn endpoint_host( service: &str, ) -> Option { let base_domain = config.base_domains.first()?; - Some(if service.is_empty() { - format!("{workspace}--{sandbox}.{base_domain}") - } else { - format!("{workspace}--{sandbox}--{service}.{base_domain}") - }) + Some(config.service_routing_gateway_name.as_deref().map_or_else( + || { + if service.is_empty() { + format!("{workspace}--{sandbox}.{base_domain}") + } else { + format!("{workspace}--{sandbox}--{service}.{base_domain}") + } + }, + |gateway_name| { + format!( + "{}.{}", + service_route_hash_label( + gateway_name, + workspace, + sandbox, + (!service.is_empty()).then_some(service), + ), + base_domain + ) + }, + )) +} + +/// Returns the stable, one-label DNS name for a service route. +pub fn service_route_hash_label( + gateway_name: &str, + workspace: &str, + sandbox: &str, + service: Option<&str>, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(gateway_name.as_bytes()); + hasher.update([0]); + hasher.update(workspace.as_bytes()); + hasher.update([0]); + hasher.update(sandbox.as_bytes()); + hasher.update([0]); + hasher.update(service.unwrap_or("").as_bytes()); + let digest = hasher.finalize(); + base32_lower_no_padding(&digest[..10]) +} + +fn base32_lower_no_padding(bytes: &[u8]) -> String { + const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; + let mut output = String::with_capacity(16); + let mut accumulator = 0_u32; + let mut bits = 0_u8; + for byte in bytes { + accumulator = (accumulator << 8) | u32::from(*byte); + bits += 8; + while bits >= 5 { + bits -= 5; + output.push(char::from( + ALPHABET[((accumulator >> bits) & 0x1f) as usize], + )); + } + } + if bits > 0 { + output.push(char::from( + ALPHABET[((accumulator << (5 - bits)) & 0x1f) as usize], + )); + } + output +} + +/// A resolved service-route destination. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ServiceRouteTarget { + pub workspace: String, + pub sandbox: String, + pub service: Option, +} + +impl ServiceRouteTarget { + fn new(workspace: &str, sandbox: &str, service: &str) -> Self { + Self { + workspace: workspace.to_string(), + sandbox: sandbox.to_string(), + service: (!service.is_empty()).then(|| service.to_string()), + } + } +} + +/// A collision between two service routes with the same hashed hostname. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct CollisionError { + label: String, +} + +impl std::fmt::Display for CollisionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "service routing hostname hash collision for {}", + self.label + ) + } +} + +impl std::error::Error for CollisionError {} + +/// In-memory lookup table for opaque service-routing host labels. +#[derive(Debug)] +pub struct ServiceRouteIndex { + gateway_name: Option, + entries: Arc>>, +} + +impl ServiceRouteIndex { + pub fn new(gateway_name: Option) -> Self { + Self { + gateway_name, + entries: Arc::new(RwLock::new(HashMap::new())), + } + } + + fn route_label(&self, target: &ServiceRouteTarget) -> Option { + self.gateway_name.as_deref().map(|gateway_name| { + service_route_hash_label( + gateway_name, + &target.workspace, + &target.sandbox, + target.service.as_deref(), + ) + }) + } + + pub fn register( + &self, + workspace: &str, + sandbox: &str, + service: &str, + ) -> Result<(), CollisionError> { + self.reserve(workspace, sandbox, service).map(|_| ()) + } + + /// Reserves a route and returns whether this call inserted it. + pub(crate) fn reserve( + &self, + workspace: &str, + sandbox: &str, + service: &str, + ) -> Result { + let target = ServiceRouteTarget::new(workspace, sandbox, service); + let Some(label) = self.route_label(&target) else { + return Ok(false); + }; + let mut entries = self.entries.write().unwrap(); + if let Some(existing) = entries.get(&label) { + return if existing == &target { + Ok(false) + } else { + Err(CollisionError { label }) + }; + } + entries.insert(label, target); + Ok(true) + } + + pub fn remove(&self, workspace: &str, sandbox: &str, service: &str) { + let target = ServiceRouteTarget::new(workspace, sandbox, service); + let Some(label) = self.route_label(&target) else { + return; + }; + let mut entries = self.entries.write().unwrap(); + if entries.get(&label) == Some(&target) { + entries.remove(&label); + } + } + + pub fn lookup(&self, label: &str) -> Option { + self.entries.read().unwrap().get(label).cloned() + } + + #[cfg(test)] + pub(crate) fn seed_for_test(&self, label: String, target: ServiceRouteTarget) { + self.entries.write().unwrap().insert(label, target); + } + + pub async fn rebuild(&self, store: &Store) -> Result<(), CollisionError> { + let Some(gateway_name) = self.gateway_name.as_deref() else { + return Ok(()); + }; + let endpoints = store + .collect_messages::(ObjectListQuery::AllWorkspaces) + .await + .map_err(|error| { + warn!(error = %error, "failed to rebuild service routing index"); + CollisionError { + label: "persistence error".to_string(), + } + })?; + let mut rebuilt = HashMap::new(); + for endpoint in endpoints { + let target = ServiceRouteTarget::new( + endpoint.object_workspace(), + &endpoint.sandbox, + &endpoint.name, + ); + let label = service_route_hash_label( + gateway_name, + &target.workspace, + &target.sandbox, + target.service.as_deref(), + ); + if let Some(existing) = rebuilt.get(&label) + && existing != &target + { + return Err(CollisionError { label }); + } + rebuilt.insert(label, target); + } + *self.entries.write().unwrap() = rebuilt; + Ok(()) + } } // The `--` delimiter is unambiguous because both workspace and sandbox name @@ -231,22 +442,91 @@ pub fn parse_host(host: &str, config: &ServiceRoutingConfig) -> Option<(String, None } +/// Resolve either a legacy readable service host or an opaque hashed host. +pub async fn resolve_service_host(state: &ServerState, host: &str) -> Option { + if let Some((workspace, sandbox, service)) = parse_host(host, &state.config.service_routing) { + return Some(ServiceRouteTarget::new(&workspace, &sandbox, &service)); + } + + let gateway_name = state + .config + .service_routing + .service_routing_gateway_name + .as_deref()?; + let label = service_host_label(host, &state.config.service_routing)?; + if !is_hash_label(label) { + return None; + } + if let Some(target) = state.service_route_index.lookup(label) { + return Some(target); + } + + // A gateway can begin receiving traffic before an endpoint is exposed on + // this replica, so reload once from durable state before treating it as a + // normal, non-service request. + if let Err(error) = state.service_route_index.rebuild(&state.store).await { + warn!(error = %error, gateway_name, "failed to refresh service routing index"); + return None; + } + state.service_route_index.lookup(label) +} + +fn service_host_label<'a>(host: &'a str, config: &ServiceRoutingConfig) -> Option<&'a str> { + let host = host.split_once(':').map_or(host, |(name, _)| name); + for base_domain in &config.base_domains { + let suffix = format!(".{base_domain}"); + if let Some(label) = host.strip_suffix(&suffix) + && !label.is_empty() + && !label.contains('.') + { + return Some(label); + } + } + None +} + +fn is_hash_label(label: &str) -> bool { + label.len() == 16 + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || matches!(byte, b'2'..=b'7')) +} + +#[cfg(test)] pub fn is_sandbox_service_request(req: &Request, config: &ServiceRoutingConfig) -> bool { request_host(req).is_some_and(|host| parse_host(host, config).is_some()) } +pub fn service_request_host(req: &Request) -> Option { + request_host(req).map(str::to_string) +} + +pub async fn is_sandbox_service_request_with_state( + host: Option, + state: &ServerState, +) -> bool { + let Some(host) = host else { + return false; + }; + resolve_service_host(state, &host).await.is_some() +} + pub async fn proxy_sandbox_service_request( state: Arc, req: Request, ) -> impl IntoResponse { - let Some(host) = request_host(&req) else { + let Some(host) = request_host(&req).map(str::to_string) else { return StatusCode::NOT_FOUND.into_response(); }; - let Some((workspace, sandbox_name, service_name)) = - parse_host(host, &state.config.service_routing) - else { + let Some(target) = resolve_service_host(&state, &host).await else { return StatusCode::NOT_FOUND.into_response(); }; + let ServiceRouteTarget { + workspace, + sandbox: sandbox_name, + service, + } = target; + let service_name = service.unwrap_or_default(); match proxy_to_endpoint(state, req, &workspace, sandbox_name, service_name).await { Ok(response) => response.into_response(), @@ -851,8 +1131,7 @@ pub fn emit_cross_origin_service_http_rejection(state: &ServerState, req: &Reque let Some(host) = request_host(req) else { return; }; - let Some((_workspace, sandbox_name, service_name)) = - parse_host(host, &state.config.service_routing) + let Some((_, sandbox_name, service_name)) = parse_host(host, &state.config.service_routing) else { return; }; @@ -1165,6 +1444,122 @@ mod tests { ); } + #[test] + fn service_route_hash_label_uses_stable_fixed_vectors() { + assert_eq!( + service_route_hash_label("gateway-a", "default", "my-sandbox", Some("web")), + "enh47uqyj7bpyttz" + ); + assert_eq!( + service_route_hash_label("gateway-a", "default", "my-sandbox", None), + "eiud2z5apv3yeucg" + ); + assert_eq!( + service_route_hash_label("gateway-a", "default", "my-sandbox", Some("web")), + service_route_hash_label("gateway-a", "default", "my-sandbox", Some("web")) + ); + } + + #[test] + fn endpoint_url_uses_hash_label_when_gateway_name_is_configured() { + let cfg = openshell_core::Config::new(Some(tls_config())) + .with_bind_address("127.0.0.1:8080".parse().unwrap()) + .with_server_sans(["*.dev.openshell.localhost"]) + .with_service_routing_gateway_name(Some("gateway-a")); + + assert_eq!( + endpoint_url(&cfg, "default", "my-sandbox", "web").as_deref(), + Some("http://enh47uqyj7bpyttz.dev.openshell.localhost:8080/") + ); + assert_eq!( + endpoint_url(&cfg, "default", "my-sandbox", "").as_deref(), + Some("http://eiud2z5apv3yeucg.dev.openshell.localhost:8080/") + ); + } + + #[test] + fn service_route_index_is_idempotent_and_removes_matching_target() { + let index = ServiceRouteIndex::new(Some("gateway-a".to_string())); + index.register("default", "my-sandbox", "web").unwrap(); + index.register("default", "my-sandbox", "web").unwrap(); + let label = service_route_hash_label("gateway-a", "default", "my-sandbox", Some("web")); + assert!(index.lookup(&label).is_some()); + index.remove("default", "other-sandbox", "web"); + assert!(index.lookup(&label).is_some()); + index.remove("default", "my-sandbox", "web"); + assert!(index.lookup(&label).is_none()); + } + + #[tokio::test] + async fn service_route_index_rebuilds_persisted_endpoints() { + use prost::Message; + + let store = crate::persistence::test_store().await; + let endpoint = endpoint(); + store + .put_if( + ServiceEndpoint::object_type(), + endpoint.object_id(), + "my-sandbox--web", + "default", + &endpoint.encode_to_vec(), + None, + crate::persistence::WriteCondition::MustCreate, + ) + .await + .unwrap(); + let index = ServiceRouteIndex::new(Some("gateway-a".to_string())); + index.rebuild(&store).await.unwrap(); + let label = service_route_hash_label("gateway-a", "default", "my-sandbox", Some("web")); + assert_eq!( + index.lookup(&label), + Some(ServiceRouteTarget::new("default", "my-sandbox", "web")) + ); + } + + #[test] + fn service_route_index_rejects_seeded_collision() { + let index = ServiceRouteIndex::new(Some("gateway-a".to_string())); + let label = service_route_hash_label("gateway-a", "default", "my-sandbox", Some("web")); + index + .entries + .write() + .unwrap() + .insert(label, ServiceRouteTarget::new("other", "sandbox", "web")); + assert!(index.register("default", "my-sandbox", "web").is_err()); + } + + #[tokio::test] + async fn hashed_hosts_resolve_and_unknown_hashes_are_not_service_requests() { + let mut state = crate::grpc::test_support::test_server_state().await; + let state_mut = Arc::get_mut(&mut state).unwrap(); + state_mut.config = state_mut + .config + .clone() + .with_server_sans(["*.dev.openshell.localhost"]) + .with_service_routing_gateway_name(Some("gateway-a")); + state_mut.service_route_index = + Arc::new(ServiceRouteIndex::new(Some("gateway-a".to_string()))); + state + .service_route_index + .register("default", "my-sandbox", "web") + .unwrap(); + + let hash = service_route_hash_label("gateway-a", "default", "my-sandbox", Some("web")); + assert_eq!( + resolve_service_host(&state, &format!("{hash}.dev.openshell.localhost")).await, + Some(ServiceRouteTarget::new("default", "my-sandbox", "web")) + ); + assert_eq!( + resolve_service_host(&state, "aaaaaaaaaaaaaaaa.dev.openshell.localhost").await, + None + ); + assert_eq!( + resolve_service_host(&state, "default--my-sandbox--web.dev.openshell.localhost").await, + Some(ServiceRouteTarget::new("default", "my-sandbox", "web")) + ); + } + #[test] fn parses_sandbox_service_host() { assert_eq!( diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index fcb7a108e4..8f2d850e6f 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -566,6 +566,7 @@ discovery endpoint or its TLS CA. | server.sandboxJwt.ttlSecs | int | `3600` | Token TTL in seconds. Defaults to 3600 (1h). | | server.sandboxNamespace | string | `""` | Namespace where sandbox pods are created. Defaults to the Helm release namespace (.Release.Namespace) when left empty. | | server.sandboxUid | string | `""` | UID for sandbox pods (`sandbox_uid`). Empty (default) = use the OpenShift SCC namespace annotation if present, otherwise the driver default. Must be an integer between 1 and 4294967294. | +| server.serviceRoutingGatewayName | string | `""` | Optional routing name that disambiguates exposed-service hostnames when multiple gateways share one service base domain behind a single ingress and wildcard certificate. Must be unique among those gateways. When set, exposed-service hostnames become opaque fixed-length hashed labels (for example enh47uqyj7bpyttz.apps.example.com) instead of the readable workspace--sandbox--service composite. Unset preserves the composite hostnames. | | server.telemetryEnabled | bool | `true` | Enable anonymous OpenShell telemetry from the gateway and the sandbox supervisors it launches. | | server.tls.certSecretName | string | `"openshell-server-tls"` | K8s secret (type kubernetes.io/tls) with tls.crt and tls.key for the server. | | server.tls.clientCaSecretName | string | `"openshell-server-client-ca"` | K8s secret with ca.crt for client certificate verification (mTLS). Only used when enableMtls is true. Set to "" to disable client certificate verification for HTTPS-only mode. | diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index 121c93159d..5d3de7018a 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -110,6 +110,10 @@ Top-level lists represent TOML arrays of tables and preserve their YAML order. * {{- range $legacyKey, $runtimeKey := dict "logLevel" "log_level" "enableLoopbackServiceHttp" "enable_loopback_service_http" "enableWebsocketTunnel" "enable_websocket_tunnel" "policyValidationFailureMode" "policy_validation_failure_mode" -}} {{- if not (hasKey $gateway $runtimeKey) -}}{{- $_ := set $gateway $runtimeKey (get $legacyServer $legacyKey) -}}{{- end -}} {{- end -}} +{{/* An empty service routing name means "unset"; only translate it when set. */}} +{{- if and (get $legacyServer "serviceRoutingGatewayName") (not (hasKey $gateway "service_routing_gateway_name")) -}} +{{- $_ := set $gateway "service_routing_gateway_name" (get $legacyServer "serviceRoutingGatewayName") -}} +{{- end -}} {{- if not (hasKey $gateway "compute_driver") -}}{{- $_ := set $gateway "compute_driver" "kubernetes" -}}{{- end -}} {{- $serverDnsNames := .Values.pkiInitJob.serverDnsNames | default list -}} {{- if .Values.certManager.enabled -}}{{- $serverDnsNames = .Values.certManager.serverDnsNames | default list -}}{{- end -}} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index c433bbed69..5e7d43ddab 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -29,6 +29,34 @@ tests: path: data["gateway.toml"] pattern: '(?m)^enable_websocket_tunnel\s*=\s*true$' + - it: omits service_routing_gateway_name unless a routing name is configured + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'service_routing_gateway_name\s*=' + + - it: renders service_routing_gateway_name from server.serviceRoutingGatewayName + template: templates/gateway-config.yaml + set: + server.serviceRoutingGatewayName: gateway-west + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^service_routing_gateway_name\s*=\s*"gateway-west"$' + + - it: preserves an explicit gatewayConfig service_routing_gateway_name over the legacy value + template: templates/gateway-config.yaml + set: + server.serviceRoutingGatewayName: gateway-west + gatewayConfig: + openshell.gateway: + service_routing_gateway_name: gateway-east + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?m)^service_routing_gateway_name\s*=\s*"gateway-east"$' + - it: defaults to label admission with caller driver config disabled template: templates/gateway-config.yaml asserts: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 04fc1152a4..d3e8d768c4 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -467,6 +467,14 @@ server: # -- Enable plaintext HTTP routing for loopback sandbox service URLs on # TLS-enabled gateways. enableLoopbackServiceHttp: true + # -- Optional routing name that disambiguates exposed-service hostnames + # when multiple gateways share one service base domain behind a single + # ingress and wildcard certificate. Must be unique among those gateways. + # When set, exposed-service hostnames become opaque fixed-length hashed + # labels (for example enh47uqyj7bpyttz.apps.example.com) instead of the + # readable workspace--sandbox--service composite. Unset preserves the + # composite hostnames. + serviceRoutingGatewayName: "" # -- Enable the WebSocket tunnel used by CLI/SDK clients behind an # authenticated edge proxy. Leave disabled for direct gateway installs. enableWebsocketTunnel: false diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 7f16929759..f86e9fca1a 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -199,6 +199,12 @@ config_delivery_mode = "poll" server_sans = ["openshell", "*.dev.openshell.localhost"] # Allow plaintext HTTP routing for loopback sandbox service URLs. enable_loopback_service_http = true +# Disambiguate service hostnames when multiple gateways share one service +# base domain behind a single ingress. When set, service URLs use opaque +# hashed labels (https://./) instead of the readable +# ---- composite. Changing the name changes +# every issued URL. +service_routing_gateway_name = "gateway-west" # Serve the WebSocket tunnel for CLI clients behind an authenticating edge proxy. enable_websocket_tunnel = false @@ -341,6 +347,46 @@ External compute drivers declare this requirement by adding `openshell.compute.l `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. +## Service Routing + +Sandbox service URLs default to a readable composite hostname: +`https://----./` for named services +and `https://--./` for the unnamed endpoint. +The base domain is the first wildcard DNS SAN in `server_sans`; gateways +without SANs use the built-in `openshell.localhost`. + +`service_routing_gateway_name` disambiguates service hostnames when multiple +gateways share one service base domain behind a single ingress and wildcard +certificate. Identically named workspaces, sandboxes, and services on two such +gateways produce identical composite hostnames, so the shared ingress cannot +select the correct gateway. Assign each gateway a distinct routing name with +`[openshell.gateway].service_routing_gateway_name`, +`--service-routing-gateway-name`, or +`OPENSHELL_SERVICE_ROUTING_GATEWAY_NAME`. The field is unset by default. Its +precedence is `--service-routing-gateway-name` > +`OPENSHELL_SERVICE_ROUTING_GATEWAY_NAME` > TOML > unset. + +The value must match `[a-z0-9]([-a-z0-9]*[a-z0-9])?` and be at most 63 bytes. +It is a hostname label only: it is unrelated to the gateway installation +`name` (used for telemetry identity) and to `gateway_jwt.gateway_id` (the +sandbox JWT issuer and Kubernetes ownership identity). + +When set, service URLs replace the readable composite label with a +deterministic 16-character hash of the routing name, workspace, sandbox, and +service name: `https://./` (for example, +`https://enh47uqyj7bpyttz.apps.example.com/`, the label a gateway named +`gateway-a` issues for workspace `default`, sandbox `my-sandbox`, service +`web`). The hash stays one DNS label +deep, so the existing single wildcard certificate keeps working, and it is +unique per gateway, so exact-host ingress rules can route it unambiguously. +The gateway still accepts requests for legacy composite hostnames while a +routing name is configured. + +Service URLs are recomputed from the current configuration, not persisted. +Changing or removing the routing name changes every hash URL issued before +the change; previously issued URLs stop resolving until callers request a +fresh URL. + ## Database Pool `[openshell.gateway] database_max_connections` sets the connection ceiling for the gateway's database pool. One pool is shared by every database-backed RPC, so this value is also the gateway's ceiling on concurrent database work. When the pool is saturated, callers queue on connection acquisition and sqlx logs that the time to acquire a connection exceeded its slow threshold. The background readiness monitor draws from the same pool, so sustained saturation can push its database ping past the check timeout and flip `/readyz` to not-ready while the gateway is otherwise serving. diff --git a/docs/how-it-works/gateways/overview.mdx b/docs/how-it-works/gateways/overview.mdx index fe12dc6b1f..cd3761c719 100644 --- a/docs/how-it-works/gateways/overview.mdx +++ b/docs/how-it-works/gateways/overview.mdx @@ -42,7 +42,7 @@ For driver setup, including Docker, Podman, MicroVM, and Kubernetes paths, refer Sandbox service routing is enabled for gateways by default. Users expose long-running sandbox services with `openshell service expose`, and the gateway routes browser traffic from the printed service URL to the loopback port inside the sandbox. -Loopback gateways use `openshell.localhost` service URLs. OpenShell prints a URL in the form `http://.openshell.localhost:/` or `http://--.openshell.localhost:/` when a service name is provided. +Loopback gateways use `openshell.localhost` service URLs. OpenShell prints a URL in the form `http://--.openshell.localhost:/`, or `http://----.openshell.localhost:/` when a service name is provided. Browser traffic enters the same gateway listener as mTLS-protected gRPC, but plaintext HTTP is accepted only from loopback clients and only for sandbox service hostnames. Gateway APIs, auth routes, health endpoints, and non-service hostnames remain unavailable over plaintext HTTP. Cross-origin and sibling-subdomain browser requests are rejected before reaching the sandbox service. @@ -50,6 +50,8 @@ Disable the local browser path with `--enable-loopback-service-http=false` or `O Custom HTTPS service domains use the gateway server SAN configuration. Add a wildcard DNS SAN such as `*.apps.example.com` to the gateway certificate and pass the same SAN to the gateway with `--server-san` or `OPENSHELL_SERVER_SAN`. +When multiple gateways share one service domain behind a single ingress, each gateway can set a service routing name with `--service-routing-gateway-name` or `[openshell.gateway].service_routing_gateway_name`. With a routing name set, service URLs use an opaque fixed-length hashed label — `https://.apps.example.com/` — that is unique per gateway, instead of the readable composite hostname. See [Configuration](/how-it-works/gateways/configuration#service-routing) for the field, its validation, and the effects of changing it. + For remote or non-loopback gateways, browser service URLs remain HTTPS and require normal gateway authentication. ## Register an Existing Gateway diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 430e90c92f..aadfdbe281 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -684,6 +684,12 @@ certificate—still applies. The application is responsible for authentication enabled on its endpoint, and an upstream edge proxy may still apply its own access policy. For gateway service-domain configuration, refer to [Manage Gateways](/how-it-works/gateways/overview#configure-service-forwarding). + +Consume the URL the CLI or SDK prints rather than constructing hostnames from +workspace, sandbox, and service names. The hostname format is a gateway +configuration detail: gateways configured with a service routing name return +opaque hashed hostnames (`https://./`) instead of the +readable composite. ## Monitor and Debug diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index b17d80640c..32640cccc8 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -267,6 +267,49 @@ Sandbox SSH uses the gateway endpoint registered with the CLI. No separate Helm When a Gateway terminates TLS, register its externally reachable `https://` endpoint with the CLI. OpenShell uses that registered endpoint for `sandbox connect`, `sandbox exec`, and port forwarding, even when the gateway pod reports an internal plaintext endpoint. +## Route Sandbox Service Traffic + +The chart creates only a `Gateway` and a `GRPCRoute`. Browser and WebSocket +traffic to exposed sandbox services — the URLs printed by +`openshell service expose` — has no chart-managed route. Operators configure +it themselves. + +Service URLs use a hostname under the gateway's service base domain, which +comes from the wildcard SAN in `pkiInitJob.serverDnsNames` or +`certManager.serverDnsNames` (for example `*.apps.example.com`). Create an +`HTTPRoute` or `Ingress` that matches those hostnames and forwards to the +OpenShell gateway Service, terminating TLS on the shared wildcard certificate. +The route must preserve the original `Host` header: the gateway parses it to +select the sandbox service. + +### Multiple gateways behind one ingress + +When two gateways share a cluster, one ingress, and one wildcard certificate, +identically named workspaces, sandboxes, and services produce identical +composite hostnames, so the ingress cannot select the correct gateway. Give +each gateway a unique routing name: + +```shell +helm upgrade openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set server.serviceRoutingGatewayName=gateway-a +``` + +With a routing name set, each gateway emits opaque, fixed-length hashed +hostnames (`https://.apps.example.com/`) for its exposed services. Hash +hostnames are deterministic and unique per gateway, so exact-host +`HTTPRoute`/`Ingress` rules route them unambiguously with the single shared +wildcard certificate. The hostnames exist only once services are exposed, so +the rules must be created as services are exposed — for example by automation +that reads `openshell service list` or the gateway API and reconciles +exact-host routes per gateway. The chart does not create or manage these +routes. + +See [Gateway configuration](/how-it-works/gateways/configuration#service-routing) +for the routing-name field, its validation, and the effects of changing it. + ## Next Steps Return to [Setup](/kubernetes/setup) to complete the installation. diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index ae0be7d027..be315f9a9c 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -305,6 +305,7 @@ The most commonly changed values are: | `server.disableTls` | Run the gateway over plaintext HTTP. Use only behind a trusted transport. | | `server.auth.allowUnauthenticatedUsers` | Accept user-facing calls without OIDC or mTLS credentials. Use only for trusted local development or a fully trusted access proxy. | | `server.enableLoopbackServiceHttp` | Enable local plaintext HTTP for loopback sandbox service URLs. Defaults to `true`. | +| `server.serviceRoutingGatewayName` | Optional routing name that makes exposed-service hostnames unique per gateway when multiple gateways share one service base domain behind a single ingress. Unset keeps the readable composite hostnames. Refer to [Ingress](/kubernetes/ingress#multiple-gateways-behind-one-ingress). | | `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. | | `supervisor.sandboxRuntime.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. | | `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). |