From 1c8e5a4d41b38c1eb735b6ecc9861647099dbf5b Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 26 Oct 2026 21:07:09 -0700 Subject: [PATCH] feat(auth): add certificate-bound discovery readers and workspace scopes Implement discovery-only certificate grants, bounded single/set/all workspace views, and live grant and certificate expiry checks. Preserve existing user authorization and reject discovery credentials on other APIs and application requests. Closes #4668 Signed-off-by: Taylor Mutch --- crates/openshell-core/src/config.rs | 3 + crates/openshell-server/README.md | 17 + crates/openshell-server/src/auth/audit.rs | 7 +- crates/openshell-server/src/auth/discovery.rs | 192 +++++++ crates/openshell-server/src/auth/guard.rs | 5 +- crates/openshell-server/src/auth/mod.rs | 1 + crates/openshell-server/src/auth/principal.rs | 10 + .../src/auth/service_access.rs | 9 + .../src/auth/workspace_authz.rs | 4 +- crates/openshell-server/src/cli.rs | 13 + crates/openshell-server/src/grpc/policy.rs | 8 +- crates/openshell-server/src/grpc/provider.rs | 3 +- .../src/grpc/service_discovery.rs | 494 +++++++++++++++++- crates/openshell-server/src/grpc/workspace.rs | 8 +- crates/openshell-server/src/lib.rs | 122 ++++- crates/openshell-server/src/multiplex.rs | 98 +++- crates/openshell-server/src/storage_proto.rs | 2 +- docs/how-it-works/gateways/authentication.mdx | 9 + docs/how-it-works/gateways/configuration.mdx | 46 ++ proto/README.md | 4 +- proto/openshell.proto | 8 +- sdk/go/proto/openshellv1/openshell.pb.go | 44 +- skills/debug-openshell-cluster/SKILL.md | 10 + 23 files changed, 1072 insertions(+), 45 deletions(-) create mode 100644 crates/openshell-server/src/auth/discovery.rs diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 7305d667e5..0bec8e7448 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -536,6 +536,9 @@ pub struct MtlsAuthConfig { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct GatewayAuthConfig { + /// Operator-managed JSON discovery certificate grant file, reloaded on requests/watches. + #[serde(default)] + pub discovery_grants_file: Option, /// When true, unauthenticated user/CLI calls are accepted as a local /// developer principal. This is an unsafe local-development escape hatch /// for trusted, non-shared gateways. Sandbox supervisor calls still use diff --git a/crates/openshell-server/README.md b/crates/openshell-server/README.md index aa54a0218c..841382f25a 100644 --- a/crates/openshell-server/README.md +++ b/crates/openshell-server/README.md @@ -273,3 +273,20 @@ return preferred `url`, configured `application_url`, and a gateway alternative deployment origins are recomputed for durable replay without repeating the mutation or changing resource identities. These are configured addresses, not installation, ingress acceptance, or application health signals. + +Discovery accepts an explicit workspace, up to 64 distinct `workspaces`, or the +existing all-workspaces selector. Omission and combined selectors fail. +Workspace users retain role/scope/membership authorization for every requested +workspace; all-workspaces access requires platform administration or a dedicated +certificate grant. Discovery certificates carry `OU=discovery`, use the configured +TLS client CA, and receive authority only through the operator-managed fingerprint +mapping at `gateway.auth.discovery_grants_file`. They never become user/operator +principals. Certificate expiry and grant removal are rechecked on active watches. + +All-workspaces collection includes future active workspaces and omits terminating +ones, with a 64-workspace scan bound. Selected-workspace collection does not scan +unrelated workspaces. Combined snapshots retain the 4,096-route/4-MiB output bound +and ten-second collection deadline; each workspace scan retains its existing +1,000-record/4-MiB input bounds. Overflow and storage failures are errors, never +partial successful views. A workspace-set change during collection requires a +retry; this is not a cross-resource transactional snapshot. diff --git a/crates/openshell-server/src/auth/audit.rs b/crates/openshell-server/src/auth/audit.rs index 6c9136aa6f..56ab76dbbf 100644 --- a/crates/openshell-server/src/auth/audit.rs +++ b/crates/openshell-server/src/auth/audit.rs @@ -216,7 +216,8 @@ fn authenticated(principal: &Principal) -> bool { match principal { Principal::User(user) => user.identity.provider != IdentityProvider::LocalDev, Principal::Anonymous => false, - Principal::Operator(_) + Principal::Discovery(_) + | Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) | Principal::Supervisor(_) => true, @@ -226,7 +227,7 @@ fn authenticated(principal: &Principal) -> bool { /// The mechanism that established `principal`, when it has an audit label. fn mechanism(principal: &Principal) -> Option { match principal { - Principal::Operator(_) => Some(AuthMechanism::Mtls), + Principal::Discovery(_) | Principal::Operator(_) => Some(AuthMechanism::Mtls), Principal::User(user) => match user.identity.provider { IdentityProvider::Oidc => Some(AuthMechanism::Oidc), IdentityProvider::Mtls => Some(AuthMechanism::Mtls), @@ -253,6 +254,7 @@ fn credential_digest( peer_certificate_sha256: Option<&str>, ) -> Option<[u8; 32]> { let credential = match principal { + Principal::Discovery(reader) => reader.certificate_sha256.as_str(), Principal::Operator(operator) => operator.certificate_sha256.as_str(), Principal::User(user) if user.identity.provider == IdentityProvider::Mtls => { peer_certificate_sha256.unwrap_or_default() @@ -513,6 +515,7 @@ fn principal_user(principal: &Principal) -> User { Principal::Supervisor(supervisor) => { User::other(&supervisor.identity.subject, "Supervisor") } + Principal::Discovery(reader) => User::other(&reader.certificate_sha256, "Discovery Reader"), Principal::Anonymous => User::unidentified(), } } diff --git a/crates/openshell-server/src/auth/discovery.rs b/crates/openshell-server/src/auth/discovery.rs new file mode 100644 index 0000000000..81150b7294 --- /dev/null +++ b/crates/openshell-server/src/auth/discovery.rs @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Operator-managed grants for verified discovery client certificates. + +use super::{ + identity::{Identity, IdentityProvider}, + principal::DiscoveryPrincipal, +}; +use serde::Deserialize; +use std::collections::BTreeSet; +use std::path::Path; +use tokio::io::AsyncReadExt as _; +use tonic::Status; + +pub const MAX_WORKSPACES: usize = 64; +const MAX_FILE_BYTES: usize = 256 * 1024; + +/// Certificate OU marks a restricted credential class, never an authority grant. +/// The separate operator-controlled fingerprint mapping supplies authority. +pub fn is_certificate(identity: &Identity) -> bool { + identity.provider == IdentityProvider::Mtls && identity.roles.iter().any(|r| r == "discovery") +} + +pub fn is_method(path: &str) -> bool { + matches!( + path, + "/openshell.v1.OpenShell/GetServiceRouteSnapshot" + | "/openshell.v1.OpenShell/WatchServiceRoutes" + ) && super::method_authz::lookup(path).is_some_and(|entry| { + entry.auth_mode == super::method_authz::AuthMode::Bearer + && entry.scope.as_deref() == Some("sandbox:read") + && entry.workspace_role.as_deref() == Some("user") + && entry.global_role.is_none() + }) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Grants { + readers: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Grant { + pub certificate_sha256: String, + #[serde(default)] + pub workspaces: Vec, + #[serde(default)] + pub all_workspaces: bool, +} + +pub async fn grant(path: Option<&Path>, principal: &DiscoveryPrincipal) -> Result { + tokio::time::timeout( + std::time::Duration::from_secs(2), + load_grant(path, principal), + ) + .await + .map_err(|_| Status::unavailable("discovery grant read timed out"))? +} + +async fn load_grant(path: Option<&Path>, principal: &DiscoveryPrincipal) -> Result { + if principal.expires_at_ms <= crate::persistence::current_time_ms() { + return Err(Status::unauthenticated("discovery certificate expired")); + } + let path = path.ok_or_else(|| Status::permission_denied("discovery grant required"))?; + let file = tokio::fs::File::open(path) + .await + .map_err(|_| Status::unavailable("discovery grants unavailable"))?; + let mut bytes = Vec::new(); + file.take( + u64::try_from(MAX_FILE_BYTES + 1).map_err(|_| Status::internal("invalid file limit"))?, + ) + .read_to_end(&mut bytes) + .await + .map_err(|_| Status::unavailable("discovery grants unavailable"))?; + if bytes.len() > MAX_FILE_BYTES { + return Err(Status::resource_exhausted( + "discovery grants file too large", + )); + } + let grants: Grants = serde_json::from_slice(&bytes) + .map_err(|_| Status::unavailable("invalid discovery grants"))?; + if grants.readers.len() > 1024 { + return Err(Status::resource_exhausted("too many discovery grants")); + } + let mut fingerprints = BTreeSet::new(); + for reader in &grants.readers { + if reader.certificate_sha256.len() != 64 + || !reader + .certificate_sha256 + .bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) + || !fingerprints.insert(&reader.certificate_sha256) + || reader.workspaces.len() > MAX_WORKSPACES + || reader.all_workspaces != reader.workspaces.is_empty() + { + return Err(Status::unavailable("invalid discovery grant mapping")); + } + let mut names = BTreeSet::new(); + for workspace in &reader.workspaces { + crate::grpc::workspace::validate_workspace_name(workspace) + .map_err(|_| Status::unavailable("invalid discovery workspace grant"))?; + if !names.insert(workspace) { + return Err(Status::unavailable("duplicate discovery workspace grant")); + } + } + } + grants + .readers + .into_iter() + .find(|g| g.certificate_sha256 == principal.certificate_sha256) + .ok_or_else(|| Status::permission_denied("discovery grant revoked or missing")) +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn fingerprint_grants_reload_and_never_derive_authority_from_subject_roles() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("grants.json"); + let reader = DiscoveryPrincipal { + certificate_sha256: "a".repeat(64), + expires_at_ms: crate::persistence::current_time_ms() + 60_000, + }; + let write = |readers: serde_json::Value| { + std::fs::write( + &path, + serde_json::to_vec(&serde_json::json!({"readers":readers})).unwrap(), + ) + .unwrap(); + }; + write( + serde_json::json!([{"certificate_sha256":reader.certificate_sha256,"workspaces":["default"]}]), + ); + assert_eq!( + grant(Some(&path), &reader).await.unwrap().workspaces, + vec!["default"] + ); + let unrelated = DiscoveryPrincipal { + certificate_sha256: "b".repeat(64), + ..reader.clone() + }; + assert_eq!( + grant(Some(&path), &unrelated).await.err().unwrap().code(), + tonic::Code::PermissionDenied + ); + write(serde_json::json!([])); + assert_eq!( + grant(Some(&path), &reader).await.err().unwrap().code(), + tonic::Code::PermissionDenied + ); + write( + serde_json::json!([{"certificate_sha256":reader.certificate_sha256,"all_workspaces":true,"workspaces":["default"]}]), + ); + assert_eq!( + grant(Some(&path), &reader).await.err().unwrap().code(), + tonic::Code::Unavailable + ); + assert_eq!( + grant( + Some(&path), + &DiscoveryPrincipal { + expires_at_ms: 0, + ..reader + } + ) + .await + .err() + .unwrap() + .code(), + tonic::Code::Unauthenticated + ); + } + #[test] + fn every_other_rpc_is_denied_including_unclassified_methods() { + for path in super::super::method_authz::all_paths() { + assert_eq!( + is_method(path), + matches!( + path, + "/openshell.v1.OpenShell/GetServiceRouteSnapshot" + | "/openshell.v1.OpenShell/WatchServiceRoutes" + ), + "{path}" + ); + } + assert!(!is_method("/openshell.v1.OpenShell/FutureReadMethod")); + } +} diff --git a/crates/openshell-server/src/auth/guard.rs b/crates/openshell-server/src/auth/guard.rs index 2578ec44be..95f3fd4344 100644 --- a/crates/openshell-server/src/auth/guard.rs +++ b/crates/openshell-server/src/auth/guard.rs @@ -30,7 +30,7 @@ use tonic::Status; pub fn ensure_sandbox_scope(principal: &Principal, claimed_sandbox_id: &str) -> Result<(), Status> { match principal { Principal::User(_) | Principal::Operator(_) => Ok(()), - Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( + Principal::Discovery(_) | Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, "gateway peer and supervisor principals may not call sandbox-scoped methods", @@ -88,7 +88,8 @@ pub fn ensure_sandbox_principal_scope( ensure_sandbox_scope(principal, claimed_sandbox_id)?; Ok(p.clone()) } - Principal::User(_) + Principal::Discovery(_) + | Principal::User(_) | Principal::Operator(_) | Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( diff --git a/crates/openshell-server/src/auth/mod.rs b/crates/openshell-server/src/auth/mod.rs index f4ae2d08a9..d51ef2e97d 100644 --- a/crates/openshell-server/src/auth/mod.rs +++ b/crates/openshell-server/src/auth/mod.rs @@ -13,6 +13,7 @@ pub mod authenticator; pub mod authz; pub mod compute_driver; pub mod descriptor_authz; +pub mod discovery; pub mod extension_mint_limit; pub mod guard; mod http; diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs index 495be046d8..da15b400fa 100644 --- a/crates/openshell-server/src/auth/principal.rs +++ b/crates/openshell-server/src/auth/principal.rs @@ -23,6 +23,8 @@ use openshell_core::jwt::CredentialEpoch; /// retrieve it via `req.extensions().get::()`. #[derive(Debug, Clone)] pub enum Principal { + /// Restricted verified certificate; authority is reloaded from an administrator grant. + Discovery(DiscoveryPrincipal), /// Human caller authenticated via OIDC (Keycloak, Entra ID, Okta, etc.). User(UserPrincipal), /// Gateway-wide administrator authenticated only by a verified operator certificate. @@ -59,6 +61,13 @@ pub struct OperatorPrincipal { pub certificate_sha256: String, } +/// Verified TLS discovery identity. No general user, role, or application authority. +#[derive(Debug, Clone)] +pub struct DiscoveryPrincipal { + pub certificate_sha256: String, + pub expires_at_ms: i64, +} + impl Principal { /// Identity of a human/API caller, preserving its distinct principal variant. pub fn user_identity(&self) -> Option<&Identity> { @@ -148,6 +157,7 @@ impl Principal { Self::Sandbox(sandbox) => format!("sandbox:{}", sandbox.sandbox_id), Self::Peer(peer) => format!("peer:{}", peer.replica_id), Self::Supervisor(supervisor) => format!("supervisor:{}", supervisor.identity.subject), + Self::Discovery(reader) => format!("discovery:{}", reader.certificate_sha256), Self::Anonymous => "anonymous".to_string(), } } diff --git a/crates/openshell-server/src/auth/service_access.rs b/crates/openshell-server/src/auth/service_access.rs index 76236d8c1e..abf80f541c 100644 --- a/crates/openshell-server/src/auth/service_access.rs +++ b/crates/openshell-server/src/auth/service_access.rs @@ -57,6 +57,15 @@ async fn authorize_with_chain( workspace: &str, chain: Option, ) -> Result { + if peer + .identity + .as_ref() + .is_some_and(super::discovery::is_certificate) + { + return Err(Status::permission_denied( + "discovery certificates cannot access applications", + )); + } let origin = RequestOrigin::new(peer.source, headers); let principal = authenticate(state, headers, peer, chain).await.map_err( |(status, mechanism, reason)| { diff --git a/crates/openshell-server/src/auth/workspace_authz.rs b/crates/openshell-server/src/auth/workspace_authz.rs index bc5cac8778..89e3261011 100644 --- a/crates/openshell-server/src/auth/workspace_authz.rs +++ b/crates/openshell-server/src/auth/workspace_authz.rs @@ -207,7 +207,7 @@ pub async fn authorize_workspace( workspace, grant: AuthGrant::Sandbox, }), - Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( + Principal::Discovery(_) | Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, "gateway peer and supervisor principals cannot perform workspace operations", @@ -244,7 +244,7 @@ pub fn require_platform_admin(admin_role: &str, principal: &Principal) -> Result } Principal::User(_) => "platform admin role required for cross-workspace operations", Principal::Sandbox(_) => "sandbox principals cannot perform cross-workspace operations", - Principal::Peer(_) | Principal::Supervisor(_) => { + Principal::Discovery(_) | Principal::Peer(_) | Principal::Supervisor(_) => { "gateway peer and supervisor principals cannot perform cross-workspace operations" } Principal::Anonymous => return Err(Status::unauthenticated("authentication required")), diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index f4465ced1d..cedfb26426 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -729,7 +729,20 @@ fn prepare_server_config_with_drivers( /// Reject an implicit development gateway before starting any listeners or services. fn validate_user_auth(config: &openshell_core::Config) -> Result<()> { + if config.auth.discovery_grants_file.is_some() + && config + .tls + .as_ref() + .and_then(|tls| tls.client_ca_path.as_ref()) + .is_none() + { + return Err(miette::miette!( + "discovery grants require TLS and a client CA" + )); + } + if !config.auth.allow_unauthenticated_users + && config.auth.discovery_grants_file.is_none() && !config.mtls_auth.enabled && !config.mtls_auth.operator_enabled && config.oidc.is_none() diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 5b97da32cd..a045a12450 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -2788,9 +2788,11 @@ async fn resolve_sandbox_by_name_for_principal( Principal::User(_) | Principal::Operator(_) => { sandbox.ok_or_else(|| Status::not_found("sandbox not found")) } - Principal::Peer(_) | Principal::Supervisor(_) => Err(Status::permission_denied( - "gateway peer and supervisor principals may not resolve sandbox configuration", - )), + Principal::Discovery(_) | Principal::Peer(_) | Principal::Supervisor(_) => { + Err(Status::permission_denied( + "gateway peer and supervisor principals may not resolve sandbox configuration", + )) + } Principal::Anonymous => Err(Status::unauthenticated( "sandbox-scoped methods require an authenticated caller", )), diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 4b5a6b2207..ebaa96a700 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -4290,7 +4290,8 @@ async fn exchange_provider_subject_token_inner( let auth_code = match request.extensions().get::() { Some(Principal::Sandbox(principal)) if principal.sandbox_id == req.sandbox_id => None, Some( - Principal::Sandbox(_) + Principal::Discovery(_) + | Principal::Sandbox(_) | Principal::User(_) | Principal::Operator(_) | Principal::Peer(_) diff --git a/crates/openshell-server/src/grpc/service_discovery.rs b/crates/openshell-server/src/grpc/service_discovery.rs index d552833d51..180232d62e 100644 --- a/crates/openshell-server/src/grpc/service_discovery.rs +++ b/crates/openshell-server/src/grpc/service_discovery.rs @@ -20,7 +20,7 @@ use super::service_advertisement::{Installation, OBJECT_TYPE, is_current}; use crate::ServerState; use crate::auth::principal::Principal; use crate::auth::workspace_authz::{ - MinWorkspaceRole, authorize_workspace, selected_workspace_name, + MinWorkspaceRole, authorize_workspace, require_platform_admin, selected_workspace_name, }; use crate::persistence::{ObjectCursor, ObjectRecord, current_time_ms}; @@ -57,15 +57,202 @@ async fn authorize( Ok(()) } +#[derive(Clone, Debug)] +enum Selection { + Named(Vec), + All, +} + +fn selection( + scope: Option<&openshell_core::proto::WorkspaceSelector>, + names: &[String], +) -> Result { + if !names.is_empty() { + if scope.is_some() || names.len() > crate::auth::discovery::MAX_WORKSPACES { + return Err(Status::invalid_argument( + "choose a workspace selector or up to 64 workspaces", + )); + } + let mut names = names.to_vec(); + for name in &names { + super::workspace::validate_workspace_name(name)?; + } + names.sort(); + if names.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(Status::invalid_argument("duplicate workspace selection")); + } + return Ok(Selection::Named(names)); + } + if scope.is_some_and(|scope| { + matches!( + scope.selection, + Some(openshell_core::proto::workspace_selector::Selection::AllWorkspaces(_)) + ) + }) { + return Ok(Selection::All); + } + Ok(Selection::Named(vec![ + selected_workspace_name(scope)?.to_string(), + ])) +} + +async fn authorize_selection( + state: &ServerState, + principal: &Principal, + selection: &Selection, +) -> Result<(), Status> { + if let Principal::Discovery(reader) = principal { + let grant = crate::auth::discovery::grant( + state.config.auth.discovery_grants_file.as_deref(), + reader, + ) + .await?; + match selection { + Selection::All if !grant.all_workspaces => { + return Err(Status::permission_denied( + "all-workspaces discovery grant required", + )); + } + Selection::Named(names) + if !grant.all_workspaces + && names.iter().any(|name| !grant.workspaces.contains(name)) => + { + return Err(Status::permission_denied( + "workspace not granted for discovery", + )); + } + _ => return Ok(()), + } + } + match selection { + Selection::All => { + if let Principal::User(user) = principal { + let explicit_dev = state.config.auth.allow_unauthenticated_users + && user.identity.provider == crate::auth::identity::IdentityProvider::LocalDev; + if !explicit_dev && state.admin_role.is_empty() { + return Err(Status::permission_denied( + "explicit platform discovery authority required", + )); + } + } + require_platform_admin(&state.admin_role, principal) + } + Selection::Named(names) => { + for name in names { + authorize(state, principal, name).await?; + } + Ok(()) + } + } +} + +async fn active_workspaces(state: &ServerState) -> Result, Status> { + let mut cursor: Option = None; + let mut names = Vec::new(); + let mut count = 0; + let mut bytes = 0usize; + loop { + let page = state + .store + .list_by_type_after("workspace", cursor.as_ref(), 65) + .await + .map_err(|_| Status::unavailable("workspace discovery storage unavailable"))?; + count += page.len(); + for record in &page { + bytes = bytes.saturating_add(record.payload.len()); + if count > crate::auth::discovery::MAX_WORKSPACES || bytes > MAX_BYTES { + return Err(Status::resource_exhausted( + "workspace discovery limit exceeded", + )); + } + let workspace = openshell_core::proto::Workspace::decode(record.payload.as_slice()) + .map_err(|_| Status::internal("invalid workspace record"))?; + let metadata = workspace + .metadata + .ok_or_else(|| Status::internal("workspace metadata missing"))?; + if metadata.deletion_time.is_none() { + names.push(metadata.name); + } + } + cursor = page.last().map(ObjectCursor::from); + if page.len() < 65 { + break; + } + } + names.sort(); + Ok(names) +} + +async fn selection_workspaces( + state: &ServerState, + selection: &Selection, +) -> Result, Status> { + match selection { + Selection::All => active_workspaces(state).await, + Selection::Named(names) => { + let mut active = Vec::new(); + for name in names { + match super::workspace::resolve_workspace(state.store.as_ref(), name).await { + Ok(workspace) if !workspace.terminating => active.push(name.clone()), + Ok(_) => {} + Err(error) if error.code() == tonic::Code::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(active) + } + } +} + +async fn selected_snapshot( + state: &ServerState, + principal: &Principal, + selection: &Selection, +) -> Result { + tokio::time::timeout(SNAPSHOT_TIMEOUT, async { + authorize_selection(state, principal, selection).await?; + let before = selection_workspaces(state, selection).await?; + let names = before.clone(); + let mut result = ServiceRouteSnapshot::default(); + let mut bytes = 0usize; + for name in names { + let part = snapshot(state, &name).await?; + bytes = bytes.saturating_add(part.encoded_len()); + if bytes > MAX_BYTES - 128 || result.routes.len() + part.routes.len() > MAX_ROUTES { + return Err(Status::resource_exhausted( + "combined discovery limit exceeded", + )); + } + result.routes.extend(part.routes); + } + if before != selection_workspaces(state, selection).await? { + return Err(Status::unavailable( + "workspace set changed during discovery; retry", + )); + } + // Recheck grants/membership after collection, before publishing a view. + authorize_selection(state, principal, selection).await?; + result.revision = hex::encode( + openshell_crypto::sha256(&result.encode_to_vec()) + .map_err(|_| Status::internal("snapshot fingerprint failed"))?, + ); + Ok(result) + }) + .await + .map_err(|_| Status::unavailable("service discovery collection timed out"))? +} + pub async fn get_snapshot( state: &Arc, request: Request, ) -> Result, Status> { let principal = super::extract_principal(&request)?; - let workspace = selected_workspace_name(request.get_ref().workspace_scope.as_ref())?; - authorize(state, &principal, workspace).await?; + let selection = selection( + request.get_ref().workspace_scope.as_ref(), + &request.get_ref().workspaces, + )?; Ok(Response::new(GetServiceRouteSnapshotResponse { - snapshot: Some(snapshot(state, workspace).await?), + snapshot: Some(selected_snapshot(state, &principal, &selection).await?), })) } @@ -74,15 +261,22 @@ pub async fn watch( request: Request, ) -> Result, Status> { let principal = super::extract_principal(&request)?; - let workspace = - selected_workspace_name(request.get_ref().workspace_scope.as_ref())?.to_string(); - authorize(state, &principal, &workspace).await?; + let selection = selection( + request.get_ref().workspace_scope.as_ref(), + &request.get_ref().workspaces, + )?; + tokio::time::timeout( + SNAPSHOT_TIMEOUT, + authorize_selection(state, &principal, &selection), + ) + .await + .map_err(|_| Status::unavailable("discovery authorization timed out"))??; let permit = state .discovery_watches .clone() .try_acquire_owned() .map_err(|_| Status::resource_exhausted("too many service discovery watches"))?; - let initial = snapshot(state, &workspace).await?; + let initial = selected_snapshot(state, &principal, &selection).await?; let mut revision = initial.revision.clone(); let (tx, rx) = tokio::sync::mpsc::channel(1); tx.try_send(Ok(WatchServiceRoutesResponse { @@ -102,10 +296,7 @@ pub async fn watch( () = tokio::time::sleep_until(deadline) => return, _ = interval.tick() => {} } - let refresh = async { - authorize(&state, &principal, &workspace).await?; - snapshot(&state, &workspace).await - }; + let refresh = async { selected_snapshot(&state, &principal, &selection).await }; let result = tokio::select! { () = tx.closed() => return, () = tokio::time::sleep_until(deadline) => return, @@ -297,6 +488,7 @@ mod tests { fn request() -> GetServiceRouteSnapshotRequest { GetServiceRouteSnapshotRequest { workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() } } @@ -371,6 +563,7 @@ mod tests { &state, authed_request(WatchServiceRoutesRequest { workspace_scope: request().workspace_scope, + ..Default::default() }), ) .await @@ -435,11 +628,12 @@ mod tests { get_snapshot( &state, authed_request(GetServiceRouteSnapshotRequest { - workspace_scope: Some(openshell_core::proto::all_workspaces_selector()) + workspace_scope: Some(openshell_core::proto::all_workspaces_selector()), + ..Default::default() }) ) .await - .is_err() + .is_ok() ); let mut outsider = authed_request(request()); if let Some(Principal::User(user)) = outsider.extensions_mut().get_mut::() { @@ -466,6 +660,7 @@ mod tests { &state, authed_request(WatchServiceRoutesRequest { workspace_scope: request().workspace_scope, + ..Default::default() }), ) .await @@ -490,6 +685,7 @@ mod tests { state.store.put_message(&member).await.unwrap(); let mut request = authed_request(WatchServiceRoutesRequest { workspace_scope: request().workspace_scope, + ..Default::default() }); if let Some(Principal::User(user)) = request.extensions_mut().get_mut::() { user.identity.roles = vec!["openshell-user".into()]; @@ -519,6 +715,7 @@ mod tests { &state, authed_request(WatchServiceRoutesRequest { workspace_scope: request().workspace_scope, + ..Default::default() }), ) .await @@ -583,4 +780,273 @@ mod tests { tonic::Code::Unavailable ); } + async fn reader_fixture(all: bool) -> (Arc, Principal, tempfile::TempDir) { + let (mut state, owner, report) = installed_fixture().await; + accept(&state, &owner, &report, current_time_ms() + 10_000) + .await + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("grants.json"); + let fingerprint = "a".repeat(64); + let grant = if all { + serde_json::json!({"certificate_sha256":fingerprint,"all_workspaces":true}) + } else { + serde_json::json!({"certificate_sha256":fingerprint,"workspaces":["default","team-b"]}) + }; + std::fs::write( + &path, + serde_json::to_vec(&serde_json::json!({"readers":[grant]})).unwrap(), + ) + .unwrap(); + Arc::get_mut(&mut state) + .unwrap() + .config + .auth + .discovery_grants_file = Some(path); + let principal = Principal::Discovery(crate::auth::principal::DiscoveryPrincipal { + certificate_sha256: fingerprint, + expires_at_ms: current_time_ms() + 60_000, + }); + (state, principal, dir) + } + + fn reader_request(body: T, principal: &Principal) -> Request { + let mut request = Request::new(body); + request.extensions_mut().insert(principal.clone()); + request + } + + #[tokio::test] + async fn machine_selection_does_not_grant_workspace_or_application_permissions() { + let (state, principal, _dir) = reader_fixture(false).await; + assert!( + get_snapshot(&state, reader_request(request(), &principal)) + .await + .is_ok() + ); + assert!( + authorize_workspace( + &state.store, + &state.admin_role, + &principal, + "default", + MinWorkspaceRole::User + ) + .await + .is_err() + ); + for scope in [Selection::All, Selection::Named(vec!["not-granted".into()])] { + assert_eq!( + authorize_selection(&state, &principal, &scope) + .await + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); + } + assert!(selection(request().workspace_scope.as_ref(), &["default".into()]).is_err()); + assert!(selection(None, &["default".into(), "default".into()]).is_err()); + let selected = GetServiceRouteSnapshotRequest { + workspaces: vec!["default".into(), "team-b".into()], + ..Default::default() + }; + assert!( + get_snapshot(&state, reader_request(selected, &principal)) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn machine_watch_rechecks_grants_and_certificate_expiry() { + let (state, principal, dir) = reader_fixture(true).await; + let body = WatchServiceRoutesRequest { + workspace_scope: Some(openshell_core::proto::all_workspaces_selector()), + ..Default::default() + }; + let mut stream = watch(&state, reader_request(body, &principal)) + .await + .unwrap() + .into_inner(); + assert!(stream.next().await.unwrap().is_ok()); + std::fs::write(dir.path().join("grants.json"), r#"{"readers":[]}"#).unwrap(); + let error = tokio::time::timeout(Duration::from_secs(3), stream.next()) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + assert!(stream.next().await.is_none()); + let expired = Principal::Discovery(crate::auth::principal::DiscoveryPrincipal { + certificate_sha256: "a".repeat(64), + expires_at_ms: 0, + }); + assert_eq!( + get_snapshot(&state, reader_request(request(), &expired)) + .await + .unwrap_err() + .code(), + tonic::Code::Unauthenticated + ); + } + + #[tokio::test] + async fn all_workspace_views_follow_creation_and_deletion_with_bounded_output() { + use openshell_core::proto::{ObjectMeta, Workspace}; + let (state, principal, _dir) = reader_fixture(true).await; + assert_eq!( + selection_workspaces(&state, &Selection::All).await.unwrap(), + vec!["default"] + ); + let mut workspace = Workspace { + metadata: Some(ObjectMeta { + id: "team-b".into(), + name: "team-b".into(), + ..Default::default() + }), + ..Default::default() + }; + state.store.put_message(&workspace).await.unwrap(); + assert_eq!( + selection_workspaces(&state, &Selection::All).await.unwrap(), + vec!["default", "team-b"] + ); + workspace.metadata.as_mut().unwrap().deletion_time = + Some(openshell_core::time::timestamp_from_millis(current_time_ms()).unwrap()); + state.store.put_message(&workspace).await.unwrap(); + assert_eq!( + selection_workspaces(&state, &Selection::All).await.unwrap(), + vec!["default"] + ); + for n in 0..crate::auth::discovery::MAX_WORKSPACES { + let name = format!("w-{n}"); + state + .store + .put_message(&Workspace { + metadata: Some(ObjectMeta { + id: name.clone(), + name, + ..Default::default() + }), + ..Default::default() + }) + .await + .unwrap(); + } + assert_eq!( + selected_snapshot(&state, &principal, &Selection::All) + .await + .unwrap_err() + .code(), + tonic::Code::ResourceExhausted + ); + assert!( + selected_snapshot( + &state, + &principal, + &Selection::Named(vec!["default".into()]) + ) + .await + .is_ok() + ); + } + #[tokio::test] + async fn finite_and_platform_snapshots_combine_actual_routes_and_withdraw_deleted_workspaces() { + use openshell_core::proto::{ObjectMeta, ServiceEndpoint, Workspace}; + let (state, principal, _dir) = reader_fixture(true).await; + state + .store + .put_message(&Workspace { + metadata: Some(ObjectMeta { + id: "workspace-b".into(), + name: "team-b".into(), + ..Default::default() + }), + ..Default::default() + }) + .await + .unwrap(); + let mut sandbox = state + .store + .get_message::("sandbox") + .await + .unwrap() + .unwrap(); + let meta = sandbox.metadata.as_mut().unwrap(); + meta.id = "sandbox-b".into(); + meta.workspace = "team-b".into(); + state.store.put_message(&sandbox).await.unwrap(); + let mut endpoint = state + .store + .get_message::("endpoint") + .await + .unwrap() + .unwrap(); + endpoint.sandbox_id = "sandbox-b".into(); + let meta = endpoint.metadata.as_mut().unwrap(); + meta.id = "endpoint-b".into(); + meta.workspace = "team-b".into(); + state.store.put_message(&endpoint).await.unwrap(); + let owner = crate::supervisor_owner::SupervisorOwnerIndex::new( + state.store.clone(), + crate::supervisor_owner::OWNER_TTL, + ) + .publish( + "sandbox-b", + "session-b", + "instance-b", + 1, + "replica", + "local://replica", + ) + .await + .unwrap(); + let config = super::super::policy::build_sandbox_config_snapshot(&state, &sandbox) + .await + .unwrap(); + accept( + &state, + &owner, + &openshell_core::proto::SupervisorServiceAdvertisement { + service_domain: "services.example.com".into(), + lease_sequence: 1, + configuration_instance_id: config.configuration_instance_id, + config_revision: config.config_revision, + routes: config.service_routes, + }, + current_time_ms() + 10_000, + ) + .await + .unwrap(); + let selected = Selection::Named(vec!["default".into(), "team-b".into()]); + let both = selected_snapshot(&state, &principal, &selected) + .await + .unwrap(); + assert_eq!( + both.routes + .iter() + .map(|route| route.workspace.as_str()) + .collect::>(), + vec!["default", "team-b"] + ); + assert_eq!( + both, + selected_snapshot(&state, &principal, &Selection::All) + .await + .unwrap() + ); + state + .store + .delete("workspace", "workspace-b") + .await + .unwrap(); + assert_eq!( + selected_snapshot(&state, &principal, &Selection::All) + .await + .unwrap() + .routes + .len(), + 1 + ); + } } diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index 56156f7d3f..ff07e8f295 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -68,9 +68,11 @@ fn membership_filter_subject<'a>( } } Principal::Sandbox(_) | Principal::Operator(_) => Ok(None), - Principal::Peer(_) | Principal::Supervisor(_) => Err(Status::permission_denied( - "gateway peer and supervisor principals cannot list workspaces", - )), + Principal::Discovery(_) | Principal::Peer(_) | Principal::Supervisor(_) => { + Err(Status::permission_denied( + "gateway peer and supervisor principals cannot list workspaces", + )) + } Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 7e9a5c5fee..832aaabac1 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1314,11 +1314,14 @@ fn spawn_gateway_connection( return; } }; + let certificate_expiry = + multiplex::extract_peer_certificate_expiry(&tls_stream); if let Err(e) = service - .serve_with_verified_certificate( + .serve_with_certificate_expiry( tls_stream, peer_identity, certificate_sha256, + certificate_expiry, ConnectionSource::Peer(addr), ) .await @@ -2999,6 +3002,123 @@ mod tests { } } + #[tokio::test] + async fn discovery_certificate_authenticates_only_route_rpcs_over_verified_tls() { + use rustls::pki_types::CertificateDer; + let dir = tempdir().unwrap(); + let (ca, ca_key) = generate_test_certs_with_ca(dir.path()); + let mut params = rcgen::CertificateParams::new(Vec::::new()).unwrap(); + params + .distinguished_name + .push(rcgen::DnType::CommonName, "discovery-proxy"); + params + .distinguished_name + .push(rcgen::DnType::OrganizationalUnitName, "discovery"); + params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth]; + let client_key = openshell_crypto::pki::generate_keypair().unwrap(); + let client = openshell_crypto::pki::signed_by(params, &client_key, &ca, &ca_key).unwrap(); + let client_cert = CertificateDer::from(client.der().to_vec()); + let key_pem = client_key.serialize_pem().unwrap(); + let client_pem_key = key_pem.clone(); + let fingerprint = hex::encode(openshell_crypto::sha256(client_cert.as_ref()).unwrap()); + std::fs::write(dir.path().join("grants.json"),serde_json::to_vec(&serde_json::json!({"readers":[{"certificate_sha256":fingerprint,"all_workspaces":true}]})).unwrap()).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut state = test_state(addr, false).await; + let inner = Arc::get_mut(&mut state).unwrap(); + inner.config.auth.allow_unauthenticated_users = false; + inner.config.mtls_auth.enabled = false; + inner.config.auth.discovery_grants_file = Some(dir.path().join("grants.json")); + inner.config.supervisor_app_listener.enabled = false; + inner.config.tls = Some(openshell_core::TlsConfig { + cert_path: dir.path().join("server-cert.pem"), + key_path: dir.path().join("server-key.pem"), + client_ca_path: Some(dir.path().join("ca.pem")), + require_client_auth: false, + external_cert_path: None, + external_key_path: None, + external_server_names: vec![], + }); + let acceptor = TlsAcceptor::from_files( + &dir.path().join("server-cert.pem"), + &dir.path().join("server-key.pem"), + Some(&dir.path().join("ca.pem")), + false, + None, + None, + vec![], + ) + .unwrap(); + let (shutdown, rx) = watch::channel(false); + let handle = tokio::spawn(serve_gateway_listener( + BoundGatewayListener { + listener, + address: addr, + }, + MultiplexService::new(state), + Some(acceptor), + false, + rx, + )); + + let tls = tonic::transport::ClientTlsConfig::new() + .domain_name("localhost") + .ca_certificate(tonic::transport::Certificate::from_pem(ca.pem())) + .identity(tonic::transport::Identity::from_pem( + client.pem(), + client_pem_key, + )); + let channel = + tonic::transport::Endpoint::from_shared(format!("https://localhost:{}", addr.port())) + .unwrap() + .tls_config(tls) + .unwrap() + .connect() + .await + .unwrap(); + let mut client = OpenShellClient::new(channel); + let request = || openshell_core::proto::GetServiceRouteSnapshotRequest { + workspace_scope: Some(openshell_core::proto::all_workspaces_selector()), + ..Default::default() + }; + assert!(client.get_service_route_snapshot(request()).await.is_ok()); + assert_eq!( + client + .get_current_user(openshell_core::proto::GetCurrentUserRequest {}) + .await + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); + let mut watch = client + .watch_service_routes(openshell_core::proto::WatchServiceRoutesRequest { + workspace_scope: request().workspace_scope, + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(watch.message().await.unwrap().is_some()); + std::fs::write(dir.path().join("grants.json"), r#"{"readers":[]}"#).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(3), watch.message()) + .await + .unwrap() + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); + assert_eq!( + client + .get_service_route_snapshot(request()) + .await + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); + stop_listener(shutdown, handle).await; + } + #[tokio::test] async fn plaintext_service_http_does_not_expose_grpc_gateway() { let (addr, shutdown, handle, _tls_dir) = diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index f85e75ffcf..c0af4d75ac 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -273,7 +273,31 @@ impl MultiplexService { where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - let operator = if self.state.config.mtls_auth.operator_enabled { + self.serve_with_certificate_expiry(stream, peer_identity, certificate_sha256, None, source) + .await + } + + /// TLS listener entry point carrying verified certificate validity. + pub async fn serve_with_certificate_expiry( + &self, + stream: S, + peer_identity: Option, + certificate_sha256: Option, + certificate_expires_at_ms: Option, + source: ConnectionSource, + ) -> Result<(), Box> + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let discovery = peer_identity + .as_ref() + .filter(|identity| crate::auth::discovery::is_certificate(identity)) + .and(certificate_sha256.as_ref()) + .map(|fingerprint| crate::auth::principal::DiscoveryPrincipal { + certificate_sha256: fingerprint.clone(), + expires_at_ms: certificate_expires_at_ms.unwrap_or(0), + }); + let operator = if discovery.is_none() && self.state.config.mtls_auth.operator_enabled { peer_identity .as_ref() .filter(|identity| { @@ -326,6 +350,10 @@ impl MultiplexService { ) .with_peer_certificate(certificate_sha256.clone()) .with_operator(operator) + .with_discovery( + discovery, + self.state.config.auth.discovery_grants_file.clone(), + ) .with_connection_source(source) .with_logons(self.state.logons.clone()); let grpc_service = @@ -827,6 +855,13 @@ fn gateway_principal_fields(principal: &Principal) -> BTreeMap { fields.insert("subject".to_string(), supervisor.identity.subject.clone()); fields.insert("token_id".to_string(), supervisor.identity.token_id.clone()); } + Principal::Discovery(reader) => { + fields.insert("kind".to_string(), "discovery".to_string()); + fields.insert( + "certificate_sha256".to_string(), + reader.certificate_sha256.clone(), + ); + } Principal::Anonymous => { fields.insert("kind".to_string(), "anonymous".to_string()); } @@ -1123,6 +1158,8 @@ pub struct AuthGrpcRouter { mtls_auth_enabled: bool, allow_unauthenticated_users: bool, operator: Option, + discovery: Option, + discovery_grants_file: Option, connection_source: Option, logons: audit::Logons, } @@ -1154,11 +1191,23 @@ impl AuthGrpcRouter { mtls_auth_enabled, allow_unauthenticated_users, operator: None, + discovery: None, + discovery_grants_file: None, connection_source: None, logons: audit::Logons::default(), } } + fn with_discovery( + mut self, + discovery: Option, + path: Option, + ) -> Self { + self.discovery = discovery; + self.discovery_grants_file = path; + self + } + fn with_operator(mut self, operator: Option) -> Self { self.operator = operator; self @@ -1221,6 +1270,8 @@ where let mtls_auth_enabled = self.mtls_auth_enabled; let allow_unauthenticated_users = self.allow_unauthenticated_users; let operator = self.operator.clone(); + let discovery = self.discovery.clone(); + let discovery_grants_file = self.discovery_grants_file.clone(); let connection_source = self.connection_source; let logons = self.logons.clone(); let peer_certificate_sha256 = self.peer_certificate_sha256.clone(); @@ -1240,7 +1291,33 @@ where return inner.ready().await?.call(req).await; } - let principal = if let Some(operator) = operator { + let principal = if let Some(reader) = discovery { + if req.headers().contains_key(http::header::AUTHORIZATION) + || !crate::auth::discovery::is_method(&path) + { + audit::emit_authorization_denial_for( + &path, + &origin(req.headers()), + &Principal::Discovery(reader), + AuthzDenial::MethodNotPermitted, + ); + return Ok(status_response(tonic::Status::permission_denied( + "discovery certificates may call only discovery methods without bearer credentials", + ))); + } + if let Err(status) = + crate::auth::discovery::grant(discovery_grants_file.as_deref(), &reader).await + { + audit::emit_authorization_denial_for( + &path, + &origin(req.headers()), + &Principal::Discovery(reader), + AuthzDenial::MethodNotPermitted, + ); + return Ok(status_response(status)); + } + Principal::Discovery(reader) + } else if let Some(operator) = operator { if req.headers().contains_key(http::header::AUTHORIZATION) { audit::emit_authentication_failure( &path, @@ -1332,6 +1409,9 @@ where Some((denial, tonic::Status::permission_denied(message))) }; let denial = match principal { + Principal::Discovery(_) if !crate::auth::discovery::is_method(&path) => { + method_denied("discovery method required") + } Principal::Operator(_) if !crate::auth::method_authz::is_operator_callable(&path) => { @@ -1364,7 +1444,8 @@ where "anonymous callers may not call authenticated methods", ))); } - Principal::Operator(_) + Principal::Discovery(_) + | Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) | Principal::Supervisor(_) => None, @@ -1573,6 +1654,17 @@ where }) } +/// Validity metadata parsed from the TLS-verified leaf certificate. X.509 +/// decoding is dependency-owned parsing; verification remains in the TLS backend. +pub fn extract_peer_certificate_expiry( + stream: &tokio_rustls::server::TlsStream, +) -> Option { + use x509_parser::prelude::*; + let leaf = stream.get_ref().1.peer_certificates()?.first()?; + let (_, cert) = X509Certificate::from_der(leaf.as_ref()).ok()?; + cert.validity().not_after.timestamp().checked_mul(1000) +} + /// Fingerprint of the leaf certificate verified during the TLS handshake. pub fn extract_peer_certificate_sha256( stream: &tokio_rustls::server::TlsStream, diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 274799a610..33ed0d394e 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -164,7 +164,7 @@ mod tests { // SandboxPhase enum. Stored rows never carried those values, so prior // payloads decode unchanged. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "8176663e842c3febc30686fc78341eaeca8249e1161336117abbeeb25e19c331"; + "0308847a7f463a2a6c6ab44de685d491a9dd17deb0a9354b00f9a8db309b84fe"; const DURABLE_SCHEMA_SHA256: &str = "ffb20595e36032a9476d1c6956e2dcdc814f85286fc4735751e5ec6f499ea9ed"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = diff --git a/docs/how-it-works/gateways/authentication.mdx b/docs/how-it-works/gateways/authentication.mdx index 0801bc6e7b..42a0c48b4b 100644 --- a/docs/how-it-works/gateways/authentication.mdx +++ b/docs/how-it-works/gateways/authentication.mdx @@ -478,3 +478,12 @@ Installers can also seed read-only defaults under `/etc/openshell/` (or a non-em Only `active_gateway` and `metadata.json` fall back to the system layout. mTLS bundles, OIDC tokens, edge tokens, and `last_sandbox` remain per-user state. For OIDC gateways, `metadata.json` also stores the issuer, CLI client ID, optional audience, and requested scopes. Treat `oidc_token.json` as a credential. OpenShell writes it with owner-only file permissions. + +### Discovery-only certificates + +An administrator can grant an `OU=discovery` client certificate access to route +snapshots and watches for selected workspaces or the whole platform. These +certificates cannot act as users or operators, even when ordinary mTLS user +authentication or local development access is enabled. See +[discovery reader configuration](./configuration#discovery-reader-grants) for +fingerprint mappings, rotation, and revocation. diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 8287c91153..67d67fa0d3 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1819,3 +1819,49 @@ TLS gateways advertise HTTPS for authenticated gateway service access, including on loopback, so clients can present an mTLS identity. The separate plaintext loopback service listener cannot accept a client-certificate identity or browser sessions; use an explicit gateway bearer credential there. + +### Discovery reader grants + +Configure an administrator-managed JSON file to authorize noninteractive route +readers without workspace membership or operator access: + +```toml +[openshell.gateway.auth] +discovery_grants_file = "/run/discovery/grants.json" +``` + +This requires gateway TLS and a configured client CA. Issue each reader a +client-authentication certificate from that CA with `OU=discovery`. This OU +restricts the certificate to discovery; it does not grant access. Map the exact +lowercase SHA-256 fingerprint of its DER leaf certificate in the grant file: + +```json +{ + "readers": [ + {"certificate_sha256": "<64 lowercase hex digits>", "workspaces": ["team-a", "team-b"]}, + {"certificate_sha256": "", "all_workspaces": true} + ] +} +``` + +Each entry selects either a nonempty workspace set or all workspaces. Only +`GetServiceRouteSnapshot` and `WatchServiceRoutes` accept this identity. It grants +no user, application, exec, forwarding, provider, or administrative access. +Combining a discovery certificate with bearer credentials is rejected. + +Upgrade every gateway replica to a version supporting discovery-only certificates +before provisioning reader identities. Mount the same administrator-controlled +mapping on every replica. Keep +it writable only by the administrator and update it atomically. The gateway +reloads it on requests and watch refreshes. Removing a fingerprint revokes its +authority on existing connections. Grant-file reads have a two-second deadline; +watches recheck at one-second refreshes and after bounded collection. Allow up +to sixteen seconds for collection and a backpressured watch to terminate, plus +file propagation to each replica in the deployment's revocation bound. Missing or invalid files fail +closed; consumers must independently expire cached routes during errors. +Certificates are also checked against their handshake-derived expiry. + +For rotation, add the replacement fingerprint, project the new certificate/key +to the consumer, then remove the old fingerprint. The proxy reloads mounted +identity and trust files on reconnect. No certificate issuance or renewal is +performed by the gateway. diff --git a/proto/README.md b/proto/README.md index 32efbbbd20..383e6e413b 100644 --- a/proto/README.md +++ b/proto/README.md @@ -54,7 +54,9 @@ identifies the service being exposed. - Accept `WorkspaceSelector.all_workspaces` only on collection-list RPCs that explicitly document and authorize cross-workspace access. The supported public collections are sandboxes, sandbox templates, providers, and - services. + services. Service-route snapshot/watch discovery also accepts this selector + with explicit platform discovery authorization, or a request-specific finite + `workspaces` set mutually exclusive with `workspace_scope`. - Requests whose primary resource is a workspace use `name`, not `workspace_scope`. - Document every request that permits an omitted selector. Current exceptions diff --git a/proto/openshell.proto b/proto/openshell.proto index 3522916f1a..0eeab8708f 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -4223,10 +4223,12 @@ message SupervisorServiceAdvertisement { string service_domain = 5; } -// One explicit active workspace is required. Omitted and all-workspaces -// selectors are rejected; discovery-only machine grants are separate work. +// Select one workspace, or all active workspaces with an explicit platform grant. +// Alternatively provide a finite set in workspaces; never combine selectors. message GetServiceRouteSnapshotRequest { openshell.datamodel.v1.WorkspaceSelector workspace_scope = 1; + // Up to 64 distinct canonical workspace references; exclusive with workspace_scope. + repeated string workspaces = 2; } message GetServiceRouteSnapshotResponse { ServiceRouteSnapshot snapshot = 1; @@ -4237,6 +4239,8 @@ message GetServiceRouteSnapshotResponse { // routes during discovery errors; errors are not empty replacements. message WatchServiceRoutesRequest { openshell.datamodel.v1.WorkspaceSelector workspace_scope = 1; + // Up to 64 distinct canonical workspace references; exclusive with workspace_scope. + repeated string workspaces = 2; } message WatchServiceRoutesResponse { ServiceRouteSnapshot snapshot = 1; diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 360278ed6b..0bc1794e6e 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -19752,13 +19752,15 @@ func (x *SupervisorServiceAdvertisement) GetServiceDomain() string { return "" } -// One explicit active workspace is required. Omitted and all-workspaces -// selectors are rejected; discovery-only machine grants are separate work. +// Select one workspace, or all active workspaces with an explicit platform grant. +// Alternatively provide a finite set in workspaces; never combine selectors. type GetServiceRouteSnapshotRequest struct { state protoimpl.MessageState `protogen:"open.v1"` WorkspaceScope *datamodelv1.WorkspaceSelector `protobuf:"bytes,1,opt,name=workspace_scope,json=workspaceScope,proto3" json:"workspace_scope,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Up to 64 distinct canonical workspace references; exclusive with workspace_scope. + Workspaces []string `protobuf:"bytes,2,rep,name=workspaces,proto3" json:"workspaces,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetServiceRouteSnapshotRequest) Reset() { @@ -19798,6 +19800,13 @@ func (x *GetServiceRouteSnapshotRequest) GetWorkspaceScope() *datamodelv1.Worksp return nil } +func (x *GetServiceRouteSnapshotRequest) GetWorkspaces() []string { + if x != nil { + return x.Workspaces + } + return nil +} + type GetServiceRouteSnapshotResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Snapshot *ServiceRouteSnapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` @@ -19849,8 +19858,10 @@ func (x *GetServiceRouteSnapshotResponse) GetSnapshot() *ServiceRouteSnapshot { type WatchServiceRoutesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` WorkspaceScope *datamodelv1.WorkspaceSelector `protobuf:"bytes,1,opt,name=workspace_scope,json=workspaceScope,proto3" json:"workspace_scope,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Up to 64 distinct canonical workspace references; exclusive with workspace_scope. + Workspaces []string `protobuf:"bytes,2,rep,name=workspaces,proto3" json:"workspaces,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WatchServiceRoutesRequest) Reset() { @@ -19890,6 +19901,13 @@ func (x *WatchServiceRoutesRequest) GetWorkspaceScope() *datamodelv1.WorkspaceSe return nil } +func (x *WatchServiceRoutesRequest) GetWorkspaces() []string { + if x != nil { + return x.Workspaces + } + return nil +} + type WatchServiceRoutesResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Snapshot *ServiceRouteSnapshot `protobuf:"bytes,1,opt,name=snapshot,proto3" json:"snapshot,omitempty"` @@ -21631,13 +21649,19 @@ const file_openshell_proto_rawDesc = "" + "\x19configuration_instance_id\x18\x02 \x01(\tR\x17configurationInstanceId\x12'\n" + "\x0fconfig_revision\x18\x03 \x01(\x04R\x0econfigRevision\x12A\n" + "\x06routes\x18\x04 \x03(\v2).openshell.sandbox.v1.SandboxServiceRouteR\x06routes\x12%\n" + - "\x0eservice_domain\x18\x05 \x01(\tR\rserviceDomain\"t\n" + + "\x0eservice_domain\x18\x05 \x01(\tR\rserviceDomain\"\x94\x01\n" + "\x1eGetServiceRouteSnapshotRequest\x12R\n" + - "\x0fworkspace_scope\x18\x01 \x01(\v2).openshell.datamodel.v1.WorkspaceSelectorR\x0eworkspaceScope\"a\n" + + "\x0fworkspace_scope\x18\x01 \x01(\v2).openshell.datamodel.v1.WorkspaceSelectorR\x0eworkspaceScope\x12\x1e\n" + + "\n" + + "workspaces\x18\x02 \x03(\tR\n" + + "workspaces\"a\n" + "\x1fGetServiceRouteSnapshotResponse\x12>\n" + - "\bsnapshot\x18\x01 \x01(\v2\".openshell.v1.ServiceRouteSnapshotR\bsnapshot\"o\n" + + "\bsnapshot\x18\x01 \x01(\v2\".openshell.v1.ServiceRouteSnapshotR\bsnapshot\"\x8f\x01\n" + "\x19WatchServiceRoutesRequest\x12R\n" + - "\x0fworkspace_scope\x18\x01 \x01(\v2).openshell.datamodel.v1.WorkspaceSelectorR\x0eworkspaceScope\"\\\n" + + "\x0fworkspace_scope\x18\x01 \x01(\v2).openshell.datamodel.v1.WorkspaceSelectorR\x0eworkspaceScope\x12\x1e\n" + + "\n" + + "workspaces\x18\x02 \x03(\tR\n" + + "workspaces\"\\\n" + "\x1aWatchServiceRoutesResponse\x12>\n" + "\bsnapshot\x18\x01 \x01(\v2\".openshell.v1.ServiceRouteSnapshotR\bsnapshot\"p\n" + "\x14ServiceRouteSnapshot\x12<\n" + diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 7f298e5a60..341d561850 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1657,3 +1657,13 @@ application Authorization header. Gateway headers and reserved cookies are stripped; application cookies and explicit bearer passthrough remain available. Only explicit local-development unauthenticated access bypasses gateway auth. Direct listener authentication belongs to the ingress provider. + +For discovery-only readers, verify the TLS client CA and an `OU=discovery` +certificate whose lowercase DER SHA-256 fingerprint appears in the +administrator-managed `openshell.gateway.auth.discovery_grants_file` JSON. +Missing mappings, duplicate fingerprints, invalid scope combinations, and expired +certificates fail closed. Verify the projected grant file on every replica; +propagation precedes the gateway's one-second refresh and bounded collection +rechecks. Do not replace a discovery identity with an operator certificate. +Single, finite-set, and all-workspaces discovery require explicit selectors; +workspace users retain membership checks and cannot request platform scope.