diff --git a/Cargo.lock b/Cargo.lock index 044be7e9f4..7f0c1753f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4863,8 +4863,10 @@ dependencies = [ "futures", "http 1.4.0", "http-body-util", + "hyper-rustls", "hyper-util", "metrics", + "openshell-crypto", "openshell-ocsf", "openshell-otel-test-support", "opentelemetry", @@ -4873,6 +4875,8 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", + "rustls", + "rustls-native-certs", "serde_json", "tempfile", "thiserror 2.0.20", @@ -4882,6 +4886,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "zeroize", ] [[package]] @@ -4898,8 +4903,10 @@ dependencies = [ "opentelemetry-proto", "prost", "rcgen", + "rustls", "tempfile", "tokio", + "tokio-rustls", "tokio-stream", "tonic", "tracing", diff --git a/crates/openshell-core/src/jwt.rs b/crates/openshell-core/src/jwt.rs index b4e8b4b75c..84ffe98ec0 100644 --- a/crates/openshell-core/src/jwt.rs +++ b/crates/openshell-core/src/jwt.rs @@ -388,6 +388,102 @@ mod session { } } + /// A PEM private key. Redacted from diagnostics and cleared on drop. + #[derive(Clone)] + pub struct SecretPrivateKeyPem(Zeroizing); + + impl SecretPrivateKeyPem { + #[must_use] + pub fn new(value: String) -> Self { + Self(Zeroizing::new(value)) + } + + #[must_use] + pub fn expose_secret(&self) -> &str { + &self.0 + } + } + + impl fmt::Debug for SecretPrivateKeyPem { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("SecretPrivateKeyPem([REDACTED])") + } + } + + impl Serialize for SecretPrivateKeyPem { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.expose_secret()) + } + } + + impl<'de> Deserialize<'de> for SecretPrivateKeyPem { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + String::deserialize(deserializer).map(Self::new) + } + } + + /// TLS material for one OTLP collector connection, as PEM. The private + /// key is redacted from diagnostics and cleared on drop. + #[derive(Clone, Default, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub struct OtlpTlsMaterial { + /// Certificates trusted to verify the collector instead of the + /// platform roots. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ca_pem: Option, + /// Client certificate presented to the collector. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client: Option, + } + + /// The headers and TLS material one supervisor export sends its + /// collector. + #[derive(Clone, Copy, Debug, Default)] + pub struct OtlpExportCredentials<'a> { + pub headers: Option<&'a SecretOtlpHeaders>, + pub tls: Option<&'a OtlpTlsMaterial>, + } + + /// A PEM client certificate chain and its private key. + #[derive(Clone, Debug, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub struct OtlpClientIdentity { + pub cert_pem: String, + pub key_pem: SecretPrivateKeyPem, + } + + #[cfg(feature = "otel")] + impl OtlpTlsMaterial { + /// The material as exporter TLS settings. + pub fn otlp_tls(&self) -> Result { + openshell_otel::OtlpTls::from_pem( + self.ca_pem.as_deref().map(str::as_bytes), + self.client.as_ref().map(|client| { + ( + client.cert_pem.as_bytes(), + client.key_pem.expose_secret().as_bytes(), + ) + }), + ) + } + } + + impl fmt::Debug for OtlpTlsMaterial { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OtlpTlsMaterial") + .field("ca_pem", &self.ca_pem.is_some()) + .field("client", &self.client) + .finish() + } + } + /// Trusted launch input delivered only to `openshell-supervisor`. #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -418,6 +514,18 @@ mod session { /// the gateway configures them. Never delivered to the workload. #[serde(default, skip_serializing_if = "Option::is_none")] pub agent_otlp_headers: Option, + /// TLS material for the supervisor's collectors, each distinct + /// material once, present only when the gateway configures it. + /// Never delivered to the workload. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub otlp_tls: Vec, + /// Position in `otlp_tls` of each collector's material. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub traces_otlp_tls: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ocsf_otlp_tls: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_otlp_tls: Option, } /// Gateway-created authentication input trusted by a compute driver. @@ -479,9 +587,48 @@ mod session { SandboxGenerationId::parse(self.runtime_generation.to_string()) .map_err(|_| SessionJwtError::InvalidRuntimeIdentity)?; SessionRotation::new(self.session_rotation.get())?; + let references = [ + self.traces_otlp_tls, + self.ocsf_otlp_tls, + self.agent_otlp_tls, + ]; + if references + .into_iter() + .flatten() + .any(|index| index >= self.otlp_tls.len()) + { + return Err(SessionJwtError::InvalidOtlpTlsReference); + } Ok(()) } + /// What the supervisor's trace export sends its collector. + pub fn traces_otlp(&self) -> OtlpExportCredentials<'_> { + self.otlp_credentials(self.traces_otlp_headers.as_ref(), self.traces_otlp_tls) + } + + /// What the supervisor's OCSF export sends its collector. + pub fn ocsf_otlp(&self) -> OtlpExportCredentials<'_> { + self.otlp_credentials(self.ocsf_otlp_headers.as_ref(), self.ocsf_otlp_tls) + } + + /// What the agent trace relay sends its collector. + pub fn agent_otlp(&self) -> OtlpExportCredentials<'_> { + self.otlp_credentials(self.agent_otlp_headers.as_ref(), self.agent_otlp_tls) + } + + /// [`Self::validate`] rejects a `tls` index outside `otlp_tls`. + fn otlp_credentials<'a>( + &'a self, + headers: Option<&'a SecretOtlpHeaders>, + tls: Option, + ) -> OtlpExportCredentials<'a> { + OtlpExportCredentials { + headers, + tls: tls.and_then(|index| self.otlp_tls.get(index)), + } + } + pub fn sandbox_bearer_slot(&self) -> Result { SessionBearerTokenSlot::new( self.sandbox_token.clone(), @@ -507,6 +654,10 @@ mod session { .field("traces_otlp_headers", &self.traces_otlp_headers) .field("ocsf_otlp_headers", &self.ocsf_otlp_headers) .field("agent_otlp_headers", &self.agent_otlp_headers) + .field("otlp_tls", &self.otlp_tls) + .field("traces_otlp_tls", &self.traces_otlp_tls) + .field("ocsf_otlp_tls", &self.ocsf_otlp_tls) + .field("agent_otlp_tls", &self.agent_otlp_tls) .finish() } } @@ -1174,6 +1325,8 @@ mod session { MissingRuntimeIdentity, #[error("sandbox runtime identity is invalid")] InvalidRuntimeIdentity, + #[error("OTLP TLS reference is outside the bundle's TLS material")] + InvalidOtlpTlsReference, #[error("key ID is invalid")] InvalidKeyId, #[error("verification key IDs must be unique")] @@ -1638,6 +1791,10 @@ mod tests { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }; bundle.validate().expect("non-expiring auth bundle"); } @@ -1760,6 +1917,50 @@ mod tests { assert!(!debug.contains(pair.sandbox.token.expose_secret())); } + #[test] + fn each_supervisor_export_reads_its_own_credentials() { + let (issuer, _gateway, _sandbox, identity) = fixture(); + let pair = issuer.mint_pair(&identity).expect("token pair"); + let headers = |lane: &str| Some(SecretOtlpHeaders::new(format!("lane={lane}"))); + let tls = |lane: &str| OtlpTlsMaterial { + ca_pem: Some(format!("{lane}-ca")), + client: None, + }; + let bundle = SupervisorAuthBundle { + session_id: SandboxSessionId::new(), + session_rotation: SessionRotation::new(1).expect("session rotation"), + runtime_generation: identity.runtime_generation.clone(), + auth_epoch: identity.auth_epoch, + gateway_token: pair.gateway.token, + gateway_expires_at: pair.gateway.expires_at, + sandbox_token: pair.sandbox.token, + sandbox_expires_at: pair.sandbox.expires_at, + ssh_host_private_key: None, + traces_otlp_headers: headers("traces"), + ocsf_otlp_headers: headers("ocsf"), + agent_otlp_headers: headers("agent"), + otlp_tls: vec![tls("agent"), tls("ocsf"), tls("traces")], + traces_otlp_tls: Some(2), + ocsf_otlp_tls: Some(1), + agent_otlp_tls: Some(0), + }; + + for (lane, credentials) in [ + ("traces", bundle.traces_otlp()), + ("ocsf", bundle.ocsf_otlp()), + ("agent", bundle.agent_otlp()), + ] { + assert_eq!( + credentials.headers.map(SecretOtlpHeaders::expose_secret), + Some(format!("lane={lane}").as_str()) + ); + assert_eq!( + credentials.tls.and_then(|tls| tls.ca_pem.as_deref()), + Some(format!("{lane}-ca").as_str()) + ); + } + } + #[test] fn supervisor_auth_bundle_round_trips_without_exposing_secrets_in_debug() { let (issuer, _gateway, _sandbox, identity) = fixture(); @@ -1785,6 +1986,16 @@ mod tests { agent_otlp_headers: Some(SecretOtlpHeaders::new( "authorization=agent-token".to_string(), )), + otlp_tls: vec![OtlpTlsMaterial { + ca_pem: Some("agent-ca".to_string()), + client: Some(OtlpClientIdentity { + cert_pem: "agent-cert".to_string(), + key_pem: SecretPrivateKeyPem::new("agent-private-key".to_string()), + }), + }], + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: Some(0), }; let encoded = serde_json::to_vec(&bundle).expect("serialize auth bundle"); @@ -1807,34 +2018,61 @@ mod tests { assert!(!debug.contains("collector-token")); assert!(!debug.contains("agent-token")); assert!(!debug.contains("traces-token")); - assert_eq!(debug.matches("[REDACTED]").count(), 6); + assert!(!debug.contains("agent-private-key")); + assert_eq!(debug.matches("[REDACTED]").count(), 7); assert_eq!( - decoded.ssh_host_private_key.unwrap().expose_secret(), + decoded + .ssh_host_private_key + .as_ref() + .unwrap() + .expose_secret(), "private-ssh-host-key" ); assert_eq!( - decoded.traces_otlp_headers.unwrap().expose_secret(), + decoded + .traces_otlp_headers + .as_ref() + .unwrap() + .expose_secret(), "authorization=traces-token" ); assert_eq!( - decoded.ocsf_otlp_headers.unwrap().expose_secret(), + decoded.ocsf_otlp_headers.as_ref().unwrap().expose_secret(), "authorization=collector-token" ); assert_eq!( - decoded.agent_otlp_headers.unwrap().expose_secret(), + decoded.agent_otlp_headers.as_ref().unwrap().expose_secret(), "authorization=agent-token" ); + let agent_tls = decoded.agent_otlp().tls.expect("agent collector TLS"); + assert_eq!(agent_tls.ca_pem.as_deref(), Some("agent-ca")); + let client = agent_tls.client.as_ref().unwrap(); + assert_eq!(client.cert_pem, "agent-cert"); + assert_eq!(client.key_pem.expose_secret(), "agent-private-key"); let mut legacy = serde_json::to_value(&bundle).unwrap(); let fields = legacy.as_object_mut().unwrap(); fields.remove("ssh_host_private_key"); fields.remove("traces_otlp_headers"); fields.remove("ocsf_otlp_headers"); fields.remove("agent_otlp_headers"); + fields.remove("otlp_tls"); + fields.remove("agent_otlp_tls"); let legacy: SupervisorAuthBundle = serde_json::from_value(legacy).unwrap(); assert!(legacy.ssh_host_private_key.is_none()); assert!(legacy.traces_otlp_headers.is_none()); assert!(legacy.ocsf_otlp_headers.is_none()); assert!(legacy.agent_otlp_headers.is_none()); + assert!(legacy.otlp_tls.is_empty() && legacy.agent_otlp_tls.is_none()); + + decoded + .validate() + .expect("references within the TLS material"); + let mut dangling = decoded.clone(); + dangling.traces_otlp_tls = Some(1); + assert!(matches!( + dangling.validate(), + Err(SessionJwtError::InvalidOtlpTlsReference) + )); } #[derive(Serialize)] diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index a287d3a36e..3ef98428c9 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -180,6 +180,9 @@ impl SupervisorOtlp { /// supervisors do. pub const OTLP_ENDPOINT_FLAG: &str = "otlp-endpoint"; pub const OTLP_PROTOCOL_FLAG: &str = "otlp-protocol"; +pub const OTLP_CA_FILE_FLAG: &str = "otlp-ca-file"; +pub const OTLP_CERT_FILE_FLAG: &str = "otlp-cert-file"; +pub const OTLP_KEY_FILE_FLAG: &str = "otlp-key-file"; pub const OTLP_OCSF_ENDPOINT_FLAG: &str = "otlp-ocsf-endpoint"; pub const OTLP_OCSF_PROTOCOL_FLAG: &str = "otlp-ocsf-protocol"; pub const OTLP_AGENT_ENDPOINT_FLAG: &str = "otlp-agent-endpoint"; @@ -197,6 +200,18 @@ pub struct SupervisorOtlpArgs { #[arg(long = OTLP_PROTOCOL_FLAG, env = OTLP_PROTOCOL, default_value_t)] pub protocol: OtlpProtocol, + /// PEM certificates trusted to verify an `https://` trace collector. + #[arg(long = OTLP_CA_FILE_FLAG, env = "OPENSHELL_OTLP_CA_FILE")] + pub ca_file: Option, + + /// PEM client certificate presented to the trace collector. + #[arg(long = OTLP_CERT_FILE_FLAG, env = "OPENSHELL_OTLP_CERT_FILE", requires = "key_file")] + pub cert_file: Option, + + /// PEM private key for `--otlp-cert-file`. + #[arg(long = OTLP_KEY_FILE_FLAG, env = "OPENSHELL_OTLP_KEY_FILE", requires = "cert_file")] + pub key_file: Option, + /// OTLP collector endpoint for supervisor OCSF event export. #[arg(long = OTLP_OCSF_ENDPOINT_FLAG, env = OTLP_OCSF_ENDPOINT)] pub ocsf_endpoint: Option, @@ -631,6 +646,49 @@ mod tests { ); } + #[cfg(feature = "clap")] + #[test] + fn driver_flags_read_the_trace_collectors_tls_files() { + use clap::Parser as _; + + #[derive(clap::Parser)] + struct Driver { + #[command(flatten)] + otlp: SupervisorOtlpArgs, + } + + let driver = Driver::try_parse_from([ + "driver", + "--otlp-endpoint", + "https://collector:4317", + "--otlp-ca-file", + "/etc/otlp/ca.pem", + "--otlp-cert-file", + "/etc/otlp/client.pem", + "--otlp-key-file", + "/etc/otlp/client-key.pem", + ]) + .expect("TLS files should parse"); + assert_eq!( + [ + driver.otlp.ca_file.as_deref(), + driver.otlp.cert_file.as_deref(), + driver.otlp.key_file.as_deref() + ], + [ + Some(std::path::Path::new("/etc/otlp/ca.pem")), + Some(std::path::Path::new("/etc/otlp/client.pem")), + Some(std::path::Path::new("/etc/otlp/client-key.pem")) + ] + ); + for flag in ["--otlp-cert-file", "--otlp-key-file"] { + assert!( + Driver::try_parse_from(["driver", flag, "/etc/otlp/file.pem"]).is_err(), + "{flag} needs its pair" + ); + } + } + #[test] fn otlp_protocol_uses_the_otel_exporter_protocol_values() { for (value, protocol) in [ diff --git a/crates/openshell-crypto/README.md b/crates/openshell-crypto/README.md index 7c040c1c31..8d411cf80d 100644 --- a/crates/openshell-crypto/README.md +++ b/crates/openshell-crypto/README.md @@ -242,6 +242,7 @@ not a whole-application crypto coverage claim: | Remaining use | Current scope and follow-up | | --- | --- | | Reqwest clients (OIDC/JWKS, token exchange, provider refresh, Vault, token grants, telemetry) | These use Rustls's process-default provider after ensuring one exists; they do not necessarily honor an explicitly selected `CryptoContext`. The OpenSSL follow-up must supply compatible preconfigured TLS or another supported transport integration and test actual provider use. | +| OTLP gRPC export TLS (`openshell-otel`, the supervisor's agent trace relay) | Tonic's `ClientTlsConfig` verifies collectors and signs for client certificates with Rustls's process-default provider, as other first-party tonic clients do. OTLP/HTTP export builds its configuration through `openshell_crypto::tls::try_client_builder`. The OpenSSL follow-up must move the gRPC path to a preconfigured connector before claiming selected-backend coverage. | | Kubernetes client TLS (`kube-client` 0.99) | The AWS-LC-enabled client attempts to install AWS-LC as the Rustls process default. It does not consult `CryptoContext`; the OpenSSL follow-up must adapt this transport before claiming selected-backend coverage. | | Outer-fence evidence digest (`openshell-isolation-interface::contract`) | The digest binds generation and native enforcement evidence using `sha2` through `Sha256Digest::compute`. This security-relevant hash remains outside facade dispatch and must be migrated with failure propagation before claiming exclusive backend ownership. | | VM image/archive digest verification and cached supervisor validation (`openshell-driver-vm`) | Security-relevant artifact integrity remains outside this runtime credential migration. Moving it requires covering the streaming readers, embedded-artifact cache, and build-time digest generation together; it must be addressed before claiming exclusive backend ownership. | diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 23c48446cb..63caf0d094 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -10,7 +10,9 @@ falls back to `/var/run/docker.sock` when no candidate responds. When the gateway configures `[openshell.gateway.otlp]`, the in-process driver exports spans to the same OTLP collector, over its configured protocol, as `openshell-driver-docker`. The standalone driver accepts -`OPENSHELL_OTLP_ENDPOINT` and `OPENSHELL_OTLP_PROTOCOL`, continues W3C trace +`OPENSHELL_OTLP_ENDPOINT` and `OPENSHELL_OTLP_PROTOCOL`, with +`OPENSHELL_OTLP_CA_FILE`, `OPENSHELL_OTLP_CERT_FILE`, and +`OPENSHELL_OTLP_KEY_FILE` for an `https://` collector, continues W3C trace context from gateway RPC metadata, and flushes spans during graceful shutdown. When `[openshell.gateway.otlp.agent]` is enabled, the driver also sets diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index c34961236c..817b840d21 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -60,6 +60,9 @@ async fn main() -> Result<()> { openshell_otel::DriverTracingConfig { endpoint: args.otlp.endpoint.as_deref(), protocol: args.otlp.protocol.into(), + ca_file: args.otlp.ca_file.as_deref(), + cert_file: args.otlp.cert_file.as_deref(), + key_file: args.otlp.key_file.as_deref(), gateway_name: args.gateway_name.as_deref(), service_version: VERSION, log_level: &args.log_level, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 174339acc0..92d20739f3 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -67,6 +67,10 @@ fn test_launch_authentication() -> Vec { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 027f0a5361..49dc465362 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -96,7 +96,9 @@ context. When `[openshell.gateway.otlp.agent]` is enabled, the driver also sets `OPENSHELL_OTLP_AGENT_ENDPOINT`, which starts the supervisor's agent trace relay. Each endpoint is accompanied by its `_PROTOCOL` variable (`OPENSHELL_OTLP_PROTOCOL`, `OPENSHELL_OTLP_AGENT_PROTOCOL`) when the collector -uses `http/protobuf`. None of these is configurable in driver TOML. +uses `http/protobuf`. Collector headers and TLS material reach the supervisor +in its launch credentials, not the Pod environment. None of these is +configurable in driver TOML. The driver creates one namespace-wide `NetworkPolicy` before it releases any workload Pod. It selects every OpenShell workload, denies all workload egress, diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 438b099cf7..8f016a95ce 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -228,6 +228,9 @@ async fn main() -> Result<()> { openshell_otel::DriverTracingConfig { endpoint: args.otlp.endpoint.as_deref(), protocol: args.otlp.protocol.into(), + ca_file: args.otlp.ca_file.as_deref(), + cert_file: args.otlp.cert_file.as_deref(), + key_file: args.otlp.key_file.as_deref(), gateway_name: args.gateway_name.as_deref(), service_version: VERSION, log_level: &args.log_level, diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index cc9c80747e..c70cc6eca1 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -2666,6 +2666,10 @@ mod tests { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 7349b9083f..f9fc790588 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -474,6 +474,10 @@ mod tests { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }, gateway_id: "gateway-test".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 557c7f0b1f..cc0ebe1a20 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -211,6 +211,9 @@ async fn main() -> Result<()> { openshell_otel::DriverTracingConfig { endpoint: args.otlp.endpoint.as_deref(), protocol: args.otlp.protocol.into(), + ca_file: args.otlp.ca_file.as_deref(), + cert_file: args.otlp.cert_file.as_deref(), + key_file: args.otlp.key_file.as_deref(), gateway_name: args.gateway_name.as_deref(), service_version: VERSION, log_level: &args.log_level, diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index d40e8e472d..b17d94141b 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -11245,6 +11245,10 @@ mod tests { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }, gateway_id: "gateway-a".to_string(), verification_keys: vec![SessionVerificationKey { diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 455c420f23..90b4c1cbc4 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -298,6 +298,9 @@ async fn main() -> Result<()> { openshell_otel::DriverTracingConfig { endpoint: args.otlp.endpoint.as_deref(), protocol: args.otlp.protocol.into(), + ca_file: args.otlp.ca_file.as_deref(), + cert_file: args.otlp.cert_file.as_deref(), + key_file: args.otlp.key_file.as_deref(), gateway_name: args.gateway_name.as_deref(), service_version: VERSION, log_level: &args.log_level, diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index b861bbe840..d1330a299e 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -729,6 +729,18 @@ fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gat sandbox_env::OTLP_AGENT_ENDPOINT_FLAG, sandbox_env::OTLP_AGENT_PROTOCOL_FLAG, ); + if let Some(traces) = otlp_config.and_then(|otlp| otlp.traces.as_ref()) { + let collector = &traces.collector; + for (flag, path) in [ + (sandbox_env::OTLP_CA_FILE_FLAG, collector.ca_file()), + (sandbox_env::OTLP_CERT_FILE_FLAG, collector.cert_file()), + (sandbox_env::OTLP_KEY_FILE_FLAG, collector.key_file()), + ] { + if let Some(path) = path { + command.arg(format!("--{flag}")).arg(path); + } + } + } // The driver's own trace exporter reads this like any OTLP SDK; keeping // it out of the arguments keeps the headers out of process listings. if let Some(headers) = openshell_server::gateway_trace_headers(otlp_config) { @@ -838,8 +850,7 @@ mod tests { use openshell_core::sandbox_env::OtlpProtocol; use openshell_core::{NetworkSupervisorTrustBundle, UpstreamProxyConfig}; use openshell_server::config_file::{ - AgentOtlpConfig, OcsfOtlpConfig, OtlpCollectorConfig, OtlpConfig, OtlpHeaders, - OtlpTracesConfig, + AgentOtlpConfig, OcsfOtlpConfig, OtlpCollectorConfig, OtlpConfig, OtlpTracesConfig, }; use std::os::unix::fs::PermissionsExt; use std::os::unix::net::UnixListener as StdUnixListener; @@ -912,6 +923,51 @@ mod tests { }); } + #[test] + fn vm_driver_receives_the_trace_collectors_tls_file_paths() { + let dir = tempdir().unwrap(); + let certified = rcgen::generate_simple_self_signed(vec!["collector".into()]).unwrap(); + let [ca, cert, key] = ["ca.pem", "client.pem", "client-key.pem"].map(|name| { + let path = dir.path().join(name); + let pem = if name == "client-key.pem" { + certified.signing_key.serialize_pem() + } else { + certified.cert.pem() + }; + std::fs::write(&path, pem).unwrap(); + path + }); + let collector = OtlpCollectorConfig::new("https://collector:4317", OtlpProtocol::Grpc) + .with_tls_files(Some(ca.clone()), Some(cert.clone()), Some(key.clone())) + .unwrap(); + assert!(collector.tls().client_identity().is_some()); + let otlp = OtlpConfig { + service_name: None, + traces: Some(OtlpTracesConfig { collector }), + ocsf: None, + agent: None, + }; + let mut command = tokio::process::Command::new("openshell-driver-vm"); + append_otlp_args(&mut command, Some(&otlp), "production-us-west"); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + for (flag, path) in [ + ("--otlp-ca-file", ca), + ("--otlp-cert-file", cert), + ("--otlp-key-file", key), + ] { + let pair = [flag.to_string(), path.display().to_string()]; + assert!( + args.windows(2).any(|window| window == pair), + "{pair:?} in {args:?}" + ); + } + } + #[test] fn vm_driver_command_includes_gateway_otlp_configuration() { let mut command = tokio::process::Command::new("openshell-driver-vm"); @@ -920,12 +976,10 @@ mod tests { Some(&OtlpConfig { service_name: Some("custom-gateway".to_string()), traces: Some(OtlpTracesConfig { - collector: OtlpCollectorConfig { - endpoint: "http://collector.internal:4317".to_string(), - protocol: OtlpProtocol::Grpc, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new( + "http://collector.internal:4317", + OtlpProtocol::Grpc, + ), }), ocsf: None, agent: None, @@ -954,12 +1008,10 @@ mod tests { Some(&OtlpConfig { service_name: None, traces: Some(OtlpTracesConfig { - collector: OtlpCollectorConfig { - endpoint: "http://collector.internal:4318".to_string(), - protocol: OtlpProtocol::HttpProtobuf, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new( + "http://collector.internal:4318", + OtlpProtocol::HttpProtobuf, + ), }), ocsf: None, agent: None, @@ -991,12 +1043,10 @@ mod tests { service_name: None, traces: None, ocsf: Some(OcsfOtlpConfig { - collector: OtlpCollectorConfig { - endpoint: "http://siem.internal:4318".to_string(), - protocol: OtlpProtocol::HttpProtobuf, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new( + "http://siem.internal:4318", + OtlpProtocol::HttpProtobuf, + ), schema_version: Some("2.0".into()), }), agent: None, @@ -1032,12 +1082,10 @@ mod tests { traces: None, ocsf: None, agent: Some(AgentOtlpConfig { - collector: OtlpCollectorConfig { - endpoint: "http://agents.internal:4318".to_string(), - protocol: OtlpProtocol::HttpProtobuf, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new( + "http://agents.internal:4318", + OtlpProtocol::HttpProtobuf, + ), }), }), "production-us-west", diff --git a/crates/openshell-otel-test-support/Cargo.toml b/crates/openshell-otel-test-support/Cargo.toml index 05b67b233d..d14fe33db8 100644 --- a/crates/openshell-otel-test-support/Cargo.toml +++ b/crates/openshell-otel-test-support/Cargo.toml @@ -16,18 +16,20 @@ http = { workspace = true } http-body-util = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true, features = ["tokio"] } +openshell-crypto = { workspace = true } openshell-otel = { path = "../openshell-otel" } opentelemetry-proto = { version = "0.33", default-features = false, features = ["gen-tonic", "logs", "trace"] } prost = { workspace = true } +rcgen = { workspace = true } +rustls = { workspace = true } tokio = { workspace = true } +tokio-rustls = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true, features = ["gzip", "tls-aws-lc"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } [dev-dependencies] -openshell-crypto = { workspace = true } -rcgen = { workspace = true } tempfile = "3" [lints] diff --git a/crates/openshell-otel-test-support/src/http.rs b/crates/openshell-otel-test-support/src/http.rs index fe2db25909..5e8f3e733e 100644 --- a/crates/openshell-otel-test-support/src/http.rs +++ b/crates/openshell-otel-test-support/src/http.rs @@ -33,10 +33,25 @@ impl OtlpHttpTestServer { /// Starts the collector, answering every request with `status`. pub async fn start_with_status(status: http::StatusCode) -> Self { + Self::start_with(status, None).await + } + + /// Serve over TLS at `https://localhost:` with `tls`, answering 200. + pub async fn start_with_tls(tls: &crate::ServerTls) -> Self { + Self::start_with(http::StatusCode::OK, Some(tls)).await + } + + async fn start_with(status: http::StatusCode, tls: Option<&crate::ServerTls>) -> Self { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("OTLP/HTTP test collector should bind a loopback listener"); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let port = listener.local_addr().unwrap().port(); + let endpoint = if tls.is_some() { + format!("https://localhost:{port}") + } else { + format!("http://127.0.0.1:{port}") + }; + let acceptor = tls.map(|tls| tokio_rustls::TlsAcceptor::from(tls.rustls_config())); let (tx, exports) = tokio::sync::mpsc::unbounded_channel(); let task = tokio::spawn(async move { loop { @@ -44,6 +59,7 @@ impl OtlpHttpTestServer { continue; }; let tx = tx.clone(); + let acceptor = acceptor.clone(); tokio::spawn(async move { let service = hyper::service::service_fn( move |request: http::Request| { @@ -64,9 +80,22 @@ impl OtlpHttpTestServer { } }, ); - let _ = hyper::server::conn::http1::Builder::new() - .serve_connection(hyper_util::rt::TokioIo::new(stream), service) - .await; + let http = hyper::server::conn::http1::Builder::new(); + match acceptor { + Some(acceptor) => { + // A refused handshake is the outcome some tests expect. + if let Ok(stream) = acceptor.accept(stream).await { + let _ = http + .serve_connection(hyper_util::rt::TokioIo::new(stream), service) + .await; + } + } + None => { + let _ = http + .serve_connection(hyper_util::rt::TokioIo::new(stream), service) + .await; + } + } }); } }); diff --git a/crates/openshell-otel-test-support/src/lib.rs b/crates/openshell-otel-test-support/src/lib.rs index f97fcd20e5..13f4e49928 100644 --- a/crates/openshell-otel-test-support/src/lib.rs +++ b/crates/openshell-otel-test-support/src/lib.rs @@ -4,8 +4,10 @@ //! Shared OTLP collector fixture for `OpenShell` tracing tests. mod http; +mod tls; pub use http::{HttpExport, OtlpHttpTestServer}; +pub use tls::{PemIdentity, ServerTls, TestPki}; /// Helpers for tests that exercise the supervisor's agent trace relay from /// the agent side: the attribution keys the relay sets and a builder for the @@ -108,8 +110,7 @@ pub async fn assert_compute_driver_tracing( let collector = OtlpTestServer::start().await; let (provider, error) = descriptor.provider_for( Some(collector.endpoint()), - openshell_otel::Protocol::Grpc, - &::http::HeaderMap::new(), + &openshell_otel::CollectorTransport::default(), service_version, Some("test-gateway"), Some(descriptor.compute_driver()), @@ -297,7 +298,18 @@ impl OtlpTestServer { /// Serve over TLS at `https://localhost:` using `identity`. pub async fn start_tls(identity: tonic::transport::Identity) -> Self { - Self::try_start_with(LOOPBACK_EPHEMERAL, Some(identity)) + Self::try_start_with( + LOOPBACK_EPHEMERAL, + Some(tonic::transport::ServerTlsConfig::new().identity(identity)), + ) + .await + .expect("OTLP test collector should bind a loopback listener") + } + + /// Serve over TLS at `https://localhost:` with `tls`, which can + /// require client certificates. + pub async fn start_with_tls(tls: &ServerTls) -> Self { + Self::try_start_with(LOOPBACK_EPHEMERAL, Some(tls.tonic_config())) .await .expect("OTLP test collector should bind a loopback listener") } @@ -318,13 +330,13 @@ impl OtlpTestServer { async fn try_start_with( addr: SocketAddr, - identity: Option, + tls: Option, ) -> std::io::Result { let received = Arc::new(Mutex::new(ReceivedTraces::default())); let exported = Arc::new(tokio::sync::Notify::new()); let listener = tokio::net::TcpListener::bind(addr).await?; let local_addr = listener.local_addr()?; - let endpoint = if identity.is_some() { + let endpoint = if tls.is_some() { format!("https://localhost:{}", local_addr.port()) } else { format!("http://{local_addr}") @@ -334,9 +346,9 @@ impl OtlpTestServer { exported: Arc::clone(&exported), }; let mut server = tonic::transport::Server::builder(); - if let Some(identity) = identity { + if let Some(tls) = tls { server = server - .tls_config(tonic::transport::ServerTlsConfig::new().identity(identity)) + .tls_config(tls) .expect("OTLP test collector should accept its TLS identity"); } let trace_service = TraceServiceServer::new(collector.clone()) diff --git a/crates/openshell-otel-test-support/src/tls.rs b/crates/openshell-otel-test-support/src/tls.rs new file mode 100644 index 0000000000..7fb7d272b0 --- /dev/null +++ b/crates/openshell-otel-test-support/src/tls.rs @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Throwaway PKI for collector TLS tests. + +use openshell_crypto::pki::{self, Certificate, KeyPair}; +use rcgen::{BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa}; + +/// A PEM certificate and its private key. +#[derive(Clone, Debug)] +pub struct PemIdentity { + pub cert_pem: String, + pub key_pem: String, +} + +/// A CA with a `localhost` server certificate and a client certificate, all +/// freshly generated. +#[derive(Clone, Debug)] +pub struct TestPki { + pub ca_pem: String, + pub server: PemIdentity, + pub client: PemIdentity, +} + +impl TestPki { + #[must_use] + pub fn generate() -> Self { + let ca_key = pki::generate_keypair().unwrap(); + let mut ca_params = CertificateParams::new(Vec::::new()).unwrap(); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca = pki::self_signed(ca_params, &ca_key).unwrap(); + Self { + ca_pem: ca.pem(), + server: issue( + &ca, + &ca_key, + vec!["localhost".to_string()], + ExtendedKeyUsagePurpose::ServerAuth, + ), + client: issue( + &ca, + &ca_key, + Vec::new(), + ExtendedKeyUsagePurpose::ClientAuth, + ), + } + } +} + +fn issue( + ca: &Certificate, + ca_key: &KeyPair, + names: Vec, + usage: ExtendedKeyUsagePurpose, +) -> PemIdentity { + let key = pki::generate_keypair().unwrap(); + let mut params = CertificateParams::new(names).unwrap(); + params.extended_key_usages = vec![usage]; + let cert = pki::signed_by(params, &key, ca, ca_key).unwrap(); + PemIdentity { + cert_pem: cert.pem(), + key_pem: key.serialize_pem().unwrap(), + } +} + +/// Server TLS settings for a test collector: its identity and, when set, the +/// CA that client certificates must chain to. +#[derive(Clone, Debug)] +pub struct ServerTls { + pub identity: PemIdentity, + pub client_ca_pem: Option, +} + +impl ServerTls { + /// Serve `pki`'s server certificate and require a client certificate + /// issued by its CA. + #[must_use] + pub fn mutual(pki: &TestPki) -> Self { + Self { + identity: pki.server.clone(), + client_ca_pem: Some(pki.ca_pem.clone()), + } + } + + /// Serve `pki`'s server certificate without asking for a client one. + #[must_use] + pub fn server_only(pki: &TestPki) -> Self { + Self { + identity: pki.server.clone(), + client_ca_pem: None, + } + } + + pub(crate) fn rustls_config(&self) -> std::sync::Arc { + use rustls::pki_types::pem::PemObject as _; + use rustls::pki_types::{CertificateDer, PrivateKeyDer}; + + let certs = CertificateDer::pem_slice_iter(self.identity.cert_pem.as_bytes()) + .collect::, _>>() + .unwrap(); + let key = PrivateKeyDer::from_pem_slice(self.identity.key_pem.as_bytes()).unwrap(); + let builder = openshell_crypto::tls::server_builder(); + let builder = match &self.client_ca_pem { + Some(ca_pem) => { + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_slice_iter(ca_pem.as_bytes()) { + roots.add(cert.unwrap()).unwrap(); + } + builder.with_client_cert_verifier( + rustls::server::WebPkiClientVerifier::builder_with_provider( + std::sync::Arc::new(roots), + openshell_crypto::tls::configuration_provider(), + ) + .build() + .unwrap(), + ) + } + None => builder.with_no_client_auth(), + }; + let mut config = builder.with_single_cert(certs, key).unwrap(); + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + std::sync::Arc::new(config) + } + + pub(crate) fn tonic_config(&self) -> tonic::transport::ServerTlsConfig { + let config = tonic::transport::ServerTlsConfig::new().identity( + tonic::transport::Identity::from_pem(&self.identity.cert_pem, &self.identity.key_pem), + ); + match &self.client_ca_pem { + Some(ca_pem) => config.client_ca_root(tonic::transport::Certificate::from_pem(ca_pem)), + None => config, + } + } +} diff --git a/crates/openshell-otel/Cargo.toml b/crates/openshell-otel/Cargo.toml index e8e1333d01..c6cd0111c9 100644 --- a/crates/openshell-otel/Cargo.toml +++ b/crates/openshell-otel/Cargo.toml @@ -16,13 +16,17 @@ bytes = { workspace = true } futures = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } +hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "logging", "tls12"] } hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } metrics = { workspace = true } +openshell-crypto = { workspace = true } openshell-ocsf = { path = "../openshell-ocsf" } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["logs"] } opentelemetry-http = { workspace = true } opentelemetry-otlp = { workspace = true, features = ["gzip-http", "http-proto", "logs"] } +rustls = { workspace = true } +rustls-native-certs = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } @@ -31,6 +35,7 @@ tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true } tonic = { workspace = true } tower-http = { workspace = true } +zeroize = { workspace = true } [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } diff --git a/crates/openshell-otel/src/driver.rs b/crates/openshell-otel/src/driver.rs index e99c819152..6c473831f4 100644 --- a/crates/openshell-otel/src/driver.rs +++ b/crates/openshell-otel/src/driver.rs @@ -73,8 +73,7 @@ impl ComputeDriverTracing { pub fn provider_for( self, endpoint: Option<&str>, - protocol: crate::Protocol, - headers: &http::HeaderMap, + transport: &crate::CollectorTransport, service_version: &'static str, gateway_name: Option<&str>, compute_driver: Option<&str>, @@ -89,8 +88,7 @@ impl ComputeDriverTracing { compute_driver, ), })? - .with_protocol(protocol) - .with_headers(headers.clone()) + .with_transport(transport) .tracer_provider() }); match provider { @@ -203,6 +201,10 @@ pub struct DriverTracingHandle { pub struct DriverTracingConfig<'a> { pub endpoint: Option<&'a str>, pub protocol: crate::Protocol, + /// PEM files for an `https://` collector; see [`crate::OtlpTls::from_files`]. + pub ca_file: Option<&'a std::path::Path>, + pub cert_file: Option<&'a std::path::Path>, + pub key_file: Option<&'a std::path::Path>, pub gateway_name: Option<&'a str>, pub service_version: &'static str, pub log_level: &'a str, @@ -218,20 +220,33 @@ impl Drop for DriverTracingHandle { } } +impl DriverTracingConfig<'_> { + /// The collector transport, with headers from the environment only. + pub fn transport(&self) -> Result { + Ok(crate::CollectorTransport { + protocol: self.protocol, + headers: http::HeaderMap::new(), + tls: crate::OtlpTls::from_files(self.ca_file, self.cert_file, self.key_file)?, + }) + } +} + /// Install standalone compute-driver logging and OTLP tracing. #[must_use] pub fn install_driver_tracing( descriptor: ComputeDriverTracing, config: DriverTracingConfig<'_>, ) -> DriverTracingHandle { - let (provider, setup_error) = descriptor.provider_for( - config.endpoint, - config.protocol, - &http::HeaderMap::new(), - config.service_version, - config.gateway_name, - Some(descriptor.compute_driver()), - ); + let (provider, setup_error) = match config.transport() { + Ok(transport) => descriptor.provider_for( + config.endpoint, + &transport, + config.service_version, + config.gateway_name, + Some(descriptor.compute_driver()), + ), + Err(error) => (None, config.endpoint.map(|_| error.into())), + }; tracing_subscriber::registry() .with( EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(config.log_level)), @@ -274,8 +289,11 @@ mod tests { let mut collector = openshell_otel_test_support::OtlpHttpTestServer::start().await; let (provider, error) = TRACING.provider_for( Some(collector.endpoint()), - crate::Protocol::HttpProtobuf, - &crate::parse_otlp_headers("authorization=driver"), + &crate::CollectorTransport { + protocol: crate::Protocol::HttpProtobuf, + headers: crate::parse_otlp_headers("authorization=driver"), + tls: crate::OtlpTls::default(), + }, "1.2.3", Some("test-gateway"), Some("docker"), @@ -293,6 +311,31 @@ mod tests { .await; } + #[test] + fn driver_transport_reads_its_tls_files() { + let pki = openshell_otel_test_support::TestPki::generate(); + let dir = tempfile::tempdir().unwrap(); + let ca = dir.path().join("ca.pem"); + std::fs::write(&ca, &pki.ca_pem).unwrap(); + let config = |ca_file, key_file| DriverTracingConfig { + endpoint: Some("https://collector:4317"), + protocol: crate::Protocol::Grpc, + ca_file, + cert_file: None, + key_file, + gateway_name: None, + service_version: "1.2.3", + log_level: "info", + }; + + let transport = config(Some(&ca), None).transport().unwrap(); + assert_eq!(transport.tls.ca_pem(), Some(pki.ca_pem.as_bytes())); + assert!(matches!( + config(None, Some(&ca)).transport(), + Err(crate::TlsFileError::KeyWithoutCert) + )); + } + #[tokio::test] async fn disabled_stream_tracing_returns_the_original_box() { let stream: BoxGrpcStream<()> = Box::pin(futures::stream::empty()); diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index 244f5e7511..78b995ee0d 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -9,6 +9,7 @@ mod headers; mod logs; mod ocsf; mod propagation; +mod tls; mod transport; #[cfg(test)] @@ -32,6 +33,8 @@ pub use propagation::{ TraceContextInterceptor, current_trace_context_carrier, current_trace_context_environment, set_parent_from_environment, set_parent_from_trace_context, trace_context_environment, }; +pub use tls::{MAX_TLS_FILE_BYTES, OtlpTls, TlsError, TlsFileError}; + pub use transport::{ CONNECT_TIMEOUT, OtlpHttpClient, TCP_KEEPALIVE_IDLE, TCP_KEEPALIVE_INTERVAL, TCP_KEEPALIVE_RETRIES, signal_url, @@ -41,7 +44,6 @@ use opentelemetry::KeyValue; use opentelemetry::trace::TracerProvider as _; pub use opentelemetry_otlp::LogExporter; use opentelemetry_otlp::tonic_types::metadata::MetadataMap; -use opentelemetry_otlp::tonic_types::transport::ClientTlsConfig; use opentelemetry_otlp::{SpanExporter, WithExportConfig, WithHttpConfig, WithTonicConfig}; use opentelemetry_sdk::Resource; use opentelemetry_sdk::trace::SdkTracer; @@ -152,6 +154,25 @@ pub struct OtlpExportConfig<'a> { pub resource_attributes: Vec, } +/// How exports reach a collector: the transport, request headers, and TLS. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct CollectorTransport { + pub protocol: Protocol, + pub headers: http::HeaderMap, + pub tls: OtlpTls, +} + +impl std::fmt::Debug for CollectorTransport { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CollectorTransport") + .field("protocol", &self.protocol) + .field("header_names", &self.headers.keys().collect::>()) + .field("tls", &self.tls) + .finish() + } +} + /// Failure to set up OTLP export. #[derive(Debug, thiserror::Error)] pub enum SetupError { @@ -173,8 +194,14 @@ pub enum SetupError { source: http::Error, }, - #[error("OTLP/HTTP export over https is not supported")] - HttpsUnsupported, + #[error("OTLP TLS settings need an https:// endpoint, not {endpoint:?}")] + TlsWithoutHttps { endpoint: String }, + + #[error("invalid OTLP TLS settings: {0}")] + Tls(#[from] TlsError), + + #[error("invalid OTLP TLS files: {0}")] + TlsFiles(#[from] TlsFileError), #[error("OTLP/HTTP export needs a tokio runtime")] NoRuntime, @@ -246,6 +273,7 @@ pub struct OtlpTarget { tls: bool, protocol: Protocol, headers: http::HeaderMap, + tls_settings: OtlpTls, resource: Resource, } @@ -257,6 +285,7 @@ impl std::fmt::Debug for OtlpTarget { .field("tls", &self.tls) .field("protocol", &self.protocol) .field("header_names", &self.headers.keys().collect::>()) + .field("tls_settings", &self.tls_settings) .field("resource", &self.resource) .finish() } @@ -271,6 +300,7 @@ impl OtlpTarget { tls: uri.scheme() == Some(&http::uri::Scheme::HTTPS), protocol: Protocol::Grpc, headers: http::HeaderMap::new(), + tls_settings: OtlpTls::default(), resource: resource_for(config), }) } @@ -291,11 +321,23 @@ impl OtlpTarget { self } + /// Reach the collector the way `transport` describes. + #[must_use] + pub fn with_transport(self, transport: &CollectorTransport) -> Self { + self.with_protocol(transport.protocol) + .with_headers(transport.headers.clone()) + .with_tls(transport.tls.clone()) + } + + /// Verify an `https://` collector and authenticate to it with `tls`. + #[must_use] + pub fn with_tls(mut self, tls: OtlpTls) -> Self { + self.tls_settings = tls; + self + } + /// The URL of `signal_path` under the endpoint, for OTLP/HTTP export. pub(crate) fn http_signal_url(&self, signal_path: &str) -> Result { - if self.tls { - return Err(SetupError::HttpsUnsupported); - } signal_url(&self.endpoint, signal_path) .map(|url| url.to_string()) .map_err(|source| SetupError::InvalidSignalUrl { @@ -313,8 +355,7 @@ impl OtlpTarget { .with_endpoint(self.endpoint.clone()) .with_metadata(MetadataMap::from_headers(self.headers.clone())); if self.tls { - // Tonic trusts nothing by default, so enable the platform roots. - builder.with_tls_config(ClientTlsConfig::new().with_enabled_roots()) + builder.with_tls_config(self.tls_settings.grpc_config()) } else { builder } @@ -342,7 +383,10 @@ impl OtlpTarget { Ok(builder .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) .with_endpoint(self.http_signal_url(signal_path)?) - .with_http_client(OtlpHttpClient::new(transport::export_timeout(timeout_var))?) + .with_http_client(OtlpHttpClient::new( + self.tls.then_some(&self.tls_settings), + transport::export_timeout(timeout_var), + )?) .with_headers(headers)) } @@ -356,8 +400,20 @@ impl OtlpTarget { &self.resource } + /// Fails when TLS settings are given for a plaintext endpoint. + pub(crate) fn check_tls(&self) -> Result<(), SetupError> { + if self.tls || self.tls_settings.is_default() { + Ok(()) + } else { + Err(SetupError::TlsWithoutHttps { + endpoint: self.endpoint.clone(), + }) + } + } + /// Build a trace provider that exports spans to this target in batches. pub fn tracer_provider(&self) -> Result { + self.check_tls()?; let exporter = match self.protocol { Protocol::Grpc => self .configure(SpanExporter::builder().with_tonic()) @@ -671,6 +727,34 @@ mod tests { ); } + #[tokio::test] + async fn tls_settings_need_an_https_endpoint() { + let pki = openshell_otel_test_support::TestPki::generate(); + let tls = OtlpTls::from_pem(Some(pki.ca_pem.as_bytes()), None).unwrap(); + for protocol in [Protocol::Grpc, Protocol::HttpProtobuf] { + let target = |endpoint| { + OtlpTarget::new(&OtlpExportConfig { + endpoint, + service_name: ServiceName::Fixed("test-service"), + service_version: None, + resource_attributes: Vec::new(), + }) + .unwrap() + .with_protocol(protocol) + .with_tls(tls.clone()) + }; + assert!(matches!( + target("http://collector:4317").tracer_provider(), + Err(SetupError::TlsWithoutHttps { .. }) + )); + assert!(matches!( + target("http://collector:4317").log_exporter(), + Err(SetupError::TlsWithoutHttps { .. }) + )); + assert!(target("https://collector:4317").tracer_provider().is_ok()); + } + } + #[test] fn debug_output_names_headers_without_their_values() { let target = OtlpTarget::new(&OtlpExportConfig { diff --git a/crates/openshell-otel/src/logs.rs b/crates/openshell-otel/src/logs.rs index e99755531e..0457f952de 100644 --- a/crates/openshell-otel/src/logs.rs +++ b/crates/openshell-otel/src/logs.rs @@ -11,6 +11,7 @@ impl OtlpTarget { /// Build a log exporter for this target. Callers attach /// [`Self::resource`] to it. pub fn log_exporter(&self) -> Result { + self.check_tls()?; match self.protocol { Protocol::Grpc => Ok(self .configure(LogExporter::builder().with_tonic()) diff --git a/crates/openshell-otel/src/tls.rs b/crates/openshell-otel/src/tls.rs new file mode 100644 index 0000000000..aa09f05631 --- /dev/null +++ b/crates/openshell-otel/src/tls.rs @@ -0,0 +1,395 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! TLS settings for collector connections. + +use std::io::Read as _; +use std::path::Path; +use std::sync::Arc; + +use rustls::pki_types::pem::PemObject as _; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use tonic::transport::{Certificate, ClientTlsConfig, Identity}; +use zeroize::Zeroizing; + +/// Largest TLS file read. Launch credentials carry each distinct file, and +/// some compute drivers store them in size-limited objects. +pub const MAX_TLS_FILE_BYTES: usize = 64 * 1024; + +/// TLS material that cannot be used. +#[derive(Debug, thiserror::Error)] +pub enum TlsError { + #[error("the CA holds no PEM certificate")] + NoCaCertificate, + #[error("the client certificate holds no PEM certificate")] + NoClientCertificate, + #[error("the client key holds no PEM private key")] + NoClientKey, + #[error("no platform trust roots were found; set a CA file")] + NoPlatformRoots, + #[error("{0}")] + Rustls(#[from] rustls::Error), +} + +/// TLS files that cannot be used. +#[derive(Debug, thiserror::Error)] +pub enum TlsFileError { + #[error("{key} {}: {source}", path.display())] + Read { + key: &'static str, + path: std::path::PathBuf, + source: std::io::Error, + }, + #[error("{key} {} is larger than {} KiB", path.display(), MAX_TLS_FILE_BYTES / 1024)] + TooLarge { + key: &'static str, + path: std::path::PathBuf, + }, + #[error("cert_file is set without key_file")] + CertWithoutKey, + #[error("key_file is set without cert_file")] + KeyWithoutCert, + #[error("key_file {} does not match cert_file {}", key.display(), cert.display())] + KeyMismatch { + cert: std::path::PathBuf, + key: std::path::PathBuf, + }, + #[error(transparent)] + Tls(#[from] TlsError), +} + +/// How to verify a collector and authenticate to it over `https://`. The +/// default trusts the platform roots and presents no client certificate. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct OtlpTls { + ca_pem: Option>, + client: Option, +} + +#[derive(Clone, PartialEq, Eq)] +struct ClientIdentity { + cert_pem: Vec, + key_pem: Zeroizing>, +} + +impl OtlpTls { + /// Trust only the certificates in `ca_pem`, when set, and present the + /// `client` certificate chain and private key, when set. All are PEM. + pub fn from_pem( + ca_pem: Option<&[u8]>, + client: Option<(&[u8], &[u8])>, + ) -> Result { + let tls = Self { + ca_pem: ca_pem.map(<[u8]>::to_vec), + client: client.map(|(cert_pem, key_pem)| ClientIdentity { + cert_pem: cert_pem.to_vec(), + key_pem: Zeroizing::new(key_pem.to_vec()), + }), + }; + // Building a configuration loads the key through the crypto provider + // and checks that it matches the certificate. + let roots = tls + .custom_roots()? + .unwrap_or_else(|| Arc::new(rustls::RootCertStore::empty())); + tls.client_config(roots)?; + Ok(tls) + } + + /// Read PEM files: `ca_file` to trust instead of the platform roots, and + /// `cert_file` with `key_file` to present as a client certificate. + pub fn from_files( + ca_file: Option<&Path>, + cert_file: Option<&Path>, + key_file: Option<&Path>, + ) -> Result { + let read = |key, path: &Path| { + let error = |source| TlsFileError::Read { + key, + path: path.to_path_buf(), + source, + }; + let mut bytes = Vec::new(); + std::fs::File::open(path) + .and_then(|file| { + file.take(MAX_TLS_FILE_BYTES as u64 + 1) + .read_to_end(&mut bytes) + }) + .map_err(error)?; + if bytes.len() > MAX_TLS_FILE_BYTES { + return Err(TlsFileError::TooLarge { + key, + path: path.to_path_buf(), + }); + } + Ok(bytes) + }; + let ca = ca_file.map(|path| read("ca_file", path)).transpose()?; + let client = match (cert_file, key_file) { + (Some(cert), Some(key)) => Some(( + read("cert_file", cert)?, + Zeroizing::new(read("key_file", key)?), + )), + (None, None) => None, + (Some(_), None) => return Err(TlsFileError::CertWithoutKey), + (None, Some(_)) => return Err(TlsFileError::KeyWithoutCert), + }; + let client_pem = client + .as_ref() + .map(|(cert, key)| (cert.as_slice(), key.as_slice())); + match ( + Self::from_pem(ca.as_deref(), client_pem), + cert_file, + key_file, + ) { + (Err(TlsError::Rustls(rustls::Error::InconsistentKeys(_))), Some(cert), Some(key)) => { + Err(TlsFileError::KeyMismatch { + cert: cert.to_path_buf(), + key: key.to_path_buf(), + }) + } + (result, ..) => Ok(result?), + } + } + + /// Whether nothing differs from the default. + #[must_use] + pub fn is_default(&self) -> bool { + self.ca_pem.is_none() && self.client.is_none() + } + + #[must_use] + pub fn ca_pem(&self) -> Option<&[u8]> { + self.ca_pem.as_deref() + } + + /// The client certificate chain and private key, when set. + #[must_use] + pub fn client_identity(&self) -> Option<(&[u8], &[u8])> { + self.client + .as_ref() + .map(|client| (client.cert_pem.as_slice(), client.key_pem.as_slice())) + } + + /// The settings as a tonic channel TLS configuration. + #[must_use] + pub fn grpc_config(&self) -> ClientTlsConfig { + // Tonic trusts nothing by default, so enable the platform roots. + let config = self.ca_pem.as_ref().map_or_else( + || ClientTlsConfig::new().with_enabled_roots(), + |ca_pem| ClientTlsConfig::new().ca_certificate(Certificate::from_pem(ca_pem)), + ); + match &self.client { + Some(client) => config.identity(Identity::from_pem( + &client.cert_pem, + client.key_pem.as_slice(), + )), + None => config, + } + } + + /// A Rustls client configuration for an `https://` collector, or, with + /// no settings, for a plaintext one, which needs no trust roots. + pub(crate) fn http_config_for(tls: Option<&Self>) -> Result { + tls.map_or_else( + || Self::default().client_config(Arc::new(rustls::RootCertStore::empty())), + Self::http_config, + ) + } + + /// The settings as a Rustls client configuration. + fn http_config(&self) -> Result { + let roots = match self.custom_roots()? { + Some(roots) => roots, + None => platform_roots()?, + }; + self.client_config(roots) + } + + fn client_config( + &self, + roots: Arc, + ) -> Result { + let builder = openshell_crypto::tls::try_client_builder()?.with_root_certificates(roots); + Ok(match self.client_identity_der()? { + Some((certs, key)) => builder.with_client_auth_cert(certs, key)?, + None => builder.with_no_client_auth(), + }) + } + + /// The roots from the configured CA, when one is set. + fn custom_roots(&self) -> Result>, TlsError> { + let Some(ca_pem) = &self.ca_pem else { + return Ok(None); + }; + let mut roots = rustls::RootCertStore::empty(); + for cert in CertificateDer::pem_slice_iter(ca_pem) { + roots.add(cert.map_err(|_| TlsError::NoCaCertificate)?)?; + } + if roots.is_empty() { + return Err(TlsError::NoCaCertificate); + } + Ok(Some(Arc::new(roots))) + } + + #[allow(clippy::type_complexity)] + fn client_identity_der( + &self, + ) -> Result>, PrivateKeyDer<'static>)>, TlsError> { + let Some(client) = &self.client else { + return Ok(None); + }; + let certs = CertificateDer::pem_slice_iter(&client.cert_pem) + .collect::, _>>() + .map_err(|_| TlsError::NoClientCertificate)?; + if certs.is_empty() { + return Err(TlsError::NoClientCertificate); + } + let key = + PrivateKeyDer::from_pem_slice(&client.key_pem).map_err(|_| TlsError::NoClientKey)?; + Ok(Some((certs, key))) + } +} + +/// The platform's trust roots, or the ones `SSL_CERT_FILE` and `SSL_CERT_DIR` +/// name instead. +fn platform_roots() -> Result, TlsError> { + let mut roots = rustls::RootCertStore::empty(); + roots.add_parsable_certificates(rustls_native_certs::load_native_certs().certs); + if roots.is_empty() { + return Err(TlsError::NoPlatformRoots); + } + Ok(Arc::new(roots)) +} + +impl std::fmt::Debug for OtlpTls { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("OtlpTls") + .field("custom_ca", &self.ca_pem.is_some()) + .field("client_certificate", &self.client.is_some()) + .finish() + } +} + +#[cfg(test)] +mod tests { + use openshell_otel_test_support::TestPki; + + use super::*; + + #[test] + fn unusable_pem_is_rejected_when_the_settings_are_built() { + let pki = TestPki::generate(); + let (cert, key) = ( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + ); + assert!(OtlpTls::from_pem(Some(pki.ca_pem.as_bytes()), Some((cert, key))).is_ok()); + assert!(matches!( + OtlpTls::from_pem(Some(b"not pem"), None), + Err(TlsError::NoCaCertificate) + )); + assert!(matches!( + OtlpTls::from_pem(None, Some((key, key))), + Err(TlsError::NoClientCertificate) + )); + assert!(matches!( + OtlpTls::from_pem(None, Some((cert, cert))), + Err(TlsError::NoClientKey) + )); + } + + #[test] + fn files_are_read_in_pairs_and_report_which_one_failed() { + let pki = TestPki::generate(); + let dir = tempfile::tempdir().unwrap(); + let path = |name: &str| dir.path().join(name); + std::fs::write(path("ca.pem"), &pki.ca_pem).unwrap(); + std::fs::write(path("client.pem"), &pki.client.cert_pem).unwrap(); + std::fs::write(path("client-key.pem"), &pki.client.key_pem).unwrap(); + let (ca, cert, key) = (path("ca.pem"), path("client.pem"), path("client-key.pem")); + + let tls = OtlpTls::from_files(Some(&ca), Some(&cert), Some(&key)).unwrap(); + assert_eq!(tls.ca_pem(), Some(pki.ca_pem.as_bytes())); + assert!(tls.client_identity().is_some()); + assert!(OtlpTls::from_files(None, None, None).unwrap().is_default()); + assert!(matches!( + OtlpTls::from_files(None, Some(&cert), None), + Err(TlsFileError::CertWithoutKey) + )); + assert!(matches!( + OtlpTls::from_files(None, None, Some(&key)), + Err(TlsFileError::KeyWithoutCert) + )); + let missing = path("missing.pem"); + let error = OtlpTls::from_files(Some(&missing), None, None).unwrap_err(); + assert!( + error + .to_string() + .starts_with(&format!("ca_file {}", missing.display())), + "{error}" + ); + assert!(matches!( + OtlpTls::from_files(Some(&key), None, None), + Err(TlsFileError::Tls(TlsError::NoCaCertificate)) + )); + let large = path("large.pem"); + std::fs::write(&large, vec![b'a'; MAX_TLS_FILE_BYTES + 1]).unwrap(); + let error = OtlpTls::from_files(Some(&large), None, None).unwrap_err(); + assert!( + matches!(error, TlsFileError::TooLarge { key: "ca_file", .. }), + "{error}" + ); + } + + #[test] + fn a_key_that_does_not_match_the_certificate_is_rejected() { + let pki = TestPki::generate(); + let other = TestPki::generate(); + let error = OtlpTls::from_pem( + None, + Some(( + pki.client.cert_pem.as_bytes(), + other.client.key_pem.as_bytes(), + )), + ) + .unwrap_err(); + assert!( + matches!(error, TlsError::Rustls(rustls::Error::InconsistentKeys(_))), + "{error:?}" + ); + + let dir = tempfile::tempdir().unwrap(); + let (cert, key) = ( + dir.path().join("client.pem"), + dir.path().join("other-key.pem"), + ); + std::fs::write(&cert, &pki.client.cert_pem).unwrap(); + std::fs::write(&key, &other.client.key_pem).unwrap(); + let error = OtlpTls::from_files(None, Some(&cert), Some(&key)).unwrap_err(); + assert_eq!( + error.to_string(), + format!( + "key_file {} does not match cert_file {}", + key.display(), + cert.display() + ) + ); + } + + #[test] + fn debug_output_omits_the_private_key() { + let pki = TestPki::generate(); + let tls = OtlpTls::from_pem( + None, + Some(( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + )), + ) + .unwrap(); + let debug = format!("{tls:?}"); + assert!(debug.contains("client_certificate: true"), "{debug}"); + assert!(!debug.contains("PRIVATE KEY"), "{debug}"); + } +} diff --git a/crates/openshell-otel/src/transport.rs b/crates/openshell-otel/src/transport.rs index bb9fc3680f..b936dcd566 100644 --- a/crates/openshell-otel/src/transport.rs +++ b/crates/openshell-otel/src/transport.rs @@ -9,6 +9,7 @@ use std::time::Duration; use bytes::Bytes; use http::uri::PathAndQuery; use http_body_util::{BodyExt as _, Full, Limited}; +use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; use hyper_util::client::legacy::Client; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::rt::TokioExecutor; @@ -63,15 +64,16 @@ pub fn signal_url(endpoint: &str, signal_path: &str) -> Result>, + client: Client, Full>, runtime: tokio::runtime::Handle, timeout: Duration, } impl OtlpHttpClient { /// Creates a client bound to the current tokio runtime whose requests - /// fail after `timeout`. Fails outside a runtime. - pub fn new(timeout: Duration) -> Result { + /// fail after `timeout`. It connects to an `https://` collector with + /// `tls`; without it, only to `http://` ones. Fails outside a runtime. + pub fn new(tls: Option<&crate::OtlpTls>, timeout: Duration) -> Result { let runtime = tokio::runtime::Handle::try_current().map_err(|_| crate::SetupError::NoRuntime)?; let mut connector = HttpConnector::new(); @@ -80,6 +82,12 @@ impl OtlpHttpClient { connector.set_keepalive_interval(Some(TCP_KEEPALIVE_INTERVAL)); connector.set_keepalive_retries(Some(TCP_KEEPALIVE_RETRIES)); connector.set_nodelay(true); + connector.enforce_http(false); + let connector = HttpsConnectorBuilder::new() + .with_tls_config(crate::OtlpTls::http_config_for(tls)?) + .https_or_http() + .enable_http1() + .wrap_connector(connector); Ok(Self { client: Client::builder(TokioExecutor::new()).build(connector), runtime, @@ -133,7 +141,7 @@ impl HttpClient for OtlpHttpClient { #[cfg(test)] mod tests { - use openshell_otel_test_support::OtlpHttpTestServer; + use openshell_otel_test_support::{OtlpHttpTestServer, OtlpTestServer, ServerTls, TestPki}; use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _}; use opentelemetry::trace::{Tracer as _, TracerProvider as _}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; @@ -141,7 +149,7 @@ mod tests { use prost::Message as _; use super::*; - use crate::{OtlpExportConfig, OtlpTarget, Protocol, ServiceName, SetupError}; + use crate::{OtlpExportConfig, OtlpTarget, Protocol, ServiceName}; /// A loopback listener that accepts connections and never answers. async fn silent_collector() -> (String, tokio::task::JoinHandle<()>) { @@ -187,7 +195,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn requests_fail_after_the_timeout() { let (endpoint, collector) = silent_collector().await; - let client = OtlpHttpClient::new(Duration::from_millis(200)).unwrap(); + let client = OtlpHttpClient::new(None, Duration::from_millis(200)).unwrap(); let result = tokio::time::timeout(Duration::from_secs(5), client.send(post(&endpoint))) .await .expect("the client gives up on its own"); @@ -202,7 +210,7 @@ mod tests { b"partial".to_vec(), ) .await; - let client = OtlpHttpClient::new(Duration::from_secs(5)).unwrap(); + let client = OtlpHttpClient::new(None, Duration::from_secs(5)).unwrap(); assert!(client.send(post(&endpoint)).await.is_err()); collector.abort(); } @@ -212,7 +220,7 @@ mod tests { let body = vec![b'x'; MAX_RESPONSE_BYTES + 1]; let head = format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\n\r\n", body.len()); let (endpoint, collector) = answering_collector(head.leak(), body).await; - let client = OtlpHttpClient::new(Duration::from_secs(5)).unwrap(); + let client = OtlpHttpClient::new(None, Duration::from_secs(5)).unwrap(); assert!(client.send(post(&endpoint)).await.is_err()); collector.abort(); } @@ -220,7 +228,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn abandoned_requests_stop() { let (endpoint, collector) = silent_collector().await; - let client = OtlpHttpClient::new(Duration::from_mins(1)).unwrap(); + let client = OtlpHttpClient::new(None, Duration::from_mins(1)).unwrap(); let alive = || { tokio::runtime::Handle::current() .metrics() @@ -253,6 +261,110 @@ mod tests { .with_protocol(Protocol::HttpProtobuf) } + fn tls_target(endpoint: &str, protocol: Protocol, tls: crate::OtlpTls) -> OtlpTarget { + OtlpTarget::new(&OtlpExportConfig { + endpoint, + service_name: ServiceName::Fixed("openshell-test"), + service_version: None, + resource_attributes: Vec::new(), + }) + .unwrap() + .with_protocol(protocol) + .with_tls(tls) + } + + /// Exports one span through `target` and reports whether it was delivered. + async fn delivered(target: OtlpTarget) -> bool { + let provider = target.tracer_provider().expect("tracer provider"); + provider.tracer("test").in_span("tls-span", |_| {}); + tokio::task::spawn_blocking(move || provider.force_flush().is_ok()) + .await + .unwrap() + } + + #[tokio::test(flavor = "multi_thread")] + async fn exports_verify_the_collector_ca_and_present_a_client_certificate() { + let pki = TestPki::generate(); + let tls = crate::OtlpTls::from_pem( + Some(pki.ca_pem.as_bytes()), + Some(( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + )), + ) + .unwrap(); + + let grpc = OtlpTestServer::start_with_tls(&ServerTls::mutual(&pki)).await; + assert!(delivered(tls_target(grpc.endpoint(), Protocol::Grpc, tls.clone())).await); + assert_eq!(grpc.shutdown().await.spans.len(), 1); + + let mut http = OtlpHttpTestServer::start_with_tls(&ServerTls::mutual(&pki)).await; + assert!(delivered(tls_target(http.endpoint(), Protocol::HttpProtobuf, tls)).await); + assert_eq!(http.next_export().await.path, "/v1/traces"); + } + + #[tokio::test(flavor = "multi_thread")] + async fn exports_fail_without_the_collector_ca_or_a_required_client_certificate() { + let pki = TestPki::generate(); + let ca_only = crate::OtlpTls::from_pem(Some(pki.ca_pem.as_bytes()), None).unwrap(); + for protocol in [Protocol::Grpc, Protocol::HttpProtobuf] { + for (tls, server, case) in [ + ( + crate::OtlpTls::default(), + ServerTls::server_only(&pki), + "platform roots do not trust the collector", + ), + ( + ca_only.clone(), + ServerTls::mutual(&pki), + "the collector requires a client certificate", + ), + ] { + let delivered = match protocol { + Protocol::Grpc => { + let collector = OtlpTestServer::start_with_tls(&server).await; + delivered(tls_target(collector.endpoint(), protocol, tls)).await + } + Protocol::HttpProtobuf => { + let collector = OtlpHttpTestServer::start_with_tls(&server).await; + delivered(tls_target(collector.endpoint(), protocol, tls)).await + } + }; + assert!(!delivered, "{protocol:?}: {case}"); + } + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn exports_fail_when_the_certificate_does_not_name_the_host() { + let pki = TestPki::generate(); + let tls = crate::OtlpTls::from_pem(Some(pki.ca_pem.as_bytes()), None).unwrap(); + let by_address = |endpoint: &str| endpoint.replace("localhost", "127.0.0.1"); + + let grpc = OtlpTestServer::start_with_tls(&ServerTls::server_only(&pki)).await; + assert!(delivered(tls_target(grpc.endpoint(), Protocol::Grpc, tls.clone())).await); + assert!( + !delivered(tls_target( + &by_address(grpc.endpoint()), + Protocol::Grpc, + tls.clone() + )) + .await + ); + + let http = OtlpHttpTestServer::start_with_tls(&ServerTls::server_only(&pki)).await; + assert!( + delivered(tls_target( + http.endpoint(), + Protocol::HttpProtobuf, + tls.clone() + )) + .await + ); + let endpoint = by_address(http.endpoint()); + assert!(!delivered(tls_target(&endpoint, Protocol::HttpProtobuf, tls)).await); + } + #[test] fn export_timeouts_prefer_the_signal_variable() { const SIGNAL: &str = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT"; @@ -294,26 +406,6 @@ mod tests { assert!(signal_url("not a uri", "v1/traces").is_err()); } - #[tokio::test(flavor = "multi_thread")] - async fn https_endpoints_fail_setup() { - for endpoint in ["https://collector:4318", "HTTPS://collector:4318"] { - assert!( - matches!( - http_target(endpoint).tracer_provider(), - Err(SetupError::HttpsUnsupported) - ), - "{endpoint}" - ); - assert!( - matches!( - http_target(endpoint).log_exporter(), - Err(SetupError::HttpsUnsupported) - ), - "{endpoint}" - ); - } - } - #[tokio::test(flavor = "multi_thread")] async fn trace_provider_exports_over_http_from_the_batch_thread() { let mut collector = OtlpHttpTestServer::start().await; diff --git a/crates/openshell-otel/tests/ca_file_replaces_roots.rs b/crates/openshell-otel/tests/ca_file_replaces_roots.rs new file mode 100644 index 0000000000..1e7a906d9c --- /dev/null +++ b/crates/openshell-otel/tests/ca_file_replaces_roots.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The only test in its binary, since it sets the process environment. + +use openshell_otel::{OtlpExportConfig, OtlpTarget, OtlpTls, Protocol, ServiceName}; +use openshell_otel_test_support::{OtlpHttpTestServer, OtlpTestServer, ServerTls, TestPki}; +use opentelemetry::trace::{Tracer as _, TracerProvider as _}; + +async fn delivered(endpoint: &str, protocol: Protocol, tls: Option<&OtlpTls>) -> bool { + let mut target = OtlpTarget::new(&OtlpExportConfig { + endpoint, + service_name: ServiceName::Fixed("ca-test"), + service_version: None, + resource_attributes: Vec::new(), + }) + .unwrap() + .with_protocol(protocol); + if let Some(tls) = tls { + target = target.with_tls(tls.clone()); + } + let provider = target.tracer_provider().unwrap(); + provider.tracer("test").in_span("ca-span", |_| {}); + tokio::task::spawn_blocking(move || provider.force_flush().is_ok()) + .await + .unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +#[allow(unsafe_code)] +async fn a_ca_file_replaces_the_platform_roots() { + let platform = TestPki::generate(); + let configured = TestPki::generate(); + let roots = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(roots.path(), &platform.ca_pem).unwrap(); + // Nothing else in this binary reads the environment. + unsafe { std::env::set_var("SSL_CERT_FILE", roots.path()) }; + let ca_file = OtlpTls::from_pem(Some(configured.ca_pem.as_bytes()), None).unwrap(); + + for protocol in [Protocol::Grpc, Protocol::HttpProtobuf] { + let server = ServerTls::server_only(&platform); + let (endpoint, _collector): (String, Box) = match protocol { + Protocol::Grpc => { + let collector = OtlpTestServer::start_with_tls(&server).await; + (collector.endpoint().to_string(), Box::new(collector)) + } + Protocol::HttpProtobuf => { + let collector = OtlpHttpTestServer::start_with_tls(&server).await; + (collector.endpoint().to_string(), Box::new(collector)) + } + }; + assert!( + delivered(&endpoint, protocol, None).await, + "{protocol:?}: the platform roots trust the collector" + ); + assert!( + !delivered(&endpoint, protocol, Some(&ca_file)).await, + "{protocol:?}: a CA file that does not trust the collector replaces them" + ); + } +} diff --git a/crates/openshell-otel/tests/platform_roots.rs b/crates/openshell-otel/tests/platform_roots.rs new file mode 100644 index 0000000000..d091383ecf --- /dev/null +++ b/crates/openshell-otel/tests/platform_roots.rs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The only test in its binary, since it sets the process environment. + +use openshell_otel::{OtlpExportConfig, OtlpTarget, Protocol, ServiceName, SetupError, TlsError}; + +fn target(endpoint: &str) -> OtlpTarget { + OtlpTarget::new(&OtlpExportConfig { + endpoint, + service_name: ServiceName::Fixed("roots-test"), + service_version: None, + resource_attributes: Vec::new(), + }) + .unwrap() + .with_protocol(Protocol::HttpProtobuf) +} + +#[tokio::test] +#[allow(unsafe_code)] +async fn only_https_collectors_need_platform_roots() { + let empty = tempfile::NamedTempFile::new().unwrap(); + // Nothing else in this binary reads the environment. + unsafe { + std::env::set_var("SSL_CERT_FILE", empty.path()); + std::env::set_var("SSL_CERT_DIR", empty.path()); + } + + assert!(target("http://collector:4318").tracer_provider().is_ok()); + assert!(matches!( + target("https://collector:4318").tracer_provider(), + Err(SetupError::Tls(TlsError::NoPlatformRoots)) + )); +} diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 7ecffef1d2..5d37d6c8a5 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -268,6 +268,10 @@ impl SandboxSessionJwtAuthority { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }, gateway_id: self.gateway_id.clone(), verification_keys: self.verification_keys.clone(), diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 39a45f0b6e..81cbd7ca6f 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -936,6 +936,7 @@ pub struct ComputeRuntime { restart_notify: Arc, ssh_identities: Arc>, supervisor_otlp_headers: Arc>, + supervisor_otlp_tls: Arc>, /// Desired assignments of the shared supervisor host, when the gateway /// coordinates one for the in-process driver. supervisor_host: Option, @@ -1052,6 +1053,7 @@ impl ComputeRuntime { restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_otlp_headers: Arc::new(OnceLock::new()), + supervisor_otlp_tls: Arc::new(OnceLock::new()), supervisor_host: None, service_route_index: Arc::new(OnceLock::new()), }) @@ -1103,6 +1105,13 @@ impl ComputeRuntime { let _ = self.supervisor_otlp_headers.set(headers); } + pub(crate) fn configure_supervisor_otlp_tls( + &self, + tls: crate::otel_tracing::SupervisorOtlpTls, + ) { + let _ = self.supervisor_otlp_tls.set(tls); + } + pub(crate) async fn prepare_ssh_identity( &self, sandbox: &mut Sandbox, @@ -1132,6 +1141,9 @@ impl ComputeRuntime { if let Some(headers) = self.supervisor_otlp_headers.get() { headers.apply(&mut authentication.supervisor); } + if let Some(tls) = self.supervisor_otlp_tls.get() { + tls.apply(&mut authentication.supervisor); + } serialize_launch_authentication(authentication) } @@ -8622,6 +8634,7 @@ pub fn new_test_runtime_with_driver( restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_otlp_headers: Arc::new(OnceLock::new()), + supervisor_otlp_tls: Arc::new(OnceLock::new()), supervisor_host: None, service_route_index: Arc::new(OnceLock::new()), } @@ -9825,6 +9838,7 @@ mod tests { restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_otlp_headers: Arc::new(OnceLock::new()), + supervisor_otlp_tls: Arc::new(OnceLock::new()), supervisor_host: None, service_route_index: Arc::new(OnceLock::new()), } @@ -13633,7 +13647,7 @@ mod tests { } #[tokio::test] - async fn launch_bundles_carry_each_supervisor_exports_headers() { + async fn launch_bundles_carry_each_supervisor_exports_headers_and_tls() { use openshell_core::jwt::SecretOtlpHeaders; let driver = ControlledDriver::new(); @@ -13644,6 +13658,14 @@ mod tests { ocsf: secret("authorization=Bearer%20secret"), agent: secret("x-mlflow-experiment-id=0"), }); + runtime.configure_supervisor_otlp_tls(crate::otel_tracing::SupervisorOtlpTls { + materials: vec![openshell_core::jwt::OtlpTlsMaterial { + ca_pem: Some("agent-ca".to_string()), + ..Default::default() + }], + agent: Some(0), + ..Default::default() + }); let sandbox = sandbox_record( "sb-otlp-headers", "sandbox-otlp-headers", @@ -13682,6 +13704,14 @@ mod tests { Some("x-mlflow-experiment-id=0".to_string()), ] ); + assert!(bundle.traces_otlp_tls.is_none() && bundle.ocsf_otlp_tls.is_none()); + assert_eq!( + bundle + .agent_otlp() + .tls + .and_then(|tls| tls.ca_pem.as_deref()), + Some("agent-ca") + ); } #[tokio::test] diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index d7c096441a..8bc981094f 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -394,6 +394,21 @@ impl OtlpConfig { .map(|agent| agent.collector.endpoint.as_str()) } + /// Bytes of TLS files sent to supervisors, counting files shared by + /// several exports once. + fn supervisor_tls_bytes(&self) -> usize { + crate::otel_tracing::supervisor_otlp_tls(Some(self)) + .materials + .iter() + .map(|material| { + material.ca_pem.as_ref().map_or(0, String::len) + + material.client.as_ref().map_or(0, |client| { + client.cert_pem.len() + client.key_pem.expose_secret().len() + }) + }) + .sum() + } + /// The exports that compute drivers pass to supervisors. pub fn supervisor_otlp(&self) -> openshell_core::sandbox_env::SupervisorOtlp { openshell_core::sandbox_env::SupervisorOtlp { @@ -407,7 +422,7 @@ impl OtlpConfig { /// One OTLP collector. A lane table with its own `endpoint` describes its /// own collector; one without inherits the whole `[openshell.gateway.otlp]` /// collector. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct OtlpCollectorConfig { /// Collector endpoint, e.g. /// `http://otel-collector.observability.svc:4317`. @@ -425,6 +440,23 @@ pub struct OtlpCollectorConfig { /// File the headers are read from, one `name=value` per line. #[serde(default, skip_serializing_if = "Option::is_none")] pub headers_file: Option, + + /// PEM certificates trusted to verify an `https://` collector instead of + /// the platform roots. + #[serde(default, skip_serializing_if = "Option::is_none")] + ca_file: Option, + + /// PEM client certificate chain presented to an `https://` collector. + #[serde(default, skip_serializing_if = "Option::is_none")] + cert_file: Option, + + /// PEM private key for `cert_file`. + #[serde(default, skip_serializing_if = "Option::is_none")] + key_file: Option, + + /// TLS settings read from the files above when the config is loaded. + #[serde(skip)] + pub(crate) tls: openshell_otel::OtlpTls, } /// Headers the exporters set themselves, or that HTTP/2 forbids. @@ -665,26 +697,90 @@ impl std::fmt::Debug for OtlpHeaders { } impl OtlpCollectorConfig { - /// Rejects an endpoint with a query string, which no exporter sends, and - /// an `https://` endpoint for `http/protobuf`, which has no TLS. - fn checked(self, table: &str) -> Result { - let endpoint = self.endpoint.trim(); - if endpoint.contains('?') { - return Err(format!( - "{table} endpoint has a query string, which OTLP export does not send; set collector parameters with headers" - )); + /// The TLS settings read from `ca_file`, `cert_file`, and `key_file`. + pub fn tls(&self) -> &openshell_otel::OtlpTls { + &self.tls + } + + #[cfg(test)] + pub(crate) fn with_headers(mut self, headers: OtlpHeaders) -> Self { + self.headers = headers; + self + } + + #[cfg(test)] + pub(crate) fn with_tls(mut self, tls: openshell_otel::OtlpTls) -> Self { + self.tls = tls; + self + } + + pub fn ca_file(&self) -> Option<&Path> { + self.ca_file.as_deref() + } + + pub fn cert_file(&self) -> Option<&Path> { + self.cert_file.as_deref() + } + + pub fn key_file(&self) -> Option<&Path> { + self.key_file.as_deref() + } + + /// The collector verified with `ca_file` and presenting `cert_file` and + /// `key_file`, read now. + pub fn with_tls_files( + self, + ca_file: Option, + cert_file: Option, + key_file: Option, + ) -> Result { + self.read_tls_files("otlp collector", ca_file, cert_file, key_file) + } + + fn read_tls_files( + mut self, + table: &str, + ca_file: Option, + cert_file: Option, + key_file: Option, + ) -> Result { + if ca_file.is_none() && cert_file.is_none() && key_file.is_none() { + return Ok(self); } - let https = endpoint + if !self + .endpoint + .trim() .get(..8) - .is_some_and(|scheme| scheme.eq_ignore_ascii_case("https://")); - if https && self.protocol == OtlpProtocol::HttpProtobuf { + .is_some_and(|scheme| scheme.eq_ignore_ascii_case("https://")) + { return Err(format!( - "{table} uses an https:// endpoint, which http/protobuf does not support" + "{table} sets TLS files, which need an https:// endpoint" )); } + self.tls = openshell_otel::OtlpTls::from_files( + ca_file.as_deref(), + cert_file.as_deref(), + key_file.as_deref(), + ) + .map_err(|error| format!("{table}: {error}"))?; + (self.ca_file, self.cert_file, self.key_file) = (ca_file, cert_file, key_file); Ok(self) } + /// A collector at `endpoint` with no headers or TLS files. + pub fn new(endpoint: impl Into, protocol: OtlpProtocol) -> Self { + Self { + endpoint: endpoint.into(), + protocol, + headers: OtlpHeaders::default(), + headers_file: None, + ca_file: None, + cert_file: None, + key_file: None, + tls: openshell_otel::OtlpTls::default(), + } + } + fn export(&self) -> openshell_core::sandbox_env::OtlpExport { openshell_core::sandbox_env::OtlpExport { endpoint: self.endpoint.clone(), @@ -700,14 +796,14 @@ fn is_grpc(protocol: &OtlpProtocol) -> bool { } /// `[openshell.gateway.otlp.traces]`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct OtlpTracesConfig { #[serde(flatten)] pub collector: OtlpCollectorConfig, } /// `[openshell.gateway.otlp.ocsf]`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct OcsfOtlpConfig { #[serde(flatten)] pub collector: OtlpCollectorConfig, @@ -718,7 +814,7 @@ pub struct OcsfOtlpConfig { } /// `[openshell.gateway.otlp.agent]`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct AgentOtlpConfig { /// The collector that sandbox supervisors forward workload traces to. #[serde(flatten)] @@ -737,6 +833,12 @@ struct RawOtlpConfig { #[serde(default)] headers_file: Option, #[serde(default)] + ca_file: Option, + #[serde(default)] + cert_file: Option, + #[serde(default)] + key_file: Option, + #[serde(default)] service_name: Option, #[serde(default)] traces: Option, @@ -759,6 +861,12 @@ struct RawAgentOtlp { headers: Option>, #[serde(default)] headers_file: Option, + #[serde(default)] + ca_file: Option, + #[serde(default)] + cert_file: Option, + #[serde(default)] + key_file: Option, } #[derive(Deserialize)] @@ -772,6 +880,12 @@ struct RawOtlpTraces { headers: Option>, #[serde(default)] headers_file: Option, + #[serde(default)] + ca_file: Option, + #[serde(default)] + cert_file: Option, + #[serde(default)] + key_file: Option, } #[derive(Deserialize)] @@ -788,15 +902,30 @@ struct RawOcsfOtlp { #[serde(default)] headers_file: Option, #[serde(default)] + ca_file: Option, + #[serde(default)] + cert_file: Option, + #[serde(default)] + key_file: Option, + #[serde(default)] schema_version: Option, } +/// Most bytes of TLS files that sandbox launch credentials carry, counting +/// each distinct collector's files once: what one collector's three files +/// may hold, which leaves room in a shared supervisor's 256 KiB launch +/// material for the rest of the credentials. +const MAX_SUPERVISOR_TLS_BYTES: usize = 3 * openshell_otel::MAX_TLS_FILE_BYTES; + /// The collector keys every OTLP table accepts. struct RawCollector { endpoint: Option, protocol: Option, headers: Option>, headers_file: Option, + ca_file: Option, + cert_file: Option, + key_file: Option, } macro_rules! raw_collector { @@ -806,6 +935,9 @@ macro_rules! raw_collector { protocol: $raw.protocol, headers: $raw.headers, headers_file: $raw.headers_file, + ca_file: $raw.ca_file, + cert_file: $raw.cert_file, + key_file: $raw.key_file, } }; } @@ -817,6 +949,9 @@ impl RawCollector { ("protocol", self.protocol.is_some()), ("headers", self.headers.is_some()), ("headers_file", self.headers_file.is_some()), + ("ca_file", self.ca_file.is_some()), + ("cert_file", self.cert_file.is_some()), + ("key_file", self.key_file.is_some()), ] .into_iter() .find_map(|(key, set)| set.then_some(key)) @@ -840,13 +975,17 @@ impl RawCollector { format!("{table} needs an endpoint, or otlp.endpoint to default to") }); }; - OtlpCollectorConfig { - endpoint, - protocol: self.protocol.unwrap_or_default(), - headers: OtlpHeaders::resolve(table, self.headers, self.headers_file.as_deref())?, - headers_file: self.headers_file, + if endpoint.contains('?') { + return Err(format!( + "{table} endpoint has a query string, which OTLP export does not send; set collector parameters with headers" + )); } - .checked(table) + let mut collector = OtlpCollectorConfig::new(endpoint, self.protocol.unwrap_or_default()); + collector = collector.read_tls_files(table, self.ca_file, self.cert_file, self.key_file)?; + collector.headers = + OtlpHeaders::resolve(table, self.headers, self.headers_file.as_deref())?; + collector.headers_file = self.headers_file; + Ok(collector) } } @@ -895,12 +1034,21 @@ impl TryFrom for OtlpConfig { .to_string(), ); } - Ok(Self { + let config = Self { service_name: raw.service_name, traces, ocsf, agent, - }) + }; + let supervisor_tls = config.supervisor_tls_bytes(); + if supervisor_tls > MAX_SUPERVISOR_TLS_BYTES { + return Err(format!( + "otlp TLS files sent to supervisors total {supervisor_tls} bytes across their \ + distinct collectors, more than {MAX_SUPERVISOR_TLS_BYTES}; share TLS files \ + between tables or use smaller files" + )); + } + Ok(config) } } @@ -1935,12 +2083,7 @@ service_name = "openshell-gateway-dev" assert_eq!( otlp.ocsf, Some(OcsfOtlpConfig { - collector: OtlpCollectorConfig { - endpoint: "http://127.0.0.1:4317".into(), - protocol: OtlpProtocol::Grpc, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new("http://127.0.0.1:4317", OtlpProtocol::Grpc), schema_version: None, }) ); @@ -1957,12 +2100,7 @@ service_name = "openshell-gateway-dev" assert_eq!( otlp.ocsf, Some(OcsfOtlpConfig { - collector: OtlpCollectorConfig { - endpoint: "http://siem:4317".into(), - protocol: OtlpProtocol::Grpc, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new("http://siem:4317", OtlpProtocol::Grpc), schema_version: None, }) ); @@ -2117,19 +2255,6 @@ service_name = "openshell-gateway-dev" let error = load(tmp.path()).expect_err("http/json is not supported"); assert!(format!("{error:#}").contains("http/json"), "{error:#}"); - for table in [ - "[openshell.gateway.otlp]\nendpoint = 'https://shared:4318'\nprotocol = 'http/protobuf'\n", - "[openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'HTTPS://agents:4318'\n\ - protocol = 'http/protobuf'\n", - ] { - let tmp = write_tmp(table); - let error = load(tmp.path()).expect_err("https is not supported over http/protobuf"); - assert!( - format!("{error:#}").contains("which http/protobuf does not support"), - "{table}: {error:#}" - ); - } - for (table, expected) in [ ( "[openshell.gateway.otlp.agent]\nenabled = true\n\ @@ -2397,14 +2522,178 @@ service_name = "openshell-gateway-dev" assert!(!debug.contains("secret-token"), "{debug}"); } + /// A directory holding a test PKI's `ca.pem`, `client.pem`, and + /// `client-key.pem`. + fn pki_files() -> (openshell_otel_test_support::TestPki, tempfile::TempDir) { + let pki = openshell_otel_test_support::TestPki::generate(); + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("ca.pem"), &pki.ca_pem).unwrap(); + std::fs::write(dir.path().join("client.pem"), &pki.client.cert_pem).unwrap(); + std::fs::write(dir.path().join("client-key.pem"), &pki.client.key_pem).unwrap(); + (pki, dir) + } + + #[test] + fn otlp_tls_files_sent_to_supervisors_fit_their_launch_bundle() { + let pki = openshell_otel_test_support::TestPki::generate(); + let dir = tempfile::tempdir().unwrap(); + let repeated = |pem: &str, bytes: usize| pem.repeat(bytes / pem.len()); + let write = |name: &str, contents: &str| { + let path = dir.path().join(name); + std::fs::write(&path, contents).unwrap(); + path.display().to_string() + }; + let key = write("client-key.pem", &pki.client.key_pem); + let files = |lane: usize| { + let ca = write( + &format!("ca-{lane}.pem"), + &repeated(&pki.ca_pem, 60 * 1024 - lane * pki.ca_pem.len()), + ); + let cert = write( + &format!("client-{lane}.pem"), + &repeated(&pki.client.cert_pem, 60 * 1024), + ); + format!("ca_file = '{ca}'\ncert_file = '{cert}'\nkey_file = '{key}'\n") + }; + + otlp(&format!( + "[openshell.gateway.otlp]\nendpoint = 'https://shared:4317'\n{}\n\ + [openshell.gateway.otlp.ocsf]\nenabled = true\n\n\ + [openshell.gateway.otlp.agent]\nenabled = true\n", + files(0) + )); + + let tmp = write_tmp(&format!( + "[openshell.gateway.otlp.traces]\nendpoint = 'https://traces:4317'\n{}\n\ + [openshell.gateway.otlp.ocsf]\nenabled = true\nendpoint = 'https://siem:4317'\n{}\n\ + [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'https://agents:4317'\n{}", + files(1), + files(2), + files(3) + )); + let error = format!("{:#}", load(tmp.path()).expect_err("TLS files too large")); + assert!( + error.contains("TLS files sent to supervisors total"), + "{error}" + ); + } + + #[test] + fn otlp_tls_files_are_read_for_the_collector_they_belong_to() { + let (pki, dir) = pki_files(); + let path = |name: &str| dir.path().join(name).display().to_string(); + let otlp = otlp(&format!( + "[openshell.gateway.otlp]\nendpoint = 'https://shared:4317'\n\ + ca_file = '{ca}'\ncert_file = '{cert}'\nkey_file = '{key}'\n\n\ + [openshell.gateway.otlp.ocsf]\nenabled = true\n\n\ + [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'https://agents:4318'\n\ + ca_file = '{ca}'\n", + ca = path("ca.pem"), + cert = path("client.pem"), + key = path("client-key.pem"), + )); + let mutual = openshell_otel::OtlpTls::from_pem( + Some(pki.ca_pem.as_bytes()), + Some(( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + )), + ) + .unwrap(); + let ca_only = openshell_otel::OtlpTls::from_pem(Some(pki.ca_pem.as_bytes()), None).unwrap(); + assert_eq!(otlp.traces.unwrap().collector.tls, mutual); + assert_eq!( + otlp.ocsf.unwrap().collector.tls, + mutual, + "inherits the shared collector" + ); + assert_eq!( + otlp.agent.unwrap().collector.tls, + ca_only, + "inherits nothing" + ); + } + + #[test] + fn otlp_tls_files_are_rejected_when_unusable() { + let (_pki, dir) = pki_files(); + let path = |name: &str| dir.path().join(name).display().to_string(); + for (table, problem) in [ + ( + format!( + "endpoint = 'http://shared:4317'\nca_file = '{}'", + path("ca.pem") + ), + "need an https:// endpoint".to_string(), + ), + ( + format!( + "endpoint = 'https://shared:4317'\ncert_file = '{}'", + path("client.pem") + ), + "otlp: cert_file is set without key_file".to_string(), + ), + ( + format!( + "endpoint = 'https://shared:4317'\nkey_file = '{}'", + path("client-key.pem") + ), + "otlp: key_file is set without cert_file".to_string(), + ), + ( + format!( + "endpoint = 'https://shared:4317'\nca_file = '{}'", + path("missing.pem") + ), + format!("otlp: ca_file {}", path("missing.pem")), + ), + ( + format!( + "endpoint = 'https://shared:4317'\nca_file = '{}'", + path("client-key.pem") + ), + "no PEM certificate".to_string(), + ), + ] { + let tmp = write_tmp(&format!("[openshell.gateway.otlp]\n{table}\n")); + let error = format!("{:#}", load(tmp.path()).expect_err(&table)); + assert!(error.contains(&problem), "{table}: {error}"); + } + + let tmp = write_tmp(&format!( + "[openshell.gateway.otlp]\nendpoint = 'https://shared:4317'\n\n\ + [openshell.gateway.otlp.traces]\nca_file = '{}'\n", + path("ca.pem") + )); + let error = format!("{:#}", load(tmp.path()).unwrap_err()); + assert!( + error.contains("sets ca_file without its own endpoint"), + "{error}" + ); + + let tmp = write_tmp(&format!( + "[openshell.gateway.otlp]\nca_file = '{}'\n\n\ + [openshell.gateway.otlp.traces]\nendpoint = 'https://traces:4317'\n", + path("ca.pem") + )); + let error = format!("{:#}", load(tmp.path()).unwrap_err()); + assert!( + error.contains("otlp sets ca_file without its own endpoint"), + "{error}" + ); + } + #[test] fn otlp_config_round_trips() { - let tmp = write_tmp( + let (_pki, pki_dir) = pki_files(); + let tmp = write_tmp(&format!( "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317'\nservice_name = 'gw'\n\n\ [openshell.gateway.otlp.ocsf]\nenabled = true\n\n\ [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'http://agents:4318'\n\ - protocol = 'http/protobuf'\nheaders = { x-mlflow-experiment-id = '0' }\n", - ); + protocol = 'http/protobuf'\nheaders = {{ x-mlflow-experiment-id = '0' }}\n\n\ + [openshell.gateway.otlp.traces]\nendpoint = 'https://traces:4317'\nca_file = '{ca}'\n", + ca = pki_dir.path().join("ca.pem").display(), + )); let file = load(tmp.path()).unwrap(); let encoded = toml::to_string(&file).unwrap(); let decoded = toml::from_str::(&encoded).unwrap(); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 9a62afa27c..2b0a542bc6 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -745,6 +745,11 @@ pub(crate) async fn run_server( .as_ref() .and_then(|file| file.openshell.gateway.otlp.as_ref()), )); + compute.configure_supervisor_otlp_tls(otel_tracing::supervisor_otlp_tls( + config_file + .as_ref() + .and_then(|file| file.openshell.gateway.otlp.as_ref()), + )); let gateway_interceptors = if let Some(issuer) = extension_jwt_issuer.as_ref() { let mut slots = BTreeMap::new(); for interceptor in &config.gateway_interceptors { diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index 61a1aac441..ca799ba411 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -23,7 +23,7 @@ //! Diagnostic logs and metrics are not exported over OTLP. They have their own //! surfaces (the gateway log stream and the Prometheus `/metrics` endpoint). -use openshell_core::jwt::SecretOtlpHeaders; +use openshell_core::jwt::{OtlpTlsMaterial, SecretOtlpHeaders}; use openshell_core::sandbox_env::OtlpProtocol; use openshell_otel::{OcsfOtlp, OtlpExportConfig, OtlpTarget, ServiceName}; pub use openshell_otel::{SetupError, TraceContextInterceptor, mark_error}; @@ -122,9 +122,8 @@ pub struct OtlpExport { /// Validated collector that supervisors relay workload traces to. The /// gateway itself exports nothing for this signal. agent: Option>, - /// Headers configured for the trace collector, which in-process compute - /// drivers also send. - traces_headers: http::HeaderMap, + /// How exports reach the trace collector. + traces_transport: openshell_otel::CollectorTransport, } /// One signal's exporter and the endpoint it sends to. @@ -157,12 +156,14 @@ impl OtlpExport { cfg: &OtlpConfig, gateway: GatewayResourceAttributes<'_>, ) -> Result { + let transport = |collector: &OtlpCollectorConfig| openshell_otel::CollectorTransport { + protocol: collector.protocol.into(), + headers: collector.headers.header_map(), + tls: collector.tls.clone(), + }; let target = |collector: &OtlpCollectorConfig| { - OtlpTarget::new(&export_config(cfg, &collector.endpoint, gateway)).map(|target| { - target - .with_protocol(collector.protocol.into()) - .with_headers(collector.headers.header_map()) - }) + OtlpTarget::new(&export_config(cfg, &collector.endpoint, gateway)) + .map(|target| target.with_transport(&transport(collector))) }; let traces = cfg .traces @@ -203,10 +204,10 @@ impl OtlpExport { traces, ocsf, agent, - traces_headers: cfg + traces_transport: cfg .traces .as_ref() - .map(|traces| traces.collector.headers.header_map()) + .map(|traces| transport(&traces.collector)) .unwrap_or_default(), }) } @@ -240,9 +241,10 @@ impl OtlpExport { .map_or(OtlpProtocol::Grpc, |traces| traces.protocol) } - /// The headers configured for the trace collector. - pub fn traces_headers(&self) -> http::HeaderMap { - self.traces_headers.clone() + /// How exports reach the trace collector, which in-process compute + /// drivers also use. + pub fn traces_transport(&self) -> openshell_otel::CollectorTransport { + self.traces_transport.clone() } /// The trace provider, when trace export is enabled. @@ -356,6 +358,70 @@ fn supervisor_otlp_headers_from( } } +/// TLS material for the supervisor exports, delivered in the launch bundle +/// like [`SupervisorOtlpHeaders`]. Each export names its collector's +/// material by position in `materials`. +#[derive(Clone, Debug, Default)] +pub struct SupervisorOtlpTls { + pub materials: Vec, + pub traces: Option, + pub ocsf: Option, + pub agent: Option, +} + +impl SupervisorOtlpTls { + pub fn apply(&self, bundle: &mut openshell_core::jwt::SupervisorAuthBundle) { + bundle.otlp_tls.clone_from(&self.materials); + bundle.traces_otlp_tls = self.traces; + bundle.ocsf_otlp_tls = self.ocsf; + bundle.agent_otlp_tls = self.agent; + } +} + +/// The TLS material each enabled supervisor export uses: its collector's, +/// with each distinct material sent once. +pub fn supervisor_otlp_tls(otlp: Option<&OtlpConfig>) -> SupervisorOtlpTls { + let Some(otlp) = otlp else { + return SupervisorOtlpTls::default(); + }; + let material = |collector: &OtlpCollectorConfig| { + let tls = collector.tls(); + let pem = |bytes: &[u8]| String::from_utf8_lossy(bytes).into_owned(); + (!tls.is_default()).then(|| OtlpTlsMaterial { + ca_pem: tls.ca_pem().map(pem), + client: tls.client_identity().map(|(cert, key)| { + openshell_core::jwt::OtlpClientIdentity { + cert_pem: pem(cert), + key_pem: openshell_core::jwt::SecretPrivateKeyPem::new(pem(key)), + } + }), + }) + }; + let lanes = [ + otlp.traces.as_ref().map(|traces| &traces.collector), + otlp.ocsf.as_ref().map(|ocsf| &ocsf.collector), + otlp.agent.as_ref().map(|agent| &agent.collector), + ]; + let mut materials = Vec::new(); + let mut seen = Vec::new(); + let [traces, ocsf, agent] = lanes.map(|collector| { + let collector = collector?; + let pem = (collector.tls().ca_pem(), collector.tls().client_identity()); + if let Some(index) = seen.iter().position(|seen| *seen == pem) { + return Some(index); + } + materials.push(material(collector)?); + seen.push(pem); + Some(materials.len() - 1) + }); + SupervisorOtlpTls { + materials, + traces, + ocsf, + agent, + } +} + /// Build the `tracing` layer that forwards spans to `provider`. /// /// Events stay on the gateway's logging layers. Spans emitted by the @@ -684,8 +750,9 @@ pub mod test_exporter { #[cfg(test)] mod tests { use super::*; - use crate::config_file::{AgentOtlpConfig, OcsfOtlpConfig, OtlpHeaders, OtlpTracesConfig}; + use crate::config_file::{AgentOtlpConfig, OcsfOtlpConfig, OtlpTracesConfig}; use crate::test_environment::Environment; + use openshell_core::sandbox_env::OtlpProtocol; fn config() -> OtlpConfig { with_traces_endpoint("http://127.0.0.1:4317") @@ -695,12 +762,7 @@ mod tests { OtlpConfig { service_name: None, traces: Some(OtlpTracesConfig { - collector: OtlpCollectorConfig { - endpoint: endpoint.into(), - protocol: OtlpProtocol::Grpc, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new(endpoint, OtlpProtocol::Grpc), }), ocsf: None, agent: None, @@ -709,12 +771,7 @@ mod tests { fn ocsf_config(endpoint: &str) -> OcsfOtlpConfig { OcsfOtlpConfig { - collector: OtlpCollectorConfig { - endpoint: endpoint.into(), - protocol: OtlpProtocol::Grpc, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new(endpoint, OtlpProtocol::Grpc), schema_version: None, } } @@ -742,12 +799,8 @@ mod tests { ocsf.collector.headers = [("authorization", "toml")].into(); otlp.ocsf = Some(ocsf); otlp.agent = Some(AgentOtlpConfig { - collector: OtlpCollectorConfig { - endpoint: "http://mlflow:5000".into(), - protocol: OtlpProtocol::HttpProtobuf, - headers: [("x-mlflow-experiment-id", "0")].into(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new("http://mlflow:5000", OtlpProtocol::HttpProtobuf) + .with_headers([("x-mlflow-experiment-id", "0")].into()), }); let headers = supervisor_otlp_headers_from( @@ -822,6 +875,78 @@ mod tests { }); } + #[test] + fn supervisors_receive_each_collectors_tls_material() { + let pki = openshell_otel_test_support::TestPki::generate(); + let mutual = openshell_otel::OtlpTls::from_pem( + Some(pki.ca_pem.as_bytes()), + Some(( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + )), + ) + .unwrap(); + let ca_only = openshell_otel::OtlpTls::from_pem(Some(pki.ca_pem.as_bytes()), None).unwrap(); + let mut otlp = with_traces_endpoint("https://traces:4317"); + otlp.traces.as_mut().unwrap().collector.tls = mutual; + otlp.agent = Some(AgentOtlpConfig { + collector: OtlpCollectorConfig::new("https://agents:4318", OtlpProtocol::Grpc) + .with_tls(ca_only), + }); + otlp.ocsf = Some(ocsf_config("https://siem:4317")); + + let tls = supervisor_otlp_tls(Some(&otlp)); + let material = |index: Option| &tls.materials[index.expect("collector TLS")]; + let traces = material(tls.traces); + assert_eq!(traces.ca_pem.as_deref(), Some(pki.ca_pem.as_str())); + let client = traces + .client + .as_ref() + .expect("trace collector client certificate"); + assert_eq!(client.cert_pem, pki.client.cert_pem); + assert_eq!(client.key_pem.expose_secret(), pki.client.key_pem); + let agent = material(tls.agent); + assert_eq!(agent.ca_pem.as_deref(), Some(pki.ca_pem.as_str())); + assert!(agent.client.is_none()); + assert!(tls.ocsf.is_none(), "the platform roots need no material"); + assert!(supervisor_otlp_tls(None).materials.is_empty()); + } + + #[test] + fn supervisors_receive_each_distinct_tls_material_once() { + let pki = openshell_otel_test_support::TestPki::generate(); + let mutual = || { + openshell_otel::OtlpTls::from_pem( + Some(pki.ca_pem.as_bytes()), + Some(( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + )), + ) + .unwrap() + }; + let collector = |endpoint: &str| { + OtlpCollectorConfig::new(endpoint, OtlpProtocol::Grpc).with_tls(mutual()) + }; + let otlp = OtlpConfig { + service_name: None, + traces: Some(OtlpTracesConfig { + collector: collector("https://shared:4317"), + }), + ocsf: Some(OcsfOtlpConfig { + collector: collector("https://shared:4317"), + schema_version: None, + }), + agent: Some(AgentOtlpConfig { + collector: collector("https://agents:4318"), + }), + }; + + let tls = supervisor_otlp_tls(Some(&otlp)); + assert_eq!(tls.materials.len(), 1); + assert_eq!([tls.traces, tls.ocsf, tls.agent], [Some(0); 3]); + } + #[tokio::test] async fn each_signal_exports_only_when_configured() { let (export, error) = @@ -893,12 +1018,7 @@ mod tests { use tracing_subscriber::prelude::*; let mut collector = openshell_otel_test_support::OtlpHttpTestServer::start().await; - let http = |endpoint: &str| OtlpCollectorConfig { - endpoint: endpoint.into(), - protocol: OtlpProtocol::HttpProtobuf, - headers: OtlpHeaders::default(), - headers_file: None, - }; + let http = |endpoint: &str| OtlpCollectorConfig::new(endpoint, OtlpProtocol::HttpProtobuf); let mut cfg = config(); cfg.traces = Some(OtlpTracesConfig { collector: http(collector.endpoint()), @@ -913,6 +1033,10 @@ mod tests { assert_eq!(export.traces_protocol(), OtlpProtocol::HttpProtobuf); assert_eq!(export.ocsf_protocol(), Some(OtlpProtocol::HttpProtobuf)); assert_eq!(export.agent_protocol(), None); + assert_eq!( + export.traces_transport().protocol, + openshell_otel::Protocol::HttpProtobuf + ); let provider = export.tracer_provider().expect("trace export").clone(); provider.tracer("test").in_span("gateway-span", |_| {}); @@ -978,7 +1102,7 @@ mod tests { assert!(error.is_none(), "unexpected setup error: {error:?}"); let export = export.expect("OTLP export"); assert_eq!( - export.traces_headers()["authorization"], + export.traces_transport().headers["authorization"], "Bearer traces-token", "in-process compute drivers send the trace collector's headers" ); @@ -1002,11 +1126,9 @@ mod tests { .set("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "authorization=logs-env") .remove("OTEL_EXPORTER_OTLP_HEADERS") .run_async(async { - let configured = |endpoint: &str, protocol| OtlpCollectorConfig { - endpoint: endpoint.into(), - protocol, - headers: [("authorization", "toml"), ("x-tenant", "blue")].into(), - headers_file: None, + let configured = |endpoint: &str, protocol| { + OtlpCollectorConfig::new(endpoint, protocol) + .with_headers([("authorization", "toml"), ("x-tenant", "blue")].into()) }; let start = |collector: OtlpCollectorConfig| { let mut cfg = config(); @@ -1049,6 +1171,60 @@ mod tests { .await; } + #[tokio::test(flavor = "multi_thread")] + async fn gateway_signals_verify_their_collectors_with_the_configured_tls() { + use openshell_ocsf::{ConfigStateChangeBuilder, ocsf_emit}; + use openshell_otel_test_support::{OtlpTestServer, ServerTls, TestPki}; + use opentelemetry::trace::{Tracer as _, TracerProvider as _}; + use tracing_subscriber::prelude::*; + + let pki = TestPki::generate(); + let collector = OtlpTestServer::start_with_tls(&ServerTls::mutual(&pki)).await; + let tls = openshell_otel::OtlpTls::from_pem( + Some(pki.ca_pem.as_bytes()), + Some(( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + )), + ) + .unwrap(); + let collector_config = + OtlpCollectorConfig::new(collector.endpoint(), OtlpProtocol::Grpc).with_tls(tls); + let mut cfg = config(); + cfg.traces = Some(OtlpTracesConfig { + collector: collector_config.clone(), + }); + cfg.ocsf = Some(OcsfOtlpConfig { + collector: collector_config, + schema_version: None, + }); + let (export, error) = OtlpExport::start(Some(&cfg), GatewayResourceAttributes::default()); + assert!(error.is_none(), "unexpected setup error: {error:?}"); + let export = export.expect("OTLP export"); + + export + .tracer_provider() + .expect("trace export") + .tracer("test") + .in_span("gateway-span", |_| {}); + let event = + ConfigStateChangeBuilder::new(&crate::gateway_ocsf::context("sandbox-1", "sandbox")) + .message("policy approved") + .build(); + let subscriber = + tracing_subscriber::registry().with(export.ocsf().expect("OCSF log export").layer()); + tracing::subscriber::with_default(subscriber, || ocsf_emit!(event)); + tokio::task::spawn_blocking(move || { + tokio::runtime::Handle::current().block_on(export.shutdown()); + }) + .await + .unwrap(); + let received = collector.shutdown().await; + + assert_eq!(received.spans.len(), 1); + assert_eq!(received.log_bodies.len(), 1); + } + #[test] fn resource_defaults_the_service_name() { Environment::new().remove("OTEL_SERVICE_NAME").run(|| { @@ -1214,12 +1390,7 @@ mod tests { bad_ocsf.ocsf = Some(ocsf_config("definitely not a url")); let mut bad_agent = config(); bad_agent.agent = Some(AgentOtlpConfig { - collector: OtlpCollectorConfig { - endpoint: "definitely not a url".into(), - protocol: OtlpProtocol::Grpc, - headers: OtlpHeaders::default(), - headers_file: None, - }, + collector: OtlpCollectorConfig::new("definitely not a url", OtlpProtocol::Grpc), }); for cfg in [ with_traces_endpoint("definitely not a url"), diff --git a/crates/openshell-server/src/persistence/mutation_lock_pg_tests.rs b/crates/openshell-server/src/persistence/mutation_lock_pg_tests.rs index c7866520f8..3bf37d1a33 100644 --- a/crates/openshell-server/src/persistence/mutation_lock_pg_tests.rs +++ b/crates/openshell-server/src/persistence/mutation_lock_pg_tests.rs @@ -88,6 +88,10 @@ async fn postgres_concurrent_sandbox_creates_with_ssh_identity_do_not_exhaust_lo traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }, gateway_id: "gateway".to_string(), verification_keys: Vec::new(), diff --git a/crates/openshell-server/src/ssh_identity.rs b/crates/openshell-server/src/ssh_identity.rs index 7401905970..c8ce7b269a 100644 --- a/crates/openshell-server/src/ssh_identity.rs +++ b/crates/openshell-server/src/ssh_identity.rs @@ -75,6 +75,10 @@ mod tests { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }, gateway_id: "gateway".to_string(), verification_keys: Vec::new(), diff --git a/crates/openshell-server/src/tracing_setup.rs b/crates/openshell-server/src/tracing_setup.rs index b0ac24af38..da92824630 100644 --- a/crates/openshell-server/src/tracing_setup.rs +++ b/crates/openshell-server/src/tracing_setup.rs @@ -71,19 +71,15 @@ pub fn install( ) -> (TracingHandle, Option) { let driver_otlp = driver.is_some().then_some(otlp).flatten(); let driver_endpoint = driver_otlp.and_then(OtlpExport::traces_endpoint); - let driver_protocol = driver_otlp.map_or(openshell_otel::Protocol::Grpc, |otlp| { - otlp.traces_protocol().into() - }); - let driver_headers = driver_otlp - .map(OtlpExport::traces_headers) + let driver_transport = driver_otlp + .map(OtlpExport::traces_transport) .unwrap_or_default(); let (driver_tracer_provider, driver_setup_error) = driver.map_or_else( || (None, None), |descriptor| { descriptor.provider_for( driver_endpoint, - driver_protocol, - &driver_headers, + &driver_transport, openshell_core::VERSION, gateway.name(), gateway.compute_driver(), diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 513e40055e..3abe904590 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -117,7 +117,7 @@ Console JSON is installed independently of optional file appenders and uses an I ## OTLP agent trace relay -The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_AGENT_ENDPOINT`, over the transport named by `OPENSHELL_OTLP_AGENT_PROTOCOL` (`grpc` when unset, or `http/protobuf`). Each export carries the collector headers from the launch bundle's `agent_otlp_headers`; the supervisor's own span and OCSF exporters use `traces_otlp_headers` and `ocsf_otlp_headers` the same way. The gateway sets those variables only when `[openshell.gateway.otlp.agent]` is enabled, so trace export alone leaves the relay inactive. Metrics and logs exports to the relay are acknowledged and discarded. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards the encoded batches to the collector with a lazy connection. +The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_AGENT_ENDPOINT`, over the transport named by `OPENSHELL_OTLP_AGENT_PROTOCOL` (`grpc` when unset, or `http/protobuf`). Each export carries the collector headers from the launch bundle's `agent_otlp_headers` and, for an `https` collector, verifies it and authenticates with `agent_otlp_tls`; the supervisor's own span and OCSF exporters use the `traces_` and `ocsf_` fields the same way. The gateway sets those variables only when `[openshell.gateway.otlp.agent]` is enabled, so trace export alone leaves the relay inactive. Metrics and logs exports to the relay are acknowledged and discarded. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards the encoded batches to the collector with a lazy connection. Limits are fixed: 2 MiB per request body, at most 512 `ResourceSpans` entries, 256 KiB of encoded `Resource` fields, and 8192 resource attributes per request (each of these answers 413), a 10 second header-read timeout, 16 concurrent connections, 32 buffered batches, and a 5 second deadline per export attempt. Only the resources are decoded; spans are copied byte for byte, at most two requests at a time. Worst-case memory attributable to the agent is about 67 MiB of buffered batches, plus 32 MiB of bodies awaiting enrichment across the 16 connections, plus about 12 MiB for the two enrichments in progress, plus about 4 MiB for the batch being exported and its encoded copy. A full buffer answers 503 with `Retry-After`. Without an agent endpoint nothing starts and the proxy refuses the relay address. After the agent exits, `run_sandbox_with_backend` flushes buffered batches for up to 5 seconds before it reports the exit to the gateway, then logs a final summary of rejected and failed counts. Flushes skip the wait while the most recent export failed, and batches still pending when the relay is dropped without a flush count as failed. The supervisor tells the sandbox runtime through `AgentSpec::otlp_relay_active` whether the relay runs, and the runtime injects the generic `OTEL_EXPORTER_OTLP_*` defaults only then. diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 3cffba396e..18564d13ec 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -162,6 +162,10 @@ impl TestSetup { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }, sandbox_id: "sandbox-1".to_string(), selected_name: TEST_BACKEND, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index 12af08ccae..7b2b60859f 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -274,6 +274,10 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }; let setup = HeldBackendSetup { session_id, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 7a592cca4d..398c4bef77 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -28,6 +28,9 @@ mod otlp_relay; mod provider_readiness; pub mod sandbox_instance_host; pub mod shared_host; +#[cfg(test)] +#[path = "../../../tests/support/environment.rs"] +mod test_environment; use config_runtime::{agent_proposals_enabled_from_settings, run_policy_poll_loop}; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -1304,7 +1307,8 @@ pub async fn run_sandbox_instance( let (mut otlp_relay, otlp_reserved_destination) = otlp_relay::activate( otlp_agent_endpoint, otlp_agent_protocol, - auth_bundle.agent_otlp_headers.clone(), + auth_bundle.agent_otlp().headers.cloned(), + auth_bundle.agent_otlp().tls.cloned(), sandbox_id.clone(), ); @@ -4653,6 +4657,10 @@ mod tests { traces_otlp_headers: None, ocsf_otlp_headers: None, agent_otlp_headers: None, + otlp_tls: Vec::new(), + traces_otlp_tls: None, + ocsf_otlp_tls: None, + agent_otlp_tls: None, }; let config = SandboxInstanceConfig { app_router: None, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index f0186e2d23..f45c9f4f38 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -11,6 +11,9 @@ use std::sync::atomic::AtomicBool; use clap::{Parser, ValueEnum}; use miette::{IntoDiagnostic, Result}; +use openshell_core::jwt::{ + OtlpExportCredentials, OtlpTlsMaterial, SecretOtlpHeaders, SupervisorAuthBundle, +}; use openshell_core::sandbox_env::OtlpProtocol; use openshell_isolation_interface::contract::BackendDescriptor; use tracing::{error, info, warn}; @@ -344,7 +347,7 @@ fn backend_setup( Ok(Box::new(registrations.resolve(admitted_backend, launch)?)) } -fn auth_bundle(args: &Args) -> Result { +fn auth_bundle(args: &Args) -> Result { let path = args.auth_bundle_file.as_deref().ok_or_else(|| { miette::miette!("--auth-bundle-file is required for --role=isolation-backend") })?; @@ -354,7 +357,7 @@ fn auth_bundle(args: &Args) -> Result path.display() ) })?; - let bundle = serde_json::from_slice::(&bytes) + let bundle = serde_json::from_slice::(&bytes) .map_err(|error| miette::miette!("decode supervisor authentication bundle: {error}"))?; bundle .validate() @@ -500,19 +503,21 @@ fn main() -> Result<()> { ), }; // The exporters must be created inside the runtime that drives them. - let headers = bundle_otlp_headers(isolation_inputs.as_ref().map(|(_, _, auth, _, _)| auth)); + let settings = bundle_export_settings( + isolation_inputs.as_ref().map(|(_, _, auth, _, _)| auth), + args.otlp_protocol, + args.otlp_ocsf_protocol, + ); let (otlp_provider, otlp_setup_error) = runtime.block_on(async { start_trace_export( args.otlp_endpoint.as_deref().map(export_config), - args.otlp_protocol, - headers.traces, + settings.traces, ) }); let (ocsf_export, ocsf_setup_error) = runtime.block_on(async { start_ocsf_export( args.otlp_ocsf_endpoint.as_deref().map(export_config), - args.otlp_ocsf_protocol, - headers.ocsf, + settings.ocsf, ) }); let otlp_layer_provider = otlp_provider.clone(); @@ -709,25 +714,41 @@ fn main() -> Result<()> { std::process::exit(exit_code?); } -/// The headers for the supervisor's own exports, from its launch bundle. -#[derive(Debug, Default, PartialEq)] -struct BundleOtlpHeaders<'a> { - traces: Option<&'a str>, - ocsf: Option<&'a str>, +/// Settings for the supervisor's own exports: each export's protocol, with +/// headers and TLS material from the launch bundle. +#[derive(Clone, Copy, Default)] +struct BundleExportSettings<'a> { + traces: ExportSettings<'a>, + ocsf: ExportSettings<'a>, } -fn bundle_otlp_headers( - auth: Option<&openshell_core::jwt::SupervisorAuthBundle>, -) -> BundleOtlpHeaders<'_> { - fn expose(headers: &Option) -> Option<&str> { - headers - .as_ref() - .map(openshell_core::jwt::SecretOtlpHeaders::expose_secret) +fn bundle_export_settings( + auth: Option<&SupervisorAuthBundle>, + traces_protocol: OtlpProtocol, + ocsf_protocol: OtlpProtocol, +) -> BundleExportSettings<'_> { + fn settings( + protocol: OtlpProtocol, + credentials: OtlpExportCredentials<'_>, + ) -> ExportSettings<'_> { + ExportSettings { + protocol, + headers: credentials.headers.map(SecretOtlpHeaders::expose_secret), + tls: credentials.tls, + } + } + BundleExportSettings { + traces: settings( + traces_protocol, + auth.map(SupervisorAuthBundle::traces_otlp) + .unwrap_or_default(), + ), + ocsf: settings( + ocsf_protocol, + auth.map(SupervisorAuthBundle::ocsf_otlp) + .unwrap_or_default(), + ), } - auth.map_or_else(BundleOtlpHeaders::default, |auth| BundleOtlpHeaders { - traces: expose(&auth.traces_otlp_headers), - ocsf: expose(&auth.ocsf_otlp_headers), - }) } /// Start OCSF event export to an OTLP collector. @@ -736,8 +757,7 @@ fn bundle_otlp_headers( /// once its tracing subscriber is installed. fn start_ocsf_export( config: Option>, - protocol: OtlpProtocol, - headers: Option<&str>, + settings: ExportSettings<'_>, ) -> ( Option, Option, @@ -745,7 +765,7 @@ fn start_ocsf_export( let Some(config) = config else { return (None, None); }; - let started = target(&config, protocol, headers).and_then(|target| { + let started = target(&config, settings).and_then(|target| { Ok(openshell_otel::OcsfOtlp::start( target.log_exporter()?, target.resource().clone(), @@ -762,31 +782,44 @@ fn start_ocsf_export( /// tracing subscriber is installed. fn start_trace_export( config: Option>, - protocol: OtlpProtocol, - headers: Option<&str>, + settings: ExportSettings<'_>, ) -> ( Option, Option, ) { - match config.map(|config| target(&config, protocol, headers)?.tracer_provider()) { + match config.map(|config| target(&config, settings)?.tracer_provider()) { None => (None, None), Some(Ok(provider)) => (Some(provider), None), Some(Err(error)) => (None, Some(error)), } } +/// How one supervisor export reaches its collector. +#[derive(Clone, Copy, Default)] +struct ExportSettings<'a> { + protocol: OtlpProtocol, + /// Headers in the `OTEL_EXPORTER_OTLP_HEADERS` format. + headers: Option<&'a str>, + tls: Option<&'a OtlpTlsMaterial>, +} + fn target( config: &openshell_otel::OtlpExportConfig<'_>, - protocol: OtlpProtocol, - headers: Option<&str>, + settings: ExportSettings<'_>, ) -> Result { - Ok(openshell_otel::OtlpTarget::new(config)? - .with_protocol(protocol.into()) - .with_headers( - headers - .map(openshell_otel::parse_otlp_headers) - .unwrap_or_default(), - )) + let transport = openshell_otel::CollectorTransport { + protocol: settings.protocol.into(), + headers: settings + .headers + .map(openshell_otel::parse_otlp_headers) + .unwrap_or_default(), + tls: settings + .tls + .map(OtlpTlsMaterial::otlp_tls) + .transpose()? + .unwrap_or_default(), + }; + Ok(openshell_otel::OtlpTarget::new(config)?.with_transport(&transport)) } /// Resource attributes identifying this supervisor's sandbox and the gateway @@ -1054,35 +1087,45 @@ mod tests { } #[tokio::test(flavor = "multi_thread")] - async fn supervisor_signals_export_with_their_protocol_and_headers() { + async fn supervisor_signals_export_with_their_protocol_headers_and_tls() { test_environment::Environment::new() .remove("OTEL_EXPORTER_OTLP_HEADERS") .remove("OTEL_EXPORTER_OTLP_TRACES_HEADERS") .remove("OTEL_EXPORTER_OTLP_LOGS_HEADERS") .run_async(async { + use openshell_core::jwt::SecretPrivateKeyPem; use openshell_ocsf::{ConfigStateChangeBuilder, ocsf_emit}; + use openshell_otel_test_support::{OtlpHttpTestServer, ServerTls, TestPki}; use opentelemetry::trace::{Tracer as _, TracerProvider as _}; use tracing_subscriber::prelude::*; - let mut collector = openshell_otel_test_support::OtlpHttpTestServer::start().await; + let pki = TestPki::generate(); + let mut collector = + OtlpHttpTestServer::start_with_tls(&ServerTls::mutual(&pki)).await; + let tls = OtlpTlsMaterial { + ca_pem: Some(pki.ca_pem.clone()), + client: Some(openshell_core::jwt::OtlpClientIdentity { + cert_pem: pki.client.cert_pem.clone(), + key_pem: SecretPrivateKeyPem::new(pki.client.key_pem.clone()), + }), + }; + let settings = |headers| ExportSettings { + protocol: OtlpProtocol::HttpProtobuf, + headers: Some(headers), + tls: Some(&tls), + }; let config = || openshell_otel::OtlpExportConfig { endpoint: collector.endpoint(), service_name: openshell_otel::ServiceName::Fixed(SERVICE_NAME), service_version: None, resource_attributes: Vec::new(), }; - let (provider, error) = start_trace_export( - Some(config()), - OtlpProtocol::HttpProtobuf, - Some("authorization=traces"), - ); + let (provider, error) = + start_trace_export(Some(config()), settings("authorization=traces")); assert!(error.is_none(), "unexpected setup error: {error:?}"); let provider = provider.expect("trace export"); - let (ocsf, error) = start_ocsf_export( - Some(config()), - OtlpProtocol::HttpProtobuf, - Some("authorization=logs"), - ); + let (ocsf, error) = + start_ocsf_export(Some(config()), settings("authorization=logs")); assert!(error.is_none(), "unexpected setup error: {error:?}"); let ocsf = ocsf.expect("OCSF export"); @@ -1119,12 +1162,16 @@ mod tests { } #[test] - fn each_export_takes_its_own_headers_from_the_launch_bundle() { + fn each_export_takes_its_own_settings_from_the_launch_bundle() { use openshell_core::jwt::{ CredentialEpoch, SecretJwt, SecretOtlpHeaders, SessionRotation, SupervisorAuthBundle, }; let secret = |value: &str| Some(SecretOtlpHeaders::new(value.to_string())); + let tls = |ca: &str| OtlpTlsMaterial { + ca_pem: Some(ca.to_string()), + ..OtlpTlsMaterial::default() + }; let bundle = SupervisorAuthBundle { session_id: openshell_core::SandboxSessionId::new(), runtime_generation: openshell_core::sandbox_generation::SandboxGenerationId::parse( @@ -1141,16 +1188,43 @@ mod tests { traces_otlp_headers: secret("lane=traces"), ocsf_otlp_headers: secret("lane=ocsf"), agent_otlp_headers: secret("lane=agent"), + otlp_tls: vec![tls("ocsf-ca"), tls("traces-ca")], + traces_otlp_tls: Some(1), + ocsf_otlp_tls: Some(0), + agent_otlp_tls: None, }; + let summary = |settings: ExportSettings<'_>| { + ( + settings.protocol, + settings.headers.map(str::to_string), + settings.tls.and_then(|tls| tls.ca_pem.clone()), + ) + }; + let settings = bundle_export_settings( + Some(&bundle), + OtlpProtocol::HttpProtobuf, + OtlpProtocol::Grpc, + ); assert_eq!( - bundle_otlp_headers(Some(&bundle)), - BundleOtlpHeaders { - traces: Some("lane=traces"), - ocsf: Some("lane=ocsf"), - } + summary(settings.traces), + ( + OtlpProtocol::HttpProtobuf, + Some("lane=traces".to_string()), + Some("traces-ca".to_string()) + ) + ); + assert_eq!( + summary(settings.ocsf), + ( + OtlpProtocol::Grpc, + Some("lane=ocsf".to_string()), + Some("ocsf-ca".to_string()) + ) ); - assert_eq!(bundle_otlp_headers(None), BundleOtlpHeaders::default()); + let settings = bundle_export_settings(None, OtlpProtocol::Grpc, OtlpProtocol::Grpc); + assert_eq!(summary(settings.traces), (OtlpProtocol::Grpc, None, None)); + assert_eq!(summary(settings.ocsf), (OtlpProtocol::Grpc, None, None)); } #[tokio::test] @@ -1161,18 +1235,20 @@ mod tests { service_version: None, resource_attributes: Vec::new(), }; - let (export, error) = start_ocsf_export(None, OtlpProtocol::Grpc, None); + let (export, error) = start_ocsf_export(None, ExportSettings::default()); assert!(export.is_none() && error.is_none()); let (export, error) = - start_ocsf_export(Some(config("not a url")), OtlpProtocol::Grpc, None); + start_ocsf_export(Some(config("not a url")), ExportSettings::default()); assert!(export.is_none()); assert!(error.is_some()); let (export, error) = start_ocsf_export( Some(config("http://127.0.0.1:4317")), - OtlpProtocol::Grpc, - Some("authorization=Bearer%20t"), + ExportSettings { + headers: Some("authorization=Bearer%20t"), + ..ExportSettings::default() + }, ); assert!(error.is_none(), "unexpected setup error: {error:?}"); export.expect("OCSF export").shutdown().await; diff --git a/crates/openshell-supervisor/src/otlp_relay/exporter.rs b/crates/openshell-supervisor/src/otlp_relay/exporter.rs index 531be544a3..ec29acdfb3 100644 --- a/crates/openshell-supervisor/src/otlp_relay/exporter.rs +++ b/crates/openshell-supervisor/src/otlp_relay/exporter.rs @@ -68,12 +68,14 @@ impl std::fmt::Display for ExportError { impl std::error::Error for ExportError {} -/// The collector endpoint could not be turned into an export target. +/// The collector transport could not be set up. #[derive(Debug)] pub enum EndpointError { Grpc(tonic::transport::Error), Http(http::Error), - Runtime(openshell_otel::SetupError), + /// The HTTP client could not be built, such as without a runtime or + /// trust roots. + Setup(openshell_otel::SetupError), } impl std::fmt::Display for EndpointError { @@ -81,7 +83,7 @@ impl std::fmt::Display for EndpointError { match self { Self::Grpc(error) => error.fmt(f), Self::Http(error) => error.fmt(f), - Self::Runtime(error) => error.fmt(f), + Self::Setup(error) => error.fmt(f), } } } @@ -111,15 +113,16 @@ impl Exporter { endpoint: &str, protocol: OtlpProtocol, headers: http::HeaderMap, + tls: Option<&openshell_otel::OtlpTls>, ) -> Result { let transport = match protocol { OtlpProtocol::Grpc => Transport::Grpc(Grpc::new( - grpc_channel(endpoint).map_err(EndpointError::Grpc)?, + grpc_channel(endpoint, tls).map_err(EndpointError::Grpc)?, )), OtlpProtocol::HttpProtobuf => Transport::Http { client: Box::new( - OtlpHttpClient::new(EXPORT_TIMEOUT + CLIENT_TIMEOUT_MARGIN) - .map_err(EndpointError::Runtime)?, + OtlpHttpClient::new(tls, EXPORT_TIMEOUT + CLIENT_TIMEOUT_MARGIN) + .map_err(EndpointError::Setup)?, ), uri: openshell_otel::signal_url(endpoint, "v1/traces") .map_err(EndpointError::Http)?, @@ -154,7 +157,10 @@ impl Exporter { } } -fn grpc_channel(endpoint: &str) -> Result { +fn grpc_channel( + endpoint: &str, + tls: Option<&openshell_otel::OtlpTls>, +) -> Result { // Liveness detection for a collector that accepted the TCP connection // and then went silent or vanished without a FIN, so the lazy channel // redials instead of timing out every batch on a dead connection for @@ -168,7 +174,11 @@ fn grpc_channel(endpoint: &str) -> Result { // that recipe connects eagerly, pings idle connections, and carries a // first-party TLS trust policy, none of which fit a third-party // collector that must not gate supervisor startup. - Ok(Endpoint::from_shared(endpoint.to_string())? + let mut channel = Endpoint::from_shared(endpoint.to_string())?; + if let Some(tls) = tls { + channel = channel.tls_config(tls.grpc_config())?; + } + Ok(channel .connect_timeout(CONNECT_TIMEOUT) .tcp_keepalive(Some(TCP_KEEPALIVE_IDLE)) .tcp_keepalive_interval(Some(TCP_KEEPALIVE_INTERVAL)) @@ -333,12 +343,21 @@ mod tests { #[test] fn new_rejects_an_invalid_uri() { - assert!(Exporter::new("not a uri", OtlpProtocol::Grpc, http::HeaderMap::new()).is_err()); + assert!( + Exporter::new( + "not a uri", + OtlpProtocol::Grpc, + http::HeaderMap::new(), + None + ) + .is_err() + ); assert!( Exporter::new( "not a uri", OtlpProtocol::HttpProtobuf, - http::HeaderMap::new() + http::HeaderMap::new(), + None ) .is_err() ); @@ -357,6 +376,7 @@ mod tests { &endpoint, OtlpProtocol::HttpProtobuf, http::HeaderMap::new(), + None, ) .unwrap(); let batch = Bytes::from(encoded_request("over-http", &[])); @@ -384,6 +404,7 @@ mod tests { collector.endpoint(), OtlpProtocol::HttpProtobuf, http::HeaderMap::new(), + None, ) .unwrap(); let error = exporter @@ -405,6 +426,7 @@ mod tests { &endpoint, OtlpProtocol::HttpProtobuf, http::HeaderMap::new(), + None, ) .unwrap(); @@ -435,6 +457,7 @@ mod tests { &endpoint, OtlpProtocol::HttpProtobuf, http::HeaderMap::new(), + None, ) .unwrap(); @@ -493,7 +516,7 @@ mod tests { let batch = || Bytes::from(encoded_request("with-headers", &[])); let grpc = OtlpTestServer::start().await; - Exporter::new(grpc.endpoint(), OtlpProtocol::Grpc, headers()) + Exporter::new(grpc.endpoint(), OtlpProtocol::Grpc, headers(), None) .unwrap() .export(batch()) .await @@ -502,7 +525,7 @@ mod tests { assert_eq!(grpc.shutdown().await.trace_authorizations, ["Bearer agent"]); let mut http = OtlpHttpTestServer::start().await; - Exporter::new(http.endpoint(), OtlpProtocol::HttpProtobuf, headers()) + Exporter::new(http.endpoint(), OtlpProtocol::HttpProtobuf, headers(), None) .unwrap() .export(batch()) .await @@ -513,6 +536,48 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn exports_verify_the_collector_and_present_a_client_certificate() { + use openshell_otel_test_support::{ServerTls, TestPki}; + + let pki = TestPki::generate(); + let tls = openshell_otel::OtlpTls::from_pem( + Some(pki.ca_pem.as_bytes()), + Some(( + pki.client.cert_pem.as_bytes(), + pki.client.key_pem.as_bytes(), + )), + ) + .unwrap(); + let batch = || Bytes::from(encoded_request("over-tls", &[])); + + let grpc = OtlpTestServer::start_with_tls(&ServerTls::mutual(&pki)).await; + Exporter::new( + grpc.endpoint(), + OtlpProtocol::Grpc, + http::HeaderMap::new(), + Some(&tls), + ) + .unwrap() + .export(batch()) + .await + .expect("gRPC export over mutual TLS succeeds"); + assert_eq!(grpc.shutdown().await.spans.len(), 1); + + let mut http = OtlpHttpTestServer::start_with_tls(&ServerTls::mutual(&pki)).await; + Exporter::new( + http.endpoint(), + OtlpProtocol::HttpProtobuf, + http::HeaderMap::new(), + Some(&tls), + ) + .unwrap() + .export(batch()) + .await + .expect("HTTP export over mutual TLS succeeds"); + assert_eq!(http.next_export().await.path, "/v1/traces"); + } + #[tokio::test] async fn export_reaches_the_collector_with_attributes_intact() { let collector = OtlpTestServer::start().await; @@ -520,6 +585,7 @@ mod tests { collector.endpoint(), OtlpProtocol::Grpc, http::HeaderMap::new(), + None, ) .unwrap(); let batch = Bytes::from(encoded_request( @@ -552,7 +618,7 @@ mod tests { let endpoint = format!("http://{}", listener.local_addr().unwrap()); drop(listener); let exporter = - Exporter::new(&endpoint, OtlpProtocol::Grpc, http::HeaderMap::new()).unwrap(); + Exporter::new(&endpoint, OtlpProtocol::Grpc, http::HeaderMap::new(), None).unwrap(); let started = Instant::now(); let error = exporter @@ -575,6 +641,7 @@ mod tests { &collector.endpoint, OtlpProtocol::Grpc, http::HeaderMap::new(), + None, ) .unwrap(); let started = Instant::now(); diff --git a/crates/openshell-supervisor/src/otlp_relay/mod.rs b/crates/openshell-supervisor/src/otlp_relay/mod.rs index 91cd4d4107..4b45cda49c 100644 --- a/crates/openshell-supervisor/src/otlp_relay/mod.rs +++ b/crates/openshell-supervisor/src/otlp_relay/mod.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::{Duration, Instant}; -use openshell_core::jwt::SecretOtlpHeaders; +use openshell_core::jwt::{OtlpTlsMaterial, SecretOtlpHeaders}; use openshell_core::sandbox_env::OtlpProtocol; use openshell_isolation_interface::contract::BoundaryDuplexStream; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; @@ -230,6 +230,8 @@ pub struct OtlpRelayConfig { pub protocol: OtlpProtocol, /// Headers sent with every export, from the launch bundle. pub headers: Option, + /// TLS material for an `https://` collector, from the launch bundle. + pub tls: Option, /// Sandbox identifier attached to every forwarded resource. pub sandbox_id: String, } @@ -240,23 +242,26 @@ pub enum StartError { /// The collector endpoint is empty or not a URI, judged the same way as /// for the supervisor's own exporter. InvalidEndpoint(openshell_otel::SetupError), - /// The collector endpoint is a URI the exporter does not accept. + /// The exporter could not be set up for the endpoint. Transport(exporter::EndpointError), - /// The collector endpoint uses a scheme the relay does not speak. The - /// relay forwards over plaintext only, so an `https` endpoint keeps - /// it inactive rather than downgrading; the supervisor's own exporter - /// does speak TLS. TLS on the relayed hop is a tracked follow-up. + /// The collector endpoint uses a scheme other than `http` or `https`. UnsupportedScheme(String), + /// The TLS material from the launch bundle is unusable. + Tls(openshell_otel::TlsError), + /// TLS material was given for an `http` endpoint. + TlsWithoutHttps, } impl std::fmt::Display for StartError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::InvalidEndpoint(error) => write!(f, "invalid collector endpoint: {error}"), - Self::Transport(error) => write!(f, "invalid collector endpoint: {error}"), + Self::Transport(error) => write!(f, "collector transport could not be set up: {error}"), + Self::Tls(error) => write!(f, "unusable collector TLS material: {error}"), + Self::TlsWithoutHttps => f.write_str("collector TLS material needs an https endpoint"), Self::UnsupportedScheme(scheme) => write!( f, - "unsupported collector endpoint scheme {scheme:?}: the relay speaks plaintext OTLP (http) only" + "unsupported collector endpoint scheme {scheme:?}: use http or https" ), } } @@ -319,17 +324,32 @@ fn relay_addr() -> SocketAddr { /// Starts the relay for `config`. Must be called inside a tokio runtime. /// /// Returns the relay handle and the reserved destination to install in the -/// network proxy. Fails only when the endpoint is not a valid plaintext URI; -/// an unreachable collector is handled lazily by the exporter. Surrounding +/// network proxy. Fails only when the endpoint is not a valid `http` or +/// `https` URI or the TLS material is unusable; an unreachable collector is +/// handled lazily by the exporter. Surrounding /// whitespace is trimmed, as for the supervisor's own exporter. pub fn start(config: OtlpRelayConfig) -> Result<(OtlpRelay, ReservedDestination), StartError> { let (endpoint, uri) = openshell_otel::validated_endpoint(&config.endpoint) .map_err(StartError::InvalidEndpoint)?; if let Some(scheme) = uri.scheme_str() && !scheme.eq_ignore_ascii_case("http") + && !scheme.eq_ignore_ascii_case("https") { return Err(StartError::UnsupportedScheme(scheme.to_string())); } + let tls = config + .tls + .as_ref() + .map(OtlpTlsMaterial::otlp_tls) + .transpose() + .map_err(StartError::Tls)?; + let https = uri + .scheme_str() + .is_some_and(|scheme| scheme.eq_ignore_ascii_case("https")); + if !https && tls.is_some() { + return Err(StartError::TlsWithoutHttps); + } + let tls = https.then(|| tls.unwrap_or_default()); let exporter = Exporter::new( endpoint, config.protocol, @@ -338,6 +358,7 @@ pub fn start(config: OtlpRelayConfig) -> Result<(OtlpRelay, ReservedDestination) .as_ref() .map(|headers| openshell_otel::parse_otlp_headers(headers.expose_secret())) .unwrap_or_default(), + tls.as_ref(), ) .map_err(StartError::Transport)?; if config.sandbox_id.len() > enrichment::MAX_SANDBOX_ID_BYTES { @@ -375,6 +396,7 @@ pub fn activate( endpoint: Option, protocol: OtlpProtocol, headers: Option, + tls: Option, sandbox_id: Option, ) -> (Option, Option) { let Some(endpoint) = endpoint else { @@ -404,6 +426,7 @@ pub fn activate( endpoint: endpoint.clone(), protocol, headers, + tls, sandbox_id, }) { Ok((relay, reserved)) => (Some(relay), Some(reserved)), @@ -741,7 +764,8 @@ mod tests { fn gated(endpoint: &str) -> (OtlpRelay, ReservedDestination, watch::Sender) { let (gate_tx, gate_rx) = watch::channel(false); - let exporter = Exporter::new(endpoint, OtlpProtocol::Grpc, http::HeaderMap::new()).unwrap(); + let exporter = + Exporter::new(endpoint, OtlpProtocol::Grpc, http::HeaderMap::new(), None).unwrap(); let (relay, reserved) = assemble(exporter, "sb-test".into(), Some(gate_rx)); (relay, reserved, gate_tx) } @@ -771,6 +795,7 @@ mod tests { headers: Some(SecretOtlpHeaders::new( "x-mlflow-experiment-id=7".to_string(), )), + tls: None, sandbox_id: "sb-http".into(), }) .unwrap(); @@ -814,6 +839,7 @@ mod tests { endpoint: collector.endpoint().to_string(), protocol: OtlpProtocol::Grpc, headers: None, + tls: None, sandbox_id: "sb-test".into(), }) .unwrap(); @@ -884,6 +910,7 @@ mod tests { endpoint: "not a uri".into(), protocol: OtlpProtocol::Grpc, headers: None, + tls: None, sandbox_id: "sb-test".into(), }) .err() @@ -894,6 +921,7 @@ mod tests { endpoint: blank.into(), protocol: OtlpProtocol::Grpc, headers: None, + tls: None, sandbox_id: "sb-test".into(), }) .err() @@ -919,6 +947,7 @@ mod tests { endpoint: format!(" {}\n", collector.endpoint()), protocol: OtlpProtocol::Grpc, headers: None, + tls: None, sandbox_id: "sb-test".into(), }) .expect("a padded endpoint starts the relay"); @@ -934,7 +963,8 @@ mod tests { let before = tokio::runtime::Handle::current() .metrics() .num_alive_tasks(); - let (relay, reserved) = activate(None, OtlpProtocol::Grpc, None, Some("sb-test".into())); + let (relay, reserved) = + activate(None, OtlpProtocol::Grpc, None, None, Some("sb-test".into())); assert!(relay.is_none()); assert!(reserved.is_none()); let after = tokio::runtime::Handle::current() @@ -957,6 +987,7 @@ mod tests { OtlpProtocol::Grpc, None, None, + None, ); assert!(relay.is_none()); assert!(reserved.is_none()); @@ -967,8 +998,81 @@ mod tests { drop(collector.shutdown().await); } + #[tokio::test(flavor = "multi_thread")] + async fn relays_to_an_https_collector_with_its_tls_material() { + use openshell_core::jwt::SecretPrivateKeyPem; + use openshell_otel_test_support::{ServerTls, TestPki}; + + let pki = TestPki::generate(); + let collector = OtlpTestServer::start_with_tls(&ServerTls::mutual(&pki)).await; + let (relay, reserved) = start(OtlpRelayConfig { + endpoint: collector.endpoint().to_string(), + protocol: OtlpProtocol::Grpc, + headers: None, + tls: Some(OtlpTlsMaterial { + ca_pem: Some(pki.ca_pem.clone()), + client: Some(openshell_core::jwt::OtlpClientIdentity { + cert_pem: pki.client.cert_pem.clone(), + key_pem: SecretPrivateKeyPem::new(pki.client.key_pem.clone()), + }), + }), + sandbox_id: "sb-tls".into(), + }) + .expect("an https collector starts the relay"); + let (mut client, boundary) = tokio::io::duplex(64 * 1024); + tokio::spawn(reserved.handler.serve(Box::new(boundary)).unwrap()); + + let body = encoded_request("tls-span", &[]); + client + .write_all(&request("POST", "/v1/traces", PROTOBUF, &body)) + .await + .unwrap(); + assert_eq!(read_head(&mut client).await.status, 200); + relay.flush(FLUSH_TIMEOUT).await; + let received = collector.shutdown().await; + assert_eq!(received.spans.len(), 1); + } + + #[tokio::test(flavor = "multi_thread")] + async fn relays_to_an_https_collector_trusted_by_the_platform_roots() { + let roots = crate::test_environment::fixture_dir().unwrap(); + let ca_file = roots.path().join("ca.pem"); + crate::test_environment::Environment::new() + .set("SSL_CERT_FILE", &ca_file) + .set("SSL_CERT_DIR", roots.path()) + .run_async(async { + use openshell_otel_test_support::{ServerTls, TestPki}; + + let pki = TestPki::generate(); + std::fs::write(&ca_file, &pki.ca_pem).unwrap(); + let collector = OtlpTestServer::start_with_tls(&ServerTls::server_only(&pki)).await; + let (relay, reserved) = start(OtlpRelayConfig { + endpoint: collector.endpoint().to_string(), + protocol: OtlpProtocol::Grpc, + headers: None, + tls: None, + sandbox_id: "sb-roots".into(), + }) + .expect("an https collector starts the relay"); + let (mut client, boundary) = tokio::io::duplex(64 * 1024); + tokio::spawn(reserved.handler.serve(Box::new(boundary)).unwrap()); + + let body = encoded_request("roots-span", &[]); + client + .write_all(&request("POST", "/v1/traces", PROTOBUF, &body)) + .await + .unwrap(); + assert_eq!(read_head(&mut client).await.status, 200); + relay.flush(FLUSH_TIMEOUT).await; + assert_eq!(collector.shutdown().await.spans.len(), 1); + }) + .await; + } + #[test] - fn https_endpoint_is_refused_at_start() { + fn unusable_tls_material_or_scheme_leaves_the_relay_inactive() { + use openshell_otel_test_support::TestPki; + let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -978,24 +1082,40 @@ mod tests { endpoint: "https://collector.example:4317".into(), protocol: OtlpProtocol::Grpc, headers: None, + tls: Some(OtlpTlsMaterial { + ca_pem: Some("not pem".into()), + ..OtlpTlsMaterial::default() + }), sandbox_id: "sb-test".into(), }) .err() - .expect("https is not supported on the relay hop"); - assert!( - matches!(error, StartError::UnsupportedScheme(ref scheme) if scheme == "https"), - "{error}" - ); - // Lower-case comparison, and plain http still starts. - assert!( - start(OtlpRelayConfig { - endpoint: "HTTP://collector.example:4317".into(), - protocol: OtlpProtocol::Grpc, - headers: None, - sandbox_id: "sb-test".into(), - }) - .is_ok() - ); + .expect("the CA is unusable"); + assert!(matches!(error, StartError::Tls(_)), "{error}"); + + let error = start(OtlpRelayConfig { + endpoint: "http://collector.example:4317".into(), + protocol: OtlpProtocol::Grpc, + headers: None, + tls: Some(OtlpTlsMaterial { + ca_pem: Some(TestPki::generate().ca_pem), + ..OtlpTlsMaterial::default() + }), + sandbox_id: "sb-test".into(), + }) + .err() + .expect("TLS material needs an https endpoint"); + assert!(matches!(error, StartError::TlsWithoutHttps), "{error}"); + + let error = start(OtlpRelayConfig { + endpoint: "ftp://collector.example:4317".into(), + protocol: OtlpProtocol::Grpc, + headers: None, + tls: None, + sandbox_id: "sb-test".into(), + }) + .err() + .expect("only http and https are spoken"); + assert!(matches!(error, StartError::UnsupportedScheme(ref scheme) if scheme == "ftp")); } #[tokio::test] diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 7bf6046e17..b7d87e14ed 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -615,7 +615,7 @@ endpoint = "http://traces.observability.svc:4318" protocol = "http/protobuf" ``` -Each enabled signal needs an endpoint, either in its own table or from `endpoint`. A table without its own `endpoint` uses the whole `[openshell.gateway.otlp]` collector, including its `protocol` and headers. A table with its own `endpoint` takes nothing else from `[openshell.gateway.otlp]`, so configured settings for one collector, such as its credentials, never reach another, and setting `protocol`, `headers`, or `headers_file` in a table without an `endpoint` is a configuration error. Endpoints must be valid URIs without a query string: exporters send no query parameters, so an endpoint with one is a configuration error, and a collector that needs parameters must take them as headers. If an endpoint is malformed, the gateway logs the configuration error and continues with all OTLP export disabled. It does not connect at startup: an unreachable collector produces export failures, never a failure to serve. +Each enabled signal needs an endpoint, either in its own table or from `endpoint`. A table without its own `endpoint` uses the whole `[openshell.gateway.otlp]` collector, including its `protocol`, headers, and TLS files. A table with its own `endpoint` takes nothing else from `[openshell.gateway.otlp]`, so configured settings for one collector, such as its credentials, never reach another, and setting any of them in a table without an `endpoint` is a configuration error. Endpoints must be valid URIs without a query string: exporters send no query parameters, so an endpoint with one is a configuration error, and a collector that needs parameters must take them as headers. If an endpoint is malformed, the gateway logs the configuration error and continues with all OTLP export disabled. It does not connect at startup: an unreachable collector produces export failures, never a failure to serve. `protocol` selects the transport to a collector: `grpc`, the default, or `http/protobuf`. With `http/protobuf`, the endpoint is a base URL and each signal is posted to its path under it, `/v1/traces` or `/v1/logs`, as with `OTEL_EXPORTER_OTLP_ENDPOINT`. HTTP/JSON is not supported. Point a `grpc` endpoint at a collector's gRPC receiver, conventionally port `4317`, and an `http/protobuf` endpoint at its HTTP receiver, conventionally port `4318`. A URI alone cannot distinguish the two, so a mismatched endpoint is accepted at startup and then fails on export. Sandbox supervisors and standalone compute drivers must come from the same release as the gateway to use `http/protobuf`: an older supervisor exports over gRPC, and an older driver rejects the protocol setting. @@ -625,7 +625,7 @@ Put credentials in `headers_file` rather than `headers`. It reads a collector's Supervisors receive the gateway's headers but export to the endpoints their compute driver gives them. An external compute driver takes those endpoints from its own flags, so give it the same endpoints as the gateway, or supervisors send the gateway's credentials to a different collector. Supervisors and external compute drivers from an earlier release than the gateway reject launch credentials that carry trace or agent relay headers, so when the trace or agent collector sets `headers` or `headers_file`, upgrade them with or before the gateway. A gateway rolled back to an earlier release cannot read the stored launch credentials of sandboxes created with those headers, so delete and recreate those sandboxes after a rollback. -Headers travel in cleartext unless the endpoint uses `https://`, which only `grpc` supports. The agent trace relay and `http/protobuf` collectors always receive them in cleartext, so do not send credentials to them over an untrusted network. +Headers travel in cleartext unless the endpoint uses `https://`, so use TLS when they carry credentials over an untrusted network. ```toml [openshell.gateway.otlp] @@ -638,7 +638,52 @@ protocol = "http/protobuf" headers = { x-mlflow-experiment-id = "0" } ``` -Use an `https://` endpoint for TLS over `grpc`. The gateway verifies the collector certificate against the system trust roots. An `https://` endpoint with `http/protobuf` is a configuration error. +Use an `https://` endpoint for TLS over either protocol. The gateway verifies the collector certificate against the system trust roots, or only against the PEM certificates in `ca_file` when it is set. Set `cert_file` and `key_file` together to present a PEM client certificate to a collector that requires one. The gateway reads these files at startup and fails to start if one is unreadable, larger than 64 KiB, or holds no usable PEM, if the key does not match the certificate, or if one is set for an `http://` endpoint. A relative path uses the gateway's working directory. Supervisors receive each distinct set of files once, however many tables share it, and the files they receive may total at most 192 KiB. A shared supervisor's launch credentials, these files and headers included, may total at most 256 KiB, so large headers alongside large TLS files can still stop sandboxes from launching on one. Compute drivers and supervisors use the same CA and client certificate for their own exports, and the agent trace relay uses its collector's. Supervisors receive the file contents with their launch credentials, never through the sandbox environment or file system, so the collector cannot tell a supervisor's exports from the gateway's by certificate. The private key is stored with each sandbox's launch credentials wherever its compute driver keeps them, such as a Kubernetes Secret. The VM driver receives the trace collector's file paths. A standalone compute driver reads its own with `--otlp-ca-file`, `--otlp-cert-file`, and `--otlp-key-file`. + +The files are read once. To rotate a certificate, restart the gateway; running sandboxes keep the previous certificate until they restart. Supervisors and external compute drivers from an earlier release than the gateway reject launch credentials that carry TLS material, and an earlier VM driver rejects the TLS file flags, so when a collector sets TLS files, upgrade them with or before the gateway. A gateway rolled back to an earlier release cannot read the stored launch credentials of sandboxes created with TLS material, so delete and recreate those sandboxes after a rollback. + +```toml +[openshell.gateway.otlp] +endpoint = "https://otel-collector.observability.svc:4317" +ca_file = "/etc/openshell/otlp/ca.pem" +cert_file = "/etc/openshell/otlp/client.pem" +key_file = "/etc/openshell/otlp/client-key.pem" +``` + +Each collector has its own TLS files. To trust a different CA for agent traces than for OpenShell's own spans and OCSF events, give `[openshell.gateway.otlp.agent]` its own `endpoint` and `ca_file`. Supervisors then receive both CAs and use each for its own collector: + +```toml +[openshell.gateway.otlp] +endpoint = "https://otel-collector.observability.svc:4317" +ca_file = "/etc/openshell/otlp/infra-ca.pem" + +[openshell.gateway.otlp.agent] +enabled = true +endpoint = "https://mlflow.mlops.svc:5000" +protocol = "http/protobuf" +ca_file = "/etc/openshell/otlp/mlflow-ca.pem" +``` + +For Helm deployments, store the files in a Secret, mount it with `server.extraVolumes` and `server.extraVolumeMounts`, and set the table's keys to the mounted paths through `gatewayConfig`. This example mounts a Secret named `otlp-tls` with the keys `ca.pem`, `client.pem`, and `client-key.pem`: + +```yaml +server: + otlp: + endpoint: https://otel-collector.observability.svc:4317 + extraVolumes: + - name: otlp-tls + secret: + secretName: otlp-tls + extraVolumeMounts: + - name: otlp-tls + mountPath: /etc/openshell/otlp + readOnly: true +gatewayConfig: + openshell.gateway.otlp: + ca_file: /etc/openshell/otlp/ca.pem + cert_file: /etc/openshell/otlp/client.pem + key_file: /etc/openshell/otlp/client-key.pem +``` The OpenTelemetry SDK logs export failures after startup. Spans in a failed batch are dropped rather than retried. @@ -730,11 +775,9 @@ Agents must export traces as OTLP/HTTP protobuf with compression disabled. The r Delivery is best-effort. The relay answers 200 once a batch is buffered and makes one export attempt per batch, so a batch is lost if the collector is unavailable at that moment. The supervisor logs the first failed export and a periodic summary of failures. Before the sandbox reports that its main process exited, and before a suspended sandbox's supervisor exits, the relay flushes buffered batches for up to 5 seconds, and skips that wait when its most recent export failed. It keeps serving `openshell sandbox exec` sessions after the main process exits and flushes once more when the sandbox is torn down. -The relayed hop does not yet support TLS to the collector, so headers, including credentials, travel in cleartext; an `https` endpoint leaves the relay inactive with a logged reason. Protect the hop at the network layer, for example with a collector-side ingress policy. - ### Tuning -`[openshell.gateway.otlp]` decides whether and where to export. How the SDK exports is controlled by the standard OpenTelemetry environment variables, which the gateway reads through the SDK rather than mirroring as TOML keys. They apply to the gateway and its in-process compute drivers. The VM driver also receives the trace headers, and sandbox supervisors receive headers as described for [headers](#otlp-export). No other variable reaches supervisors, which receive only the endpoints, protocols, and headers of their collectors. +`[openshell.gateway.otlp]` decides whether and where to export. How the SDK exports is controlled by the standard OpenTelemetry environment variables, which the gateway reads through the SDK rather than mirroring as TOML keys. They apply to the gateway and its in-process compute drivers. The VM driver also receives the trace headers, and sandbox supervisors receive headers as described for [headers](#otlp-export). No other variable reaches supervisors, which receive only the endpoints, protocols, [headers, and TLS material](#otlp-export) of their collectors. The gateway honors these variables: @@ -749,7 +792,7 @@ The gateway honors these variables: | `OTEL_EXPORTER_OTLP_COMPRESSION` | Set to `gzip` to compress exports of every signal. `zstd` is not supported. | | `OTEL_EXPORTER_OTLP_TRACES_HEADERS`, `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION`, `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT` | Exporter transport tuning for spans only. These override the general variables. | | `OTEL_EXPORTER_OTLP_LOGS_HEADERS`, `OTEL_EXPORTER_OTLP_LOGS_COMPRESSION`, `OTEL_EXPORTER_OTLP_LOGS_TIMEOUT` | Exporter transport tuning for OCSF log records only. These override the general variables. | -| `SSL_CERT_FILE`, `SSL_CERT_DIR` | Additional trust roots for `https://` endpoints, such as a private CA. | +| `SSL_CERT_FILE`, `SSL_CERT_DIR` | Replace the system trust roots for `https://` endpoints without a `ca_file`. They affect the gateway and the compute drivers it runs, not supervisors. | For Helm deployments, set these variables with `server.extraEnv`. Read credentials from a Secret so they stay out of the values file: @@ -773,7 +816,7 @@ These variables have no effect. Only `[openshell.gateway.otlp]` decides which si | `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | Enablement has one source, `[openshell.gateway.otlp]`, so an environment variable cannot silently turn export on or redirect it. | | `OTEL_SDK_DISABLED`, `OTEL_TRACES_EXPORTER`, `OTEL_LOGS_EXPORTER`, `OTEL_METRICS_EXPORTER` | `[openshell.gateway.otlp]` decides which signals are exported. To disable export, omit it. | | `OTEL_EXPORTER_OTLP_PROTOCOL`, `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL` | `protocol` in each table selects the transport, and supervisors follow the same setting. | -| `OTEL_EXPORTER_OTLP_INSECURE`, `OTEL_EXPORTER_OTLP_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_KEY`, and their per-signal forms | The endpoint scheme selects TLS, and collector certificates are verified against the system trust roots. Client certificates are not supported. | +| `OTEL_EXPORTER_OTLP_INSECURE`, `OTEL_EXPORTER_OTLP_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_KEY`, and their per-signal forms | The endpoint scheme selects TLS, and `ca_file`, `cert_file`, and `key_file` in each table set its trust and client certificate, so supervisors follow the same setting. | | `OTEL_PROPAGATORS` | The gateway always propagates W3C Trace Context. | | `OTEL_BLRP_SCHEDULE_DELAY`, `OTEL_BLRP_MAX_QUEUE_SIZE`, `OTEL_BLRP_MAX_EXPORT_BATCH_SIZE`, `OTEL_BLRP_EXPORT_TIMEOUT` | OCSF log export uses its own queue and batching, described in [OCSF Log Export](#ocsf-log-export). | | `OTEL_EXPORTER_OTLP_METRICS_*`, `OTEL_METRIC_EXPORT_INTERVAL` | The gateway serves metrics for Prometheus scraping and does not export them over OTLP. | diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index 22e9215626..20c13456fc 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -2085,13 +2085,17 @@ dependencies = [ "futures", "http", "http-body-util", + "hyper-rustls", "hyper-util", "metrics", + "openshell-crypto", "openshell-ocsf", "opentelemetry", "opentelemetry-http", "opentelemetry-otlp", "opentelemetry_sdk", + "rustls", + "rustls-native-certs", "serde_json", "thiserror", "tokio", @@ -2100,6 +2104,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "zeroize", ] [[package]] @@ -2111,10 +2116,14 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "openshell-crypto", "openshell-otel", "opentelemetry-proto", "prost", + "rcgen", + "rustls", "tokio", + "tokio-rustls", "tokio-stream", "tonic", "tracing", diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 7a4800aaee..1206d952e4 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2151,13 +2151,17 @@ dependencies = [ "futures", "http", "http-body-util", + "hyper-rustls", "hyper-util", "metrics", + "openshell-crypto", "openshell-ocsf", "opentelemetry", "opentelemetry-http", "opentelemetry-otlp", "opentelemetry_sdk", + "rustls", + "rustls-native-certs", "serde_json", "thiserror 2.0.20", "tokio", @@ -2166,6 +2170,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "zeroize", ] [[package]] @@ -2177,10 +2182,14 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "openshell-crypto", "openshell-otel", "opentelemetry-proto", "prost", + "rcgen", + "rustls", "tokio", + "tokio-rustls", "tokio-stream", "tonic", "tracing", @@ -4710,6 +4719,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie"