diff --git a/Cargo.lock b/Cargo.lock index 2a24331f2a..1964b5e421 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4431,6 +4431,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "chrono", + "clap", "glob", "h2", "ipnet", @@ -4857,13 +4858,21 @@ dependencies = [ name = "openshell-otel" version = "0.0.0" dependencies = [ + "async-trait", + "bytes", "futures", "http 1.4.0", + "http-body-util", + "hyper-util", "metrics", "openshell-ocsf", + "openshell-otel-test-support", "opentelemetry", + "opentelemetry-http", "opentelemetry-otlp", + "opentelemetry-proto", "opentelemetry_sdk", + "prost", "serde_json", "thiserror 2.0.20", "tokio", @@ -4878,6 +4887,11 @@ dependencies = [ name = "openshell-otel-test-support" version = "0.0.0" dependencies = [ + "bytes", + "http 1.4.0", + "http-body-util", + "hyper", + "hyper-util", "openshell-crypto", "openshell-otel", "opentelemetry-proto", @@ -5462,14 +5476,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry-http" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302249bd8ee804789faaa7b5b2cbf79bc61cb864714079c644e3828af5b7e811" +dependencies = [ + "async-trait", + "bytes", + "http 1.4.0", + "opentelemetry", +] + [[package]] name = "opentelemetry-otlp" version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0756af24841e367128ef31c831999937b8b5c5197d47914590d8f4e2e88c2e1a" dependencies = [ + "flate2", "http 1.4.0", + "httpdate", "opentelemetry", + "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", "prost", diff --git a/Cargo.toml b/Cargo.toml index 4816fc8ffe..5254c11284 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ tracing-appender = "0.2" opentelemetry-proto = { version = "0.33", default-features = false, features = ["gen-tonic", "trace"] } opentelemetry = "0.33" opentelemetry_sdk = { version = "0.33", features = ["rt-tokio"] } +opentelemetry-http = { version = "0.33", default-features = false } opentelemetry-otlp = { version = "0.33", default-features = false, features = ["grpc-tonic", "gzip-tonic", "tls-aws-lc", "tls-roots", "trace"] } tracing-opentelemetry = { version = "0.34", default-features = false, features = ["tracing-log"] } diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index c5402e5fff..aa83e10f86 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -41,6 +41,7 @@ reqwest = { workspace = true, features = ["blocking", "rustls-tls-native-roots-n tar = { version = "0.4", optional = true } tempfile = { version = "3", optional = true } openshell-otel = { path = "../openshell-otel", optional = true } +clap = { workspace = true, optional = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } @@ -58,8 +59,12 @@ telemetry = ["dep:reqwest", "dep:chrono", "reqwest?/blocking"] oauth = ["dep:reqwest"] ## Strict Ed25519 JWT issuance and verification for authenticated sandbox sessions. jwt = ["dep:jsonwebtoken", "dep:zeroize"] +## Conversions to `openshell-otel` types. +otel = ["dep:openshell-otel"] +## Command-line flags shared by compute drivers. +clap = ["dep:clap"] ## Propagate the active W3C trace context on supervisor gateway calls. -trace-context = ["dep:openshell-otel"] +trace-context = ["otel"] [build-dependencies] tonic-prost-build = { workspace = true } diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index f106a1e7ef..a287d3a36e 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -26,32 +26,227 @@ pub const SSH_SOCKET_PATH: &str = "OPENSHELL_SSH_SOCKET_PATH"; /// Log level for the sandbox supervisor (e.g. `"debug"`, `"info"`, `"warn"`). pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL"; -/// OTLP/gRPC collector endpoint for supervisor trace export. +/// OTLP collector endpoint for supervisor trace export. pub const OTLP_ENDPOINT: &str = "OPENSHELL_OTLP_ENDPOINT"; -/// OTLP/gRPC collector endpoint for supervisor OCSF event export. +/// OTLP transport for supervisor trace export. Absent means gRPC. +pub const OTLP_PROTOCOL: &str = "OPENSHELL_OTLP_PROTOCOL"; + +/// OTLP collector endpoint for supervisor OCSF event export. pub const OTLP_OCSF_ENDPOINT: &str = "OPENSHELL_OTLP_OCSF_ENDPOINT"; -/// OTLP/gRPC collector endpoint that supervisors relay workload traces to. +/// OTLP transport for supervisor OCSF event export. Absent means gRPC. +pub const OTLP_OCSF_PROTOCOL: &str = "OPENSHELL_OTLP_OCSF_PROTOCOL"; + +/// OTLP collector endpoint that supervisors relay workload traces to. /// /// Set only when the gateway enables `[openshell.gateway.otlp.agent]`; absent /// keeps the relay inactive and leaves the workload environment unchanged. pub const OTLP_AGENT_ENDPOINT: &str = "OPENSHELL_OTLP_AGENT_ENDPOINT"; +/// OTLP transport that supervisors relay workload traces over. Absent means +/// gRPC. +pub const OTLP_AGENT_PROTOCOL: &str = "OPENSHELL_OTLP_AGENT_PROTOCOL"; + /// Name of the gateway that launched the sandbox, reported as /// `openshell.gateway.name` on supervisor telemetry. pub const GATEWAY_NAME: &str = "OPENSHELL_GATEWAY_NAME"; -/// OCSF event export that a compute driver passes to its supervisors. +/// OTLP transport, named with the `OTEL_EXPORTER_OTLP_PROTOCOL` values. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub enum OtlpProtocol { + #[default] + #[serde(rename = "grpc")] + Grpc, + #[serde(rename = "http/protobuf")] + HttpProtobuf, +} + +impl OtlpProtocol { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Grpc => "grpc", + Self::HttpProtobuf => "http/protobuf", + } + } +} + +impl std::fmt::Display for OtlpProtocol { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for OtlpProtocol { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "grpc" => Ok(Self::Grpc), + "http/protobuf" => Ok(Self::HttpProtobuf), + other => Err(format!( + "unsupported OTLP protocol {other:?}: expected \"grpc\" or \"http/protobuf\"" + )), + } + } +} + +#[cfg(feature = "otel")] +impl From for openshell_otel::Protocol { + fn from(protocol: OtlpProtocol) -> Self { + match protocol { + OtlpProtocol::Grpc => Self::Grpc, + OtlpProtocol::HttpProtobuf => Self::HttpProtobuf, + } + } +} + +/// One OTLP collector that a supervisor exports to. #[derive(Clone, Debug, Eq, PartialEq)] -pub struct OcsfOtlpExport { +pub struct OtlpExport { pub endpoint: String, + pub protocol: OtlpProtocol, } -impl OcsfOtlpExport { - /// The supervisor environment variables for this export. +impl OtlpExport { + fn environment( + &self, + endpoint_var: &'static str, + protocol_var: &'static str, + ) -> impl Iterator { + std::iter::once((endpoint_var, self.endpoint.clone())).chain( + (self.protocol != OtlpProtocol::Grpc) + .then(|| (protocol_var, self.protocol.to_string())), + ) + } +} + +/// The OTLP exports that a compute driver passes to its supervisors. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct SupervisorOtlp { + /// The supervisor's own spans. + pub traces: Option, + /// OCSF events, as log records. + pub ocsf: Option, + /// Relayed workload traces. + pub agent: Option, +} + +impl SupervisorOtlp { + /// Whether no export is configured. + #[must_use] + pub fn is_empty(&self) -> bool { + self.traces.is_none() && self.ocsf.is_none() && self.agent.is_none() + } + + /// The supervisor environment variables for these exports. pub fn environment(&self) -> Vec<(&'static str, String)> { - vec![(OTLP_OCSF_ENDPOINT, self.endpoint.clone())] + let traces = self + .traces + .iter() + .flat_map(|export| export.environment(OTLP_ENDPOINT, OTLP_PROTOCOL)); + let ocsf = self + .ocsf + .iter() + .flat_map(|export| export.environment(OTLP_OCSF_ENDPOINT, OTLP_OCSF_PROTOCOL)); + let agent = self + .agent + .iter() + .flat_map(|export| export.environment(OTLP_AGENT_ENDPOINT, OTLP_AGENT_PROTOCOL)); + traces.chain(ocsf).chain(agent).collect() + } + + /// Applies `rewrite` to every collector endpoint, for drivers whose + /// supervisors reach the gateway host under another name. + #[must_use] + pub fn map_endpoints(self, rewrite: impl Fn(String) -> String) -> Self { + let map = |export: Option| { + export.map(|export| OtlpExport { + endpoint: rewrite(export.endpoint), + ..export + }) + }; + Self { + traces: map(self.traces), + ocsf: map(self.ocsf), + agent: map(self.agent), + } + } +} + +/// Compute-driver flags for the collectors given to supervisors, without the +/// leading `--`. The driver flags read the same environment variables that +/// supervisors do. +pub const OTLP_ENDPOINT_FLAG: &str = "otlp-endpoint"; +pub const OTLP_PROTOCOL_FLAG: &str = "otlp-protocol"; +pub const OTLP_OCSF_ENDPOINT_FLAG: &str = "otlp-ocsf-endpoint"; +pub const OTLP_OCSF_PROTOCOL_FLAG: &str = "otlp-ocsf-protocol"; +pub const OTLP_AGENT_ENDPOINT_FLAG: &str = "otlp-agent-endpoint"; +pub const OTLP_AGENT_PROTOCOL_FLAG: &str = "otlp-agent-protocol"; + +/// The OTLP flags every compute driver takes, flattened into its arguments. +#[cfg(feature = "clap")] +#[derive(Clone, Debug, Default, clap::Args)] +pub struct SupervisorOtlpArgs { + /// OTLP collector endpoint for the driver's and supervisors' spans. + #[arg(long = OTLP_ENDPOINT_FLAG, env = OTLP_ENDPOINT)] + pub endpoint: Option, + + /// Transport to the trace collector: `grpc` or `http/protobuf`. + #[arg(long = OTLP_PROTOCOL_FLAG, env = OTLP_PROTOCOL, default_value_t)] + pub protocol: OtlpProtocol, + + /// OTLP collector endpoint for supervisor OCSF event export. + #[arg(long = OTLP_OCSF_ENDPOINT_FLAG, env = OTLP_OCSF_ENDPOINT)] + pub ocsf_endpoint: Option, + + /// Transport to the OCSF collector: `grpc` or `http/protobuf`. + #[arg(long = OTLP_OCSF_PROTOCOL_FLAG, env = OTLP_OCSF_PROTOCOL, default_value_t)] + pub ocsf_protocol: OtlpProtocol, + + /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. + #[arg( + long = "otlp-ocsf-schema-version", + env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", + hide = true + )] + pub ocsf_schema_version: Option, + + /// OTLP collector endpoint that supervisors relay workload traces to. + /// Absent keeps the agent trace relay off. + #[arg(long = OTLP_AGENT_ENDPOINT_FLAG, env = OTLP_AGENT_ENDPOINT)] + pub agent_endpoint: Option, + + /// Transport to the agent trace relay's collector: `grpc` or `http/protobuf`. + #[arg(long = OTLP_AGENT_PROTOCOL_FLAG, env = OTLP_AGENT_PROTOCOL, default_value_t)] + pub agent_protocol: OtlpProtocol, +} + +#[cfg(feature = "clap")] +impl SupervisorOtlpArgs { + /// The exports these flags configure. + #[must_use] + pub fn supervisor_otlp(&self) -> SupervisorOtlp { + let export = |endpoint: &Option, protocol| { + endpoint + .clone() + .map(|endpoint| OtlpExport { endpoint, protocol }) + }; + SupervisorOtlp { + traces: export(&self.endpoint, self.protocol), + ocsf: export(&self.ocsf_endpoint, self.ocsf_protocol), + agent: export(&self.agent_endpoint, self.agent_protocol), + } + } + + /// Logs a warning when the deprecated schema version flag is set. + pub fn warn_deprecated(&self) { + if self.ocsf_schema_version.is_some() { + tracing::warn!( + "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" + ); + } } } @@ -381,15 +576,119 @@ mod otlp_relay_address_tests { mod tests { use super::*; + #[cfg(feature = "clap")] + #[test] + fn driver_flags_build_supervisor_otlp() { + use clap::Parser as _; + + #[derive(clap::Parser)] + struct Driver { + #[command(flatten)] + otlp: SupervisorOtlpArgs, + } + + let driver = Driver::try_parse_from([ + "driver", + "--otlp-endpoint", + "http://traces:4317", + "--otlp-ocsf-endpoint", + "http://siem:4318", + "--otlp-ocsf-protocol", + "http/protobuf", + "--otlp-ocsf-schema-version", + "2.0", + "--otlp-agent-endpoint", + "http://agents:4318", + "--otlp-agent-protocol", + "http/protobuf", + ]) + .expect("OTLP flags should parse"); + + assert_eq!(driver.otlp.ocsf_schema_version.as_deref(), Some("2.0")); + assert_eq!( + driver.otlp.supervisor_otlp(), + SupervisorOtlp { + traces: Some(OtlpExport { + endpoint: "http://traces:4317".into(), + protocol: OtlpProtocol::Grpc, + }), + ocsf: Some(OtlpExport { + endpoint: "http://siem:4318".into(), + protocol: OtlpProtocol::HttpProtobuf, + }), + agent: Some(OtlpExport { + endpoint: "http://agents:4318".into(), + protocol: OtlpProtocol::HttpProtobuf, + }), + } + ); + assert_eq!( + Driver::try_parse_from(["driver"]) + .unwrap() + .otlp + .supervisor_otlp(), + SupervisorOtlp::default() + ); + } + #[test] - fn ocsf_export_environment_carries_the_endpoint() { - let export = OcsfOtlpExport { - endpoint: "http://siem:4317".into(), + fn otlp_protocol_uses_the_otel_exporter_protocol_values() { + for (value, protocol) in [ + ("grpc", OtlpProtocol::Grpc), + ("http/protobuf", OtlpProtocol::HttpProtobuf), + ] { + assert_eq!(value.parse::(), Ok(protocol)); + assert_eq!(protocol.as_str(), value); + } + assert_eq!(OtlpProtocol::default(), OtlpProtocol::Grpc); + assert!("http/json".parse::().is_err()); + assert!("HTTP".parse::().is_err()); + } + + #[test] + fn supervisor_otlp_environment_names_each_lane() { + let export = |endpoint: &str, protocol| { + Some(OtlpExport { + endpoint: endpoint.into(), + protocol, + }) + }; + assert!(SupervisorOtlp::default().environment().is_empty()); + let otlp = SupervisorOtlp { + traces: export("http://traces:4317", OtlpProtocol::Grpc), + ocsf: export("http://siem:4318", OtlpProtocol::HttpProtobuf), + agent: export("http://agents:4318", OtlpProtocol::HttpProtobuf), }; assert_eq!( - export.environment(), - [(OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string())] + otlp.environment(), + [ + (OTLP_ENDPOINT, "http://traces:4317".to_string()), + (OTLP_OCSF_ENDPOINT, "http://siem:4318".to_string()), + (OTLP_OCSF_PROTOCOL, "http/protobuf".to_string()), + (OTLP_AGENT_ENDPOINT, "http://agents:4318".to_string()), + (OTLP_AGENT_PROTOCOL, "http/protobuf".to_string()), + ] ); + assert!(!otlp.is_empty()); + } + + #[test] + fn supervisor_otlp_rewrites_every_endpoint_and_keeps_protocols() { + let export = |endpoint: &str| { + Some(OtlpExport { + endpoint: endpoint.into(), + protocol: OtlpProtocol::HttpProtobuf, + }) + }; + let otlp = SupervisorOtlp { + traces: export("http://127.0.0.1:4317"), + ocsf: None, + agent: export("http://127.0.0.1:4318"), + } + .map_endpoints(|endpoint| endpoint.replace("127.0.0.1", "host.internal")); + assert_eq!(otlp.traces, export("http://host.internal:4317")); + assert_eq!(otlp.ocsf, None); + assert_eq!(otlp.agent, export("http://host.internal:4318")); } #[test] diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index e9751f7763..e4ba6f40c0 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -15,7 +15,7 @@ name = "openshell-driver-docker" path = "src/main.rs" [dependencies] -openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction", "otel", "clap"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } openshell-otel = { path = "../openshell-otel" } diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 7f2222bb34..23c48446cb 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -8,14 +8,16 @@ local socket that responds to an API ping. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When the gateway configures `[openshell.gateway.otlp]`, the in-process driver -exports spans to the same OTLP/gRPC collector as +exports spans to the same OTLP collector, over its configured protocol, as `openshell-driver-docker`. The standalone driver accepts -`OPENSHELL_OTLP_ENDPOINT`, continues W3C trace context from gateway RPC +`OPENSHELL_OTLP_ENDPOINT` and `OPENSHELL_OTLP_PROTOCOL`, continues W3C trace +context from gateway RPC metadata, and flushes spans during graceful shutdown. When `[openshell.gateway.otlp.agent]` is enabled, the driver also sets -`OPENSHELL_OTLP_AGENT_ENDPOINT` on the supervisor container, which starts the -agent trace relay. The standalone driver takes that endpoint as -`--otlp-agent-endpoint`. +`OPENSHELL_OTLP_AGENT_ENDPOINT` on the supervisor container, plus +`OPENSHELL_OTLP_AGENT_PROTOCOL` for a non-gRPC collector, which starts the +agent trace relay. The standalone driver takes them as `--otlp-agent-endpoint` +and `--otlp-agent-protocol`. ## Runtime Model diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 8bf5df4e22..6b761f2f59 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -201,25 +201,16 @@ pub struct DockerComputeConfig { /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, - /// OTLP/gRPC collector endpoint passed to supervisors. The gateway - /// supplies its own export endpoint; driver TOML cannot set it. - #[serde(skip)] - pub supervisor_otlp_endpoint: Option, /// Networking mode for the supervisor only. Workloads always use network=none. pub supervisor_network_mode: DockerSupervisorNetworkMode, /// Override the trusted host address. Bridge mode otherwise resolves Docker's host-gateway. pub host_gateway_ip: Option, - /// OCSF event export passed to supervisors. The gateway supplies its own - /// export; driver TOML cannot set it. - #[serde(skip)] - pub supervisor_ocsf_export: Option, - - /// OTLP/gRPC collector that supervisors relay workload traces to. The - /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it. + /// OTLP exports passed to supervisors. The gateway supplies them from + /// `[otlp]`; driver TOML cannot set them. #[serde(skip)] - pub supervisor_otlp_agent_endpoint: Option, + pub supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp, /// Gateway name passed to supervisors for their telemetry. The gateway /// supplies its own name; driver TOML cannot set it. @@ -324,11 +315,9 @@ impl Default for DockerComputeConfig { image_pull_policy: ImagePullPolicy::default(), sandbox_label: "default".to_string(), grpc_endpoint: String::new(), - supervisor_otlp_endpoint: None, supervisor_network_mode: DockerSupervisorNetworkMode::Host, host_gateway_ip: None, - supervisor_ocsf_export: None, - supervisor_otlp_agent_endpoint: None, + supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp::default(), supervisor_gateway_name: None, sandbox_runtime_image: None, supervisor_bin: None, @@ -364,13 +353,11 @@ struct DockerDriverRuntimeConfig { sandbox_binary: Arc>, supervisor_image_id: String, supervisor_grpc_endpoint: String, - supervisor_otlp_endpoint: Option, supervisor_network_mode: DockerSupervisorNetworkMode, supervisor_endpoint_explicit: bool, bridge_default_endpoint: Option, host_gateway_ip: Option, - supervisor_ocsf_export: Option, - supervisor_otlp_agent_endpoint: Option, + supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp, supervisor_gateway_name: Option, ssh_socket_path: String, guest_tls: Option, @@ -1027,15 +1014,11 @@ impl DockerComputeDriver { sandbox_binary, supervisor_image_id, supervisor_grpc_endpoint, - supervisor_otlp_endpoint: docker_config.supervisor_otlp_endpoint.clone(), + supervisor_otlp: docker_config.supervisor_otlp.clone(), supervisor_network_mode: docker_config.supervisor_network_mode, supervisor_endpoint_explicit, bridge_default_endpoint, host_gateway_ip, - supervisor_ocsf_export: docker_config.supervisor_ocsf_export.clone(), - supervisor_otlp_agent_endpoint: docker_config - .supervisor_otlp_agent_endpoint - .clone(), supervisor_gateway_name: docker_config.supervisor_gateway_name.clone(), ssh_socket_path: docker_config.ssh_socket_path.clone(), guest_tls, @@ -5970,30 +5953,19 @@ fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap /// trace, OCSF events, and relayed workload traces, identifying the gateway /// that launched it. fn supervisor_otlp_environment(config: &DockerDriverRuntimeConfig) -> Vec { - let traces = config.supervisor_otlp_endpoint.iter().flat_map(|endpoint| { - std::iter::once((openshell_core::sandbox_env::OTLP_ENDPOINT, endpoint.clone())) - .chain(openshell_otel::current_trace_context_environment()) - }); - let ocsf = config - .supervisor_ocsf_export + let exports = config.supervisor_otlp.environment(); + let trace_context = config + .supervisor_otlp + .traces .iter() - .flat_map(openshell_core::sandbox_env::OcsfOtlpExport::environment); - let agent = config - .supervisor_otlp_agent_endpoint - .iter() - .map(|endpoint| { - ( - openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT, - endpoint.clone(), - ) - }); + .flat_map(|_| openshell_otel::current_trace_context_environment()); let gateway_name = config .supervisor_gateway_name .iter() .map(|name| (openshell_core::sandbox_env::GATEWAY_NAME, name.clone())); - traces - .chain(ocsf) - .chain(agent) + exports + .into_iter() + .chain(trace_context) .chain(gateway_name) .map(|(name, value)| format!("{name}={value}")) .collect() diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index da45b1284e..c34961236c 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -8,6 +8,7 @@ use clap::Parser; use miette::{IntoDiagnostic, Result}; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::sandbox_env::SupervisorOtlpArgs; use openshell_driver_docker::{ComputeDriverService, DockerComputeConfig, DockerComputeDriver}; use tracing::info; @@ -44,21 +45,8 @@ struct Args { #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] log_level: String, - #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] - otlp_endpoint: Option, - - /// OTLP/gRPC collector endpoint for supervisor OCSF event export. - #[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")] - otlp_ocsf_endpoint: Option, - - /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. - #[arg(long, env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", hide = true)] - otlp_ocsf_schema_version: Option, - - /// OTLP/gRPC collector endpoint that supervisors relay workload traces - /// to. Absent keeps the agent trace relay off. - #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")] - otlp_agent_endpoint: Option, + #[command(flatten)] + otlp: SupervisorOtlpArgs, #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, @@ -70,18 +58,16 @@ async fn main() -> Result<()> { let _tracing = openshell_otel::install_driver_tracing( openshell_driver_docker::otel_tracing::TRACING, openshell_otel::DriverTracingConfig { - endpoint: args.otlp_endpoint.as_deref(), + endpoint: args.otlp.endpoint.as_deref(), + protocol: args.otlp.protocol.into(), gateway_name: args.gateway_name.as_deref(), service_version: VERSION, log_level: &args.log_level, }, ); - if args.otlp_ocsf_schema_version.is_some() { - tracing::warn!( - "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" - ); - } + args.otlp.warn_deprecated(); + let supervisor_otlp = args.otlp.supervisor_otlp(); let config_source = std::fs::read_to_string(&args.config).into_diagnostic()?; let mut docker_config: DockerComputeConfig = toml::from_str(&config_source).into_diagnostic()?; @@ -95,13 +81,8 @@ async fn main() -> Result<()> { if let Some(image) = args.supervisor_image { docker_config.supervisor_image = Some(image); } - docker_config.supervisor_otlp_endpoint = args.otlp_endpoint.clone(); - docker_config.supervisor_otlp_agent_endpoint = args.otlp_agent_endpoint.clone(); + docker_config.supervisor_otlp = supervisor_otlp; docker_config.supervisor_gateway_name = args.gateway_name.clone(); - docker_config.supervisor_ocsf_export = args - .otlp_ocsf_endpoint - .clone() - .map(|endpoint| openshell_core::sandbox_env::OcsfOtlpExport { endpoint }); let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config, None) .await .into_diagnostic()?; diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 7e99f37e22..125cb1d56a 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -324,13 +324,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { sandbox_binary: Arc::new(b"\x7fELFtest".to_vec()), supervisor_image_id: "sha256:supervisor-test".to_string(), supervisor_grpc_endpoint: "https://host.openshell.internal:8443".to_string(), - supervisor_otlp_endpoint: None, supervisor_network_mode: DockerSupervisorNetworkMode::Host, supervisor_endpoint_explicit: true, bridge_default_endpoint: Some("https://host.docker.internal:8443".to_string()), host_gateway_ip: None, - supervisor_ocsf_export: None, - supervisor_otlp_agent_endpoint: None, + supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp::default(), supervisor_gateway_name: None, ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), guest_tls: Some(DockerGuestTlsPaths { @@ -1517,34 +1515,66 @@ fn docker_child_environment_strips_supervisor_control_keys() { #[test] fn supervisor_otlp_environment_carries_only_configured_signals() { use openshell_core::sandbox_env::{ - GATEWAY_NAME, OTLP_AGENT_ENDPOINT, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, OcsfOtlpExport, + GATEWAY_NAME, OTLP_AGENT_ENDPOINT, OTLP_AGENT_PROTOCOL, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, + OTLP_OCSF_PROTOCOL, OTLP_PROTOCOL, OtlpExport, OtlpProtocol, + }; + let export = |endpoint: &str, protocol| { + Some(OtlpExport { + endpoint: endpoint.to_string(), + protocol, + }) }; let mut config = runtime_config(); assert!(supervisor_otlp_environment(&config).is_empty()); - config.supervisor_otlp_endpoint = Some("http://127.0.0.1:4317".to_string()); + config.supervisor_otlp.traces = export("http://127.0.0.1:4317", OtlpProtocol::Grpc); assert_eq!( supervisor_otlp_environment(&config), [format!("{OTLP_ENDPOINT}=http://127.0.0.1:4317")] ); - config.supervisor_otlp_endpoint = None; - config.supervisor_ocsf_export = Some(OcsfOtlpExport { - endpoint: "http://siem:4317".to_string(), - }); + config.supervisor_otlp.traces = export("http://127.0.0.1:4318", OtlpProtocol::HttpProtobuf); + assert_eq!( + supervisor_otlp_environment(&config), + [ + format!("{OTLP_ENDPOINT}=http://127.0.0.1:4318"), + format!("{OTLP_PROTOCOL}=http/protobuf"), + ] + ); + + config.supervisor_otlp.traces = None; + config.supervisor_otlp.ocsf = export("http://siem:4317", OtlpProtocol::Grpc); assert_eq!( supervisor_otlp_environment(&config), [format!("{OTLP_OCSF_ENDPOINT}=http://siem:4317")] ); - config.supervisor_ocsf_export = None; - config.supervisor_otlp_agent_endpoint = Some("http://agents:4317".to_string()); + config.supervisor_otlp.ocsf = export("http://siem:4318", OtlpProtocol::HttpProtobuf); + assert_eq!( + supervisor_otlp_environment(&config), + [ + format!("{OTLP_OCSF_ENDPOINT}=http://siem:4318"), + format!("{OTLP_OCSF_PROTOCOL}=http/protobuf"), + ] + ); + + config.supervisor_otlp.ocsf = None; + config.supervisor_otlp.agent = export("http://agents:4317", OtlpProtocol::Grpc); assert_eq!( supervisor_otlp_environment(&config), [format!("{OTLP_AGENT_ENDPOINT}=http://agents:4317")] ); + config.supervisor_otlp.agent = export("http://agents:4318", OtlpProtocol::HttpProtobuf); + assert_eq!( + supervisor_otlp_environment(&config), + [ + format!("{OTLP_AGENT_ENDPOINT}=http://agents:4318"), + format!("{OTLP_AGENT_PROTOCOL}=http/protobuf"), + ] + ); + config.supervisor_gateway_name = Some("prod-west".to_string()); assert_eq!( supervisor_otlp_environment(&config).last(), diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index c84713f2bf..4b7659ffdd 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -16,7 +16,7 @@ path = "src/main.rs" [dependencies] openshell-crypto = { workspace = true, features = ["tls-kube"] } -openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt"] } +openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt", "otel", "clap"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } openshell-supervisor-control = { path = "../openshell-supervisor-control" } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index b95fb6bf3f..027f0a5361 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -40,7 +40,8 @@ workspace namespace modes via `workspace_mode`: Kubernetes namespace. When the gateway configures `[openshell.gateway.otlp]`, Kubernetes -compute-driver spans export to the same OTLP/gRPC collector with the service +compute-driver spans export to the same OTLP collector, over its configured +protocol, with the service name `openshell-driver-kubernetes`. The driver preserves the gateway trace context and uses the same compute-driver RPC span names in its in-process and standalone forms. Standalone deployments set `--gateway-name` or @@ -93,7 +94,9 @@ When the gateway exports OTLP traces, the driver sets supervisor exports its spans there and parents its startup span on that context. When `[openshell.gateway.otlp.agent]` is enabled, the driver also sets `OPENSHELL_OTLP_AGENT_ENDPOINT`, which starts the supervisor's agent trace -relay. Neither endpoint is configurable in driver TOML. +relay. Each endpoint is accompanied by its `_PROTOCOL` variable +(`OPENSHELL_OTLP_PROTOCOL`, `OPENSHELL_OTLP_AGENT_PROTOCOL`) when the collector +uses `http/protobuf`. None of these is configurable in driver TOML. The driver creates one namespace-wide `NetworkPolicy` before it releases any workload Pod. It selects every OpenShell workload, denies all workload egress, diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a983459477..9a9812f837 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -364,18 +364,10 @@ pub struct KubernetesComputeConfig { /// contain at least one usable trust anchor. pub proxy_ca_bundle: Option, pub grpc_endpoint: String, - /// OTLP/gRPC collector endpoint passed to supervisor pods. The gateway - /// supplies its own export endpoint; driver TOML cannot set it. + /// OTLP exports passed to supervisor pods. The gateway supplies them + /// from `[otlp]`; driver TOML cannot set them. #[serde(skip)] - pub supervisor_otlp_endpoint: Option, - /// OCSF event export passed to supervisors. The gateway supplies its own - /// export; driver TOML cannot set it. - #[serde(skip)] - pub supervisor_ocsf_export: Option, - /// OTLP/gRPC collector that supervisors relay workload traces to. The - /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it. - #[serde(skip)] - pub supervisor_otlp_agent_endpoint: Option, + pub supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp, /// Gateway name passed to supervisors for their telemetry. The gateway /// supplies its own name; driver TOML cannot set it. @@ -504,9 +496,7 @@ impl Default for KubernetesComputeConfig { proxy_connect_by_hostname: None, proxy_ca_bundle: None, grpc_endpoint: String::new(), - supervisor_otlp_endpoint: None, - supervisor_ocsf_export: None, - supervisor_otlp_agent_endpoint: None, + supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp::default(), supervisor_gateway_name: None, ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), client_tls_secret_name: String::new(), diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 5737951d13..182f3cf185 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1112,23 +1112,10 @@ impl KubernetesComputeDriver { /// trace, OCSF events, and relayed workload traces, identifying the /// gateway that launched it. fn supervisor_otlp_environment(&self) -> Vec<(&'static str, String)> { - let mut environment = Vec::new(); - if let Some(endpoint) = self.config.supervisor_otlp_endpoint.as_deref() { - environment.push(( - openshell_core::sandbox_env::OTLP_ENDPOINT, - endpoint.to_string(), - )); + let mut environment = self.config.supervisor_otlp.environment(); + if self.config.supervisor_otlp.traces.is_some() { environment.extend(openshell_otel::current_trace_context_environment()); } - if let Some(export) = &self.config.supervisor_ocsf_export { - environment.extend(export.environment()); - } - if let Some(endpoint) = &self.config.supervisor_otlp_agent_endpoint { - environment.push(( - openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT, - endpoint.clone(), - )); - } if let Some(name) = &self.config.supervisor_gateway_name { environment.push((openshell_core::sandbox_env::GATEWAY_NAME, name.clone())); } @@ -1286,11 +1273,7 @@ impl KubernetesComputeDriver { if let Some(shared) = &shared_supervisor { shared.spawn_token_secret_writer(client.clone(), shutdown_rx.clone()); } - if shared_supervisor.is_some() - && (config.supervisor_otlp_endpoint.is_some() - || config.supervisor_ocsf_export.is_some() - || config.supervisor_otlp_agent_endpoint.is_some()) - { + if shared_supervisor.is_some() && !config.supervisor_otlp.is_empty() { warn!( "the gateway supervisor topology does not export supervisor OTLP traces or OCSF events or relay workload traces; shared sandbox instances keep console logs and gateway log push" ); @@ -8984,46 +8967,79 @@ mod tests { #[tokio::test] async fn supervisor_otlp_environment_carries_only_configured_signals() { - use openshell_core::sandbox_env::{OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, OcsfOtlpExport}; + use openshell_core::sandbox_env::{ + OTLP_AGENT_ENDPOINT, OTLP_AGENT_PROTOCOL, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, + OTLP_OCSF_PROTOCOL, OTLP_PROTOCOL, OtlpExport, OtlpProtocol, SupervisorOtlp, + }; - let names = |config: KubernetesComputeConfig| { - KubernetesComputeDriver::new_for_test(config) - .supervisor_otlp_environment() + let export = |endpoint: &str, protocol| { + Some(OtlpExport { + endpoint: endpoint.into(), + protocol, + }) + }; + let environment = |supervisor_otlp| { + KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { + supervisor_otlp, + ..Default::default() + }) + .supervisor_otlp_environment() + }; + let names = |supervisor_otlp| { + environment(supervisor_otlp) .into_iter() .map(|(name, _)| name) .collect::>() }; - assert!(names(KubernetesComputeConfig::default()).is_empty()); + assert!(names(SupervisorOtlp::default()).is_empty()); + let traces = names(SupervisorOtlp { + traces: export("http://traces:4317", OtlpProtocol::Grpc), + ..Default::default() + }); + assert_eq!(traces.first(), Some(&OTLP_ENDPOINT)); + assert!(!traces.contains(&OTLP_PROTOCOL)); assert_eq!( - names(KubernetesComputeConfig { - supervisor_otlp_endpoint: Some("http://traces:4317".into()), + names(SupervisorOtlp { + traces: export("http://traces:4318", OtlpProtocol::HttpProtobuf), ..Default::default() - }) - .first(), - Some(&OTLP_ENDPOINT) + })[..2], + [OTLP_ENDPOINT, OTLP_PROTOCOL] ); - let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { - supervisor_ocsf_export: Some(OcsfOtlpExport { - endpoint: "http://siem:4317".into(), + assert_eq!( + environment(SupervisorOtlp { + ocsf: export("http://siem:4317", OtlpProtocol::Grpc), + ..Default::default() }), - ..Default::default() - }); + [(OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string()),] + ); assert_eq!( - driver.supervisor_otlp_environment(), - [(OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string())] + environment(SupervisorOtlp { + ocsf: export("http://siem:4318", OtlpProtocol::HttpProtobuf), + ..Default::default() + }), + [ + (OTLP_OCSF_ENDPOINT, "http://siem:4318".to_string()), + (OTLP_OCSF_PROTOCOL, "http/protobuf".to_string()), + ] ); - let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { - supervisor_otlp_agent_endpoint: Some("http://agents:4317".into()), - ..Default::default() - }); assert_eq!( - driver.supervisor_otlp_environment(), - [( - openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT, - "http://agents:4317".to_string() - )] + environment(SupervisorOtlp { + agent: export("http://agents:4317", OtlpProtocol::Grpc), + ..Default::default() + }), + [(OTLP_AGENT_ENDPOINT, "http://agents:4317".to_string())] + ); + assert_eq!( + environment(SupervisorOtlp { + agent: export("http://agents:4318", OtlpProtocol::HttpProtobuf), + ..Default::default() + }), + [ + (OTLP_AGENT_ENDPOINT, "http://agents:4318".to_string()), + (OTLP_AGENT_PROTOCOL, "http/protobuf".to_string()), + ] ); let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 0b2b010949..f7ee6457fc 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -10,6 +10,7 @@ use tracing::info; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::sandbox_env::SupervisorOtlpArgs; use openshell_driver_kubernetes::{ ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, KubernetesImagePullPolicy, @@ -42,21 +43,8 @@ struct Args { #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] log_level: String, - #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] - otlp_endpoint: Option, - - /// OTLP/gRPC collector endpoint for supervisor OCSF event export. - #[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")] - otlp_ocsf_endpoint: Option, - - /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. - #[arg(long, env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", hide = true)] - otlp_ocsf_schema_version: Option, - - /// OTLP/gRPC collector endpoint that supervisors relay workload traces - /// to. Absent keeps the agent trace relay off. - #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")] - otlp_agent_endpoint: Option, + #[command(flatten)] + otlp: SupervisorOtlpArgs, #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, @@ -238,17 +226,14 @@ async fn main() -> Result<()> { let _tracing = openshell_otel::install_driver_tracing( openshell_driver_kubernetes::otel_tracing::TRACING, openshell_otel::DriverTracingConfig { - endpoint: args.otlp_endpoint.as_deref(), + endpoint: args.otlp.endpoint.as_deref(), + protocol: args.otlp.protocol.into(), gateway_name: args.gateway_name.as_deref(), service_version: VERSION, log_level: &args.log_level, }, ); - if args.otlp_ocsf_schema_version.is_some() { - tracing::warn!( - "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" - ); - } + args.otlp.warn_deprecated(); let managed_ssh_gateway_pod_selector = args .managed_ssh_gateway_pod_selector @@ -263,6 +248,7 @@ async fn main() -> Result<()> { }) .collect::>>()?; + let supervisor_otlp = args.otlp.supervisor_otlp(); let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { @@ -303,13 +289,8 @@ async fn main() -> Result<()> { proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true), proxy_ca_bundle: args.proxy_ca_bundle, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - supervisor_otlp_endpoint: args.otlp_endpoint.clone(), - supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), + supervisor_otlp, supervisor_gateway_name: args.gateway_name.clone(), - supervisor_ocsf_export: args - .otlp_ocsf_endpoint - .clone() - .map(|endpoint| openshell_core::sandbox_env::OcsfOtlpExport { endpoint }), ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), @@ -390,7 +371,7 @@ mod tests { .expect("OTLP endpoint should parse"); assert_eq!( - args.otlp_endpoint.as_deref(), + args.otlp.endpoint.as_deref(), Some("http://collector.example:4317") ); assert_eq!(args.gateway_name.as_deref(), Some("kubernetes-dev")); diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index bdbd59e3a7..3fb3c51b5d 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -15,7 +15,7 @@ name = "openshell-driver-podman" path = "src/main.rs" [dependencies] -openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction", "otel", "clap"] } openshell-otel = { path = "../openshell-otel" } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 6d5d7a20b3..1c63bb8f37 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -157,6 +157,7 @@ authenticated health signal. Gateway OTLP configuration continues to export compute-driver spans under the `openshell-driver-podman` service, preserving gateway trace context. When `[openshell.gateway.otlp.agent]` is enabled, the driver sets -`OPENSHELL_OTLP_AGENT_ENDPOINT` on the supervisor container, which starts the +`OPENSHELL_OTLP_AGENT_ENDPOINT` on the supervisor container, plus +`OPENSHELL_OTLP_AGENT_PROTOCOL` for a non-gRPC collector, which starts the agent trace relay. On Podman Machine it rewrites a loopback agent endpoint to the desktop host, as it does for the supervisor's trace endpoint. diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 2645783cce..a7a22ca686 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -43,18 +43,10 @@ pub struct PodmanComputeConfig { /// When empty, the driver selects loopback on Linux or /// `host.containers.internal` with Podman Machine, using `gateway_port`. pub grpc_endpoint: String, - /// OTLP/gRPC collector endpoint passed to supervisors. The gateway - /// supplies its own export endpoint; driver TOML cannot set it. + /// OTLP exports passed to supervisors. The gateway supplies them from + /// `[otlp]`; driver TOML cannot set them. #[serde(skip)] - pub supervisor_otlp_endpoint: Option, - /// OCSF event export passed to supervisors. The gateway supplies its own - /// export; driver TOML cannot set it. - #[serde(skip)] - pub supervisor_ocsf_export: Option, - /// OTLP/gRPC collector that supervisors relay workload traces to. The - /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it. - #[serde(skip)] - pub supervisor_otlp_agent_endpoint: Option, + pub supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp, /// Gateway name passed to supervisors for their telemetry. The gateway /// supplies its own name; driver TOML cannot set it. @@ -551,9 +543,7 @@ impl Default for PodmanComputeConfig { default_image: openshell_core::image::default_sandbox_image(), image_pull_policy: ImagePullPolicy::default(), grpc_endpoint: String::new(), - supervisor_otlp_endpoint: None, - supervisor_ocsf_export: None, - supervisor_otlp_agent_endpoint: None, + supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp::default(), supervisor_gateway_name: None, gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index a63c58d5f8..2f606dfa6e 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -735,31 +735,20 @@ fn build_env( openshell_core::sandbox_env::TELEMETRY_ENABLED.into(), openshell_core::telemetry::enabled_env_value().into(), ); - if let Some(endpoint) = &config.supervisor_otlp_endpoint { - env.insert( - openshell_core::sandbox_env::OTLP_ENDPOINT.into(), - endpoint.clone(), - ); + env.extend( + config + .supervisor_otlp + .environment() + .into_iter() + .map(|(name, value)| (name.to_string(), value)), + ); + if config.supervisor_otlp.traces.is_some() { env.extend( openshell_otel::current_trace_context_environment() .into_iter() .map(|(name, value)| (name.to_string(), value)), ); } - if let Some(export) = &config.supervisor_ocsf_export { - env.extend( - export - .environment() - .into_iter() - .map(|(name, value)| (name.to_string(), value)), - ); - } - if let Some(endpoint) = &config.supervisor_otlp_agent_endpoint { - env.insert( - openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT.into(), - endpoint.clone(), - ); - } if let Some(name) = &config.supervisor_gateway_name { env.insert( openshell_core::sandbox_env::GATEWAY_NAME.into(), @@ -2902,56 +2891,86 @@ mod tests { #[test] fn container_spec_passes_the_gateway_otlp_endpoint_to_the_supervisor() { + use openshell_core::sandbox_env::{ + OTLP_AGENT_ENDPOINT, OTLP_AGENT_PROTOCOL, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, + OTLP_OCSF_PROTOCOL, OTLP_PROTOCOL, OtlpExport, OtlpProtocol, SupervisorOtlp, + }; let sandbox = test_sandbox("test-id", "legit-name"); let spec = build_container_spec(&sandbox, &test_config()); - assert!( - spec["env"] - .get(openshell_core::sandbox_env::OTLP_ENDPOINT) - .is_none() - ); + assert!(spec["env"].get(OTLP_ENDPOINT).is_none()); - let config = PodmanComputeConfig { - supervisor_otlp_endpoint: Some("http://127.0.0.1:4317".to_string()), + let export = |endpoint: &str, protocol| { + Some(OtlpExport { + endpoint: endpoint.to_string(), + protocol, + }) + }; + let with_otlp = |supervisor_otlp| PodmanComputeConfig { + supervisor_otlp, ..test_config() }; - let spec = build_container_spec(&sandbox, &config); - assert_eq!( - spec["env"][openshell_core::sandbox_env::OTLP_ENDPOINT], - "http://127.0.0.1:4317" + + let spec = build_container_spec( + &sandbox, + &with_otlp(SupervisorOtlp { + traces: export("http://127.0.0.1:4317", OtlpProtocol::Grpc), + ..SupervisorOtlp::default() + }), ); - assert!( - spec["env"] - .get(openshell_core::sandbox_env::OTLP_OCSF_ENDPOINT) - .is_none() + assert_eq!(spec["env"][OTLP_ENDPOINT], "http://127.0.0.1:4317"); + assert!(spec["env"].get(OTLP_PROTOCOL).is_none()); + assert!(spec["env"].get(OTLP_OCSF_ENDPOINT).is_none()); + + let spec = build_container_spec( + &sandbox, + &with_otlp(SupervisorOtlp { + traces: export("http://127.0.0.1:4318", OtlpProtocol::HttpProtobuf), + ..SupervisorOtlp::default() + }), ); + assert_eq!(spec["env"][OTLP_PROTOCOL], "http/protobuf"); - let config = PodmanComputeConfig { - supervisor_ocsf_export: Some(openshell_core::sandbox_env::OcsfOtlpExport { - endpoint: "http://siem:4317".to_string(), + let spec = build_container_spec( + &sandbox, + &with_otlp(SupervisorOtlp { + ocsf: export("http://siem:4317", OtlpProtocol::Grpc), + ..SupervisorOtlp::default() }), - ..test_config() - }; - let spec = build_container_spec(&sandbox, &config); - assert_eq!( - spec["env"][openshell_core::sandbox_env::OTLP_OCSF_ENDPOINT], - "http://siem:4317" ); + assert_eq!(spec["env"][OTLP_OCSF_ENDPOINT], "http://siem:4317"); + assert!(spec["env"].get(OTLP_OCSF_PROTOCOL).is_none()); assert!( - spec["env"] - .get(openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT) - .is_none(), + spec["env"].get(OTLP_AGENT_ENDPOINT).is_none(), "the agent trace relay stays off unless the gateway enables it" ); - let config = PodmanComputeConfig { - supervisor_otlp_agent_endpoint: Some("http://agents:4317".to_string()), - ..test_config() - }; - let spec = build_container_spec(&sandbox, &config); - assert_eq!( - spec["env"][openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT], - "http://agents:4317" + let spec = build_container_spec( + &sandbox, + &with_otlp(SupervisorOtlp { + ocsf: export("http://siem:4318", OtlpProtocol::HttpProtobuf), + ..SupervisorOtlp::default() + }), + ); + assert_eq!(spec["env"][OTLP_OCSF_PROTOCOL], "http/protobuf"); + + let spec = build_container_spec( + &sandbox, + &with_otlp(SupervisorOtlp { + agent: export("http://agents:4317", OtlpProtocol::Grpc), + ..SupervisorOtlp::default() + }), + ); + assert_eq!(spec["env"][OTLP_AGENT_ENDPOINT], "http://agents:4317"); + assert!(spec["env"].get(OTLP_AGENT_PROTOCOL).is_none()); + + let spec = build_container_spec( + &sandbox, + &with_otlp(SupervisorOtlp { + agent: export("http://agents:4318", OtlpProtocol::HttpProtobuf), + ..SupervisorOtlp::default() + }), ); + assert_eq!(spec["env"][OTLP_AGENT_PROTOCOL], "http/protobuf"); let config = PodmanComputeConfig { supervisor_gateway_name: Some("prod-west".to_string()), diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 00eb79c0dd..867c09a38c 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -82,35 +82,11 @@ fn select_grpc_endpoint( ) } -fn select_supervisor_otlp_endpoint( - config: &PodmanComputeConfig, +fn select_supervisor_otlp( + otlp: openshell_core::sandbox_env::SupervisorOtlp, environment: PodmanEndpointEnvironment, -) -> Option { - config - .supervisor_otlp_endpoint - .as_deref() - .map(|endpoint| supervisor_collector_endpoint(endpoint, environment)) -} - -fn select_supervisor_otlp_agent_endpoint( - config: &PodmanComputeConfig, - environment: PodmanEndpointEnvironment, -) -> Option { - config - .supervisor_otlp_agent_endpoint - .as_deref() - .map(|endpoint| supervisor_collector_endpoint(endpoint, environment)) -} - -fn select_supervisor_ocsf_export( - config: &PodmanComputeConfig, - environment: PodmanEndpointEnvironment, -) -> Option { - config.supervisor_ocsf_export.clone().map(|export| { - openshell_core::sandbox_env::OcsfOtlpExport { - endpoint: supervisor_collector_endpoint(&export.endpoint, environment), - } - }) +) -> openshell_core::sandbox_env::SupervisorOtlp { + otlp.map_endpoints(|endpoint| supervisor_collector_endpoint(&endpoint, environment)) } /// Point a loopback collector endpoint at the desktop host on Podman Machine, @@ -691,12 +667,10 @@ impl PodmanComputeDriver { // standard desktop-host alias. let endpoint_was_selected = config.grpc_endpoint.is_empty(); config.grpc_endpoint = select_grpc_endpoint(&config, PodmanEndpointEnvironment::current()); - config.supervisor_otlp_endpoint = - select_supervisor_otlp_endpoint(&config, PodmanEndpointEnvironment::current()); - config.supervisor_ocsf_export = - select_supervisor_ocsf_export(&config, PodmanEndpointEnvironment::current()); - config.supervisor_otlp_agent_endpoint = - select_supervisor_otlp_agent_endpoint(&config, PodmanEndpointEnvironment::current()); + config.supervisor_otlp = select_supervisor_otlp( + std::mem::take(&mut config.supervisor_otlp), + PodmanEndpointEnvironment::current(), + ); if endpoint_was_selected { info!( grpc_endpoint = %config.grpc_endpoint, @@ -3521,109 +3495,69 @@ mod tests { ); } - fn otlp_config(endpoint: Option<&str>) -> PodmanComputeConfig { - PodmanComputeConfig { - supervisor_otlp_endpoint: endpoint.map(str::to_string), - ..PodmanComputeConfig::default() + fn supervisor_otlp(endpoint: &str) -> openshell_core::sandbox_env::SupervisorOtlp { + use openshell_core::sandbox_env::{OtlpExport, OtlpProtocol, SupervisorOtlp}; + let export = |protocol| { + Some(OtlpExport { + endpoint: endpoint.to_string(), + protocol, + }) + }; + SupervisorOtlp { + traces: export(OtlpProtocol::Grpc), + ocsf: export(OtlpProtocol::HttpProtobuf), + agent: export(OtlpProtocol::HttpProtobuf), } } #[test] - fn supervisor_otlp_endpoint_stays_unset_without_gateway_export() { - assert_eq!( - select_supervisor_otlp_endpoint( - &otlp_config(None), + fn supervisor_otlp_stays_empty_without_gateway_export() { + assert!( + select_supervisor_otlp( + openshell_core::sandbox_env::SupervisorOtlp::default(), PodmanEndpointEnvironment::PodmanMachine - ), - None + ) + .is_empty() ); } #[test] - fn supervisor_otlp_endpoint_keeps_loopback_on_linux() { + fn supervisor_otlp_keeps_loopback_on_linux() { assert_eq!( - select_supervisor_otlp_endpoint( - &otlp_config(Some("http://127.0.0.1:4317")), + select_supervisor_otlp( + supervisor_otlp("http://127.0.0.1:4317"), PodmanEndpointEnvironment::LinuxHost - ) - .as_deref(), - Some("http://127.0.0.1:4317") + ), + supervisor_otlp("http://127.0.0.1:4317") ); } #[test] - fn supervisor_otlp_endpoint_uses_host_alias_for_loopback_on_podman_machine() { + fn supervisor_otlp_uses_host_alias_for_loopback_on_podman_machine() { for endpoint in [ "http://127.0.0.1:4317", "http://localhost:4317", "http://[::1]:4317", ] { assert_eq!( - select_supervisor_otlp_endpoint( - &otlp_config(Some(endpoint)), + select_supervisor_otlp( + supervisor_otlp(endpoint), PodmanEndpointEnvironment::PodmanMachine - ) - .as_deref(), - Some("http://host.containers.internal:4317"), + ), + supervisor_otlp("http://host.containers.internal:4317"), "{endpoint}" ); } } #[test] - fn supervisor_ocsf_endpoint_uses_host_alias_for_loopback_on_podman_machine() { - let export = |endpoint: &str| openshell_core::sandbox_env::OcsfOtlpExport { - endpoint: endpoint.to_string(), - }; - let config = PodmanComputeConfig { - supervisor_ocsf_export: Some(export("http://127.0.0.1:4317")), - ..PodmanComputeConfig::default() - }; - assert_eq!( - select_supervisor_ocsf_export(&config, PodmanEndpointEnvironment::PodmanMachine), - Some(export("http://host.containers.internal:4317")) - ); - assert_eq!( - select_supervisor_ocsf_export( - &PodmanComputeConfig::default(), - PodmanEndpointEnvironment::PodmanMachine - ), - None - ); - } - - #[test] - fn supervisor_agent_endpoint_uses_host_alias_for_loopback_on_podman_machine() { - let config = PodmanComputeConfig { - supervisor_otlp_agent_endpoint: Some("http://127.0.0.1:4317".to_string()), - ..PodmanComputeConfig::default() - }; + fn supervisor_otlp_keeps_routable_hosts_on_podman_machine() { assert_eq!( - select_supervisor_otlp_agent_endpoint( - &config, - PodmanEndpointEnvironment::PodmanMachine - ) - .as_deref(), - Some("http://host.containers.internal:4317") - ); - assert_eq!( - select_supervisor_otlp_agent_endpoint( - &PodmanComputeConfig::default(), + select_supervisor_otlp( + supervisor_otlp("https://collector.example.com:4317"), PodmanEndpointEnvironment::PodmanMachine ), - None - ); - } - - #[test] - fn supervisor_otlp_endpoint_keeps_routable_hosts_on_podman_machine() { - assert_eq!( - select_supervisor_otlp_endpoint( - &otlp_config(Some("https://collector.example.com:4317")), - PodmanEndpointEnvironment::PodmanMachine - ) - .as_deref(), - Some("https://collector.example.com:4317") + supervisor_otlp("https://collector.example.com:4317") ); } diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 2d510e4652..557c7f0b1f 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -10,6 +10,7 @@ use std::path::PathBuf; use tracing::info; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::sandbox_env::SupervisorOtlpArgs; use openshell_core::{AppArmorProfile, ImagePullPolicy, VERSION}; use openshell_driver_podman::config::{DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS}; use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; @@ -39,21 +40,8 @@ struct Args { #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] log_level: String, - #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] - otlp_endpoint: Option, - - /// OTLP/gRPC collector endpoint for supervisor OCSF event export. - #[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")] - otlp_ocsf_endpoint: Option, - - /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. - #[arg(long, env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", hide = true)] - otlp_ocsf_schema_version: Option, - - /// OTLP/gRPC collector endpoint that supervisors relay workload traces - /// to. Absent keeps the agent trace relay off. - #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")] - otlp_agent_endpoint: Option, + #[command(flatten)] + otlp: SupervisorOtlpArgs, #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, @@ -221,18 +209,16 @@ async fn main() -> Result<()> { let _tracing = openshell_otel::install_driver_tracing( openshell_driver_podman::otel_tracing::TRACING, openshell_otel::DriverTracingConfig { - endpoint: args.otlp_endpoint.as_deref(), + endpoint: args.otlp.endpoint.as_deref(), + protocol: args.otlp.protocol.into(), gateway_name: args.gateway_name.as_deref(), service_version: VERSION, log_level: &args.log_level, }, ); - if args.otlp_ocsf_schema_version.is_some() { - tracing::warn!( - "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" - ); - } + args.otlp.warn_deprecated(); + let supervisor_otlp = args.otlp.supervisor_otlp(); // External drivers have no gateway-local staged artifact. Do not accept // a path or silently emulate destination-CA delivery outside the gateway. let driver = PodmanComputeDriver::new( @@ -243,13 +229,8 @@ async fn main() -> Result<()> { default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - supervisor_otlp_endpoint: args.otlp_endpoint.clone(), - supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), + supervisor_otlp, supervisor_gateway_name: args.gateway_name.clone(), - supervisor_ocsf_export: args - .otlp_ocsf_endpoint - .clone() - .map(|endpoint| openshell_core::sandbox_env::OcsfOtlpExport { endpoint }), gateway_port: args.gateway_port, host_gateway_ip: args .host_gateway_ip @@ -373,7 +354,7 @@ mod tests { .expect("OTLP configuration should be accepted"); assert_eq!( - args.otlp_endpoint.as_deref(), + args.otlp.endpoint.as_deref(), Some("http://collector.internal:4317") ); assert_eq!(args.gateway_name.as_deref(), Some("production-us-west")); diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index 54a24e416a..57921d4b9a 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -24,7 +24,7 @@ name = "openshell-vm-init" path = "src/bin/openshell-vm-init.rs" [dependencies] -openshell-core = { path = "../openshell-core", default-features = false } +openshell-core = { path = "../openshell-core", default-features = false, features = ["clap"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } openshell-otel = { path = "../openshell-otel", optional = true } @@ -80,6 +80,7 @@ compute-driver = [ "dep:nix", "dep:oci-client", "dep:openshell-otel", + "openshell-core/otel", "dep:openshell-policy", "dep:openshell-driver-podman", "dep:openshell-vfio", diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 6260a52842..c9d691473b 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -348,18 +348,10 @@ pub struct VmDriverConfig { #[serde(default)] pub resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig, pub grpc_endpoint: String, - /// OTLP/gRPC collector endpoint passed to supervisors. The gateway - /// supplies its own export endpoint; driver TOML cannot set it. + /// OTLP exports passed to supervisors. The gateway supplies them from + /// `[otlp]`; driver TOML cannot set them. #[serde(skip)] - pub supervisor_otlp_endpoint: Option, - /// OCSF event export passed to supervisors. The gateway supplies its own - /// export; driver TOML cannot set it. - #[serde(skip)] - pub supervisor_ocsf_export: Option, - /// OTLP/gRPC collector that supervisors relay workload traces to. The - /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it. - #[serde(skip)] - pub supervisor_otlp_agent_endpoint: Option, + pub supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp, /// Gateway name passed to supervisors for their telemetry. pub supervisor_gateway_name: Option, pub state_dir: PathBuf, @@ -433,12 +425,7 @@ impl std::fmt::Debug for VmDriverConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("VmDriverConfig") .field("grpc_endpoint", &self.grpc_endpoint) - .field("supervisor_otlp_endpoint", &self.supervisor_otlp_endpoint) - .field("supervisor_ocsf_export", &self.supervisor_ocsf_export) - .field( - "supervisor_otlp_agent_endpoint", - &self.supervisor_otlp_agent_endpoint, - ) + .field("supervisor_otlp", &self.supervisor_otlp) .field("supervisor_gateway_name", &self.supervisor_gateway_name) .field("state_dir", &self.state_dir) .field("launcher_bin", &self.launcher_bin) @@ -508,9 +495,7 @@ impl Default for VmDriverConfig { fn default() -> Self { Self { grpc_endpoint: String::new(), - supervisor_otlp_endpoint: None, - supervisor_ocsf_export: None, - supervisor_otlp_agent_endpoint: None, + supervisor_otlp: openshell_core::sandbox_env::SupervisorOtlp::default(), supervisor_gateway_name: None, allow_driver_config: false, resource_admission: @@ -1495,20 +1480,7 @@ impl VmDriver { openshell_core::sandbox_env::TELEMETRY_ENABLED, openshell_core::telemetry::enabled_env_value(), ); - if let Some(endpoint) = &self.config.supervisor_otlp_endpoint { - command - .env(openshell_core::sandbox_env::OTLP_ENDPOINT, endpoint) - .envs(openshell_otel::current_trace_context_environment()); - } - if let Some(export) = &self.config.supervisor_ocsf_export { - command.envs(export.environment()); - } - if let Some(endpoint) = &self.config.supervisor_otlp_agent_endpoint { - command.env(openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT, endpoint); - } - if let Some(name) = &self.config.supervisor_gateway_name { - command.env(openshell_core::sandbox_env::GATEWAY_NAME, name); - } + command.envs(supervisor_otlp_environment(&self.config)); if let Some(server_name) = gateway_tls_server_name { command.env( openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, @@ -8102,6 +8074,20 @@ fn isolate_host_control_environment(command: &mut Command) { command.env_clear(); } +/// Environment that lets the supervisor export spans, joining the current +/// trace, OCSF events, and relayed workload traces, identifying the gateway +/// that launched it. +fn supervisor_otlp_environment(config: &VmDriverConfig) -> Vec<(&'static str, String)> { + let mut environment = config.supervisor_otlp.environment(); + if config.supervisor_otlp.traces.is_some() { + environment.extend(openshell_otel::current_trace_context_environment()); + } + if let Some(name) = &config.supervisor_gateway_name { + environment.push((openshell_core::sandbox_env::GATEWAY_NAME, name.clone())); + } + environment +} + #[tracing::instrument( name = "vm.launch", skip(command), @@ -12705,6 +12691,75 @@ mod tests { ); } + #[test] + fn supervisor_otlp_environment_carries_only_configured_signals() { + use openshell_core::sandbox_env::{ + GATEWAY_NAME, OTLP_AGENT_ENDPOINT, OTLP_AGENT_PROTOCOL, OTLP_ENDPOINT, + OTLP_OCSF_ENDPOINT, OTLP_OCSF_PROTOCOL, OTLP_PROTOCOL, OtlpExport, OtlpProtocol, + }; + let export = |endpoint: &str, protocol| { + Some(OtlpExport { + endpoint: endpoint.to_string(), + protocol, + }) + }; + + let mut config = VmDriverConfig::default(); + assert!(supervisor_otlp_environment(&config).is_empty()); + + config.supervisor_otlp.traces = export("http://127.0.0.1:4317", OtlpProtocol::Grpc); + let environment = supervisor_otlp_environment(&config); + assert_eq!( + environment[0], + (OTLP_ENDPOINT, "http://127.0.0.1:4317".to_string()) + ); + assert!(environment.iter().all(|(name, _)| *name != OTLP_PROTOCOL)); + + config.supervisor_otlp.traces = export("http://127.0.0.1:4318", OtlpProtocol::HttpProtobuf); + assert_eq!( + supervisor_otlp_environment(&config)[1], + (OTLP_PROTOCOL, "http/protobuf".to_string()) + ); + + config.supervisor_otlp.traces = None; + config.supervisor_otlp.ocsf = export("http://siem:4317", OtlpProtocol::Grpc); + assert_eq!( + supervisor_otlp_environment(&config), + [(OTLP_OCSF_ENDPOINT, "http://siem:4317".to_string()),] + ); + + config.supervisor_otlp.ocsf = export("http://siem:4318", OtlpProtocol::HttpProtobuf); + assert_eq!( + supervisor_otlp_environment(&config), + [ + (OTLP_OCSF_ENDPOINT, "http://siem:4318".to_string()), + (OTLP_OCSF_PROTOCOL, "http/protobuf".to_string()), + ] + ); + + config.supervisor_otlp.ocsf = None; + config.supervisor_otlp.agent = export("http://agents:4317", OtlpProtocol::Grpc); + assert_eq!( + supervisor_otlp_environment(&config), + [(OTLP_AGENT_ENDPOINT, "http://agents:4317".to_string())] + ); + + config.supervisor_otlp.agent = export("http://agents:4318", OtlpProtocol::HttpProtobuf); + assert_eq!( + supervisor_otlp_environment(&config), + [ + (OTLP_AGENT_ENDPOINT, "http://agents:4318".to_string()), + (OTLP_AGENT_PROTOCOL, "http/protobuf".to_string()), + ] + ); + + config.supervisor_gateway_name = Some("prod-west".to_string()); + assert_eq!( + supervisor_otlp_environment(&config).last(), + Some(&(GATEWAY_NAME, "prod-west".to_string())) + ); + } + #[test] fn host_control_environment_contains_only_explicit_values() { let mut command = Command::new("openshell-sandbox"); diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 0ab4743771..455c420f23 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -6,6 +6,7 @@ use futures::Stream; use miette::{IntoDiagnostic, Result}; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::sandbox_env::SupervisorOtlpArgs; #[cfg(target_os = "macos")] use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; use openshell_driver_vm::{ @@ -97,21 +98,8 @@ struct Args { #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] log_level: String, - #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] - otlp_endpoint: Option, - - /// OTLP/gRPC collector endpoint for supervisor OCSF event export. - #[arg(long, env = "OPENSHELL_OTLP_OCSF_ENDPOINT")] - otlp_ocsf_endpoint: Option, - - /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. - #[arg(long, env = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION", hide = true)] - otlp_ocsf_schema_version: Option, - - /// OTLP/gRPC collector endpoint that supervisors relay workload traces - /// to. Absent keeps the agent trace relay off. - #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")] - otlp_agent_endpoint: Option, + #[command(flatten)] + otlp: SupervisorOtlpArgs, #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, @@ -308,17 +296,14 @@ async fn main() -> Result<()> { let _tracing = openshell_otel::install_driver_tracing( openshell_driver_vm::otel_tracing::TRACING, openshell_otel::DriverTracingConfig { - endpoint: args.otlp_endpoint.as_deref(), + endpoint: args.otlp.endpoint.as_deref(), + protocol: args.otlp.protocol.into(), gateway_name: args.gateway_name.as_deref(), service_version: VERSION, log_level: &args.log_level, }, ); - if args.otlp_ocsf_schema_version.is_some() { - tracing::warn!( - "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION is deprecated and ignored; OpenShell emits OCSF 1.8.0" - ); - } + args.otlp.warn_deprecated(); let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?; @@ -334,19 +319,15 @@ async fn main() -> Result<()> { ); } + let supervisor_otlp = args.otlp.supervisor_otlp(); let driver = VmDriver::new(VmDriverConfig { allow_driver_config: args.admission_config_json.allow_driver_config, resource_admission: args.admission_config_json.resource_admission.clone(), grpc_endpoint: args .grpc_endpoint .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, - supervisor_otlp_endpoint: args.otlp_endpoint.clone(), - supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), + supervisor_otlp, supervisor_gateway_name: args.gateway_name.clone(), - supervisor_ocsf_export: args - .otlp_ocsf_endpoint - .clone() - .map(|endpoint| openshell_core::sandbox_env::OcsfOtlpExport { endpoint }), state_dir: args.state_dir.clone(), launcher_bin: None, default_image: args.default_image.clone(), diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 33ec8924a0..26758f85bc 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -43,24 +43,6 @@ use openshell_server::ComputeDriverRegistration; use openshell_server::ComputeDriverRegistry; use openshell_server::config_file::OtlpConfig; -#[cfg(all( - not(target_os = "windows"), - any( - feature = "compute-driver-docker", - feature = "compute-driver-kubernetes", - feature = "compute-driver-podman" - ) -))] -/// The OCSF event export that drivers pass to supervisors, from `[otlp.ocsf]`. -fn supervisor_ocsf_export( - otlp: Option<&OtlpConfig>, -) -> Option { - let ocsf = otlp?.ocsf.as_ref()?; - Some(openshell_core::sandbox_env::OcsfOtlpExport { - endpoint: ocsf.endpoint.clone(), - }) -} - fn prefer_environment(configured: T, environment: Option) -> T { environment.unwrap_or(configured) } @@ -299,15 +281,10 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { Option, )> { let mut config = kubernetes_config(context.config_context())?; - config.supervisor_otlp_endpoint = context + config.supervisor_otlp = context .otlp_config() - .and_then(OtlpConfig::traces_endpoint) - .map(str::to_string); - config.supervisor_ocsf_export = supervisor_ocsf_export(context.otlp_config()); - config.supervisor_otlp_agent_endpoint = context - .otlp_config() - .and_then(OtlpConfig::agent_endpoint) - .map(str::to_string); + .map(OtlpConfig::supervisor_otlp) + .unwrap_or_default(); config.supervisor_gateway_name = Some(context.gateway_name().to_string()); // With the gateway supervisor topology, the gateway owns the shared // supervisor's assignments and token; the driver adapts them to @@ -400,15 +377,10 @@ impl openshell_server::ComputeDriverFactory for DockerFactory { context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config = docker_config(context.config_context())?; - config.supervisor_otlp_endpoint = context - .otlp_config() - .and_then(OtlpConfig::traces_endpoint) - .map(str::to_string); - config.supervisor_ocsf_export = supervisor_ocsf_export(context.otlp_config()); - config.supervisor_otlp_agent_endpoint = context + config.supervisor_otlp = context .otlp_config() - .and_then(OtlpConfig::agent_endpoint) - .map(str::to_string); + .map(OtlpConfig::supervisor_otlp) + .unwrap_or_default(); config.supervisor_gateway_name = Some(context.gateway_name().to_string()); require_guest_tls_for_local_driver(&context, "docker")?; apply_guest_tls(&mut config.guest_tls_ca, context.guest_tls_ca()); @@ -476,15 +448,10 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { context: openshell_server::ComputeDriverBuildContext<'_>, ) -> openshell_core::Result { let mut config = podman_config(context.config_context())?; - config.supervisor_otlp_endpoint = context - .otlp_config() - .and_then(OtlpConfig::traces_endpoint) - .map(str::to_string); - config.supervisor_ocsf_export = supervisor_ocsf_export(context.otlp_config()); - config.supervisor_otlp_agent_endpoint = context + config.supervisor_otlp = context .otlp_config() - .and_then(OtlpConfig::agent_endpoint) - .map(str::to_string); + .map(OtlpConfig::supervisor_otlp) + .unwrap_or_default(); config.supervisor_gateway_name = Some(context.gateway_name().to_string()); require_guest_tls_for_local_driver(&context, "podman")?; apply_guest_tls(&mut config.guest_tls_ca, context.guest_tls_ca()); diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 2f9ea5cc8e..fcfc8ffe83 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -31,6 +31,7 @@ use hyper_util::rt::TokioIo; use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; +use openshell_core::sandbox_env::{self, OtlpExport, OtlpProtocol}; use openshell_core::{Error, NetworkSupervisorTrustBundle, Result, UpstreamProxyConfig}; #[cfg(unix)] use openshell_otel::TraceContextInterceptor; @@ -688,22 +689,49 @@ fn append_vm_proxy_and_spiffe_args(command: &mut Command, config: &VmComputeConf } } -fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) { - let traces_endpoint = otlp_config.and_then(OtlpConfig::traces_endpoint); - let ocsf = otlp_config.and_then(|config| config.ocsf.as_ref()); - if let Some(endpoint) = traces_endpoint { - command.arg("--otlp-endpoint").arg(endpoint); - } - if let Some(ocsf) = ocsf { - command.arg("--otlp-ocsf-endpoint").arg(&ocsf.endpoint); - } - let agent_endpoint = otlp_config.and_then(OtlpConfig::agent_endpoint); - if let Some(endpoint) = agent_endpoint { - command.arg("--otlp-agent-endpoint").arg(endpoint); +fn append_otlp_export_args( + command: &mut Command, + export: Option<&OtlpExport>, + endpoint_flag: &str, + protocol_flag: &str, +) { + if let Some(export) = export { + command + .arg(format!("--{endpoint_flag}")) + .arg(&export.endpoint); + if export.protocol != OtlpProtocol::Grpc { + command + .arg(format!("--{protocol_flag}")) + .arg(export.protocol.as_str()); + } } +} + +fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gateway_name: &str) { + let otlp = otlp_config + .map(OtlpConfig::supervisor_otlp) + .unwrap_or_default(); + append_otlp_export_args( + command, + otlp.traces.as_ref(), + sandbox_env::OTLP_ENDPOINT_FLAG, + sandbox_env::OTLP_PROTOCOL_FLAG, + ); + append_otlp_export_args( + command, + otlp.ocsf.as_ref(), + sandbox_env::OTLP_OCSF_ENDPOINT_FLAG, + sandbox_env::OTLP_OCSF_PROTOCOL_FLAG, + ); + append_otlp_export_args( + command, + otlp.agent.as_ref(), + sandbox_env::OTLP_AGENT_ENDPOINT_FLAG, + sandbox_env::OTLP_AGENT_PROTOCOL_FLAG, + ); // The driver names its own spans with the gateway name, and supervisors // name their spans, OCSF records, and relayed traces with it. - if traces_endpoint.is_some() || ocsf.is_some() || agent_endpoint.is_some() { + if !otlp.is_empty() { command.arg("--gateway-name").arg(gateway_name); } } @@ -802,8 +830,11 @@ mod tests { prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, validate_vm_sandbox_identity, }; + use openshell_core::sandbox_env::OtlpProtocol; use openshell_core::{NetworkSupervisorTrustBundle, UpstreamProxyConfig}; - use openshell_server::config_file::{OcsfOtlpConfig, OtlpConfig, OtlpTracesConfig}; + use openshell_server::config_file::{ + AgentOtlpConfig, OcsfOtlpConfig, OtlpCollectorConfig, OtlpConfig, OtlpTracesConfig, + }; use std::os::unix::fs::PermissionsExt; use std::os::unix::net::UnixListener as StdUnixListener; use std::path::PathBuf; @@ -853,7 +884,10 @@ mod tests { Some(&OtlpConfig { service_name: Some("custom-gateway".to_string()), traces: Some(OtlpTracesConfig { - endpoint: "http://collector.internal:4317".to_string(), + collector: OtlpCollectorConfig { + endpoint: "http://collector.internal:4317".to_string(), + protocol: OtlpProtocol::Grpc, + }, }), ocsf: None, agent: None, @@ -875,6 +909,37 @@ mod tests { "production-us-west" ] ); + + let mut command = tokio::process::Command::new("openshell-driver-vm"); + append_otlp_args( + &mut command, + Some(&OtlpConfig { + service_name: None, + traces: Some(OtlpTracesConfig { + collector: OtlpCollectorConfig { + endpoint: "http://collector.internal:4318".to_string(), + protocol: OtlpProtocol::HttpProtobuf, + }, + }), + ocsf: None, + agent: None, + }), + "production-us-west", + ); + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args[..4], + [ + "--otlp-endpoint", + "http://collector.internal:4318", + "--otlp-protocol", + "http/protobuf", + ] + ); } #[test] @@ -886,8 +951,11 @@ mod tests { service_name: None, traces: None, ocsf: Some(OcsfOtlpConfig { - endpoint: "http://siem.internal:4317".to_string(), - schema_version: None, + collector: OtlpCollectorConfig { + endpoint: "http://siem.internal:4318".to_string(), + protocol: OtlpProtocol::HttpProtobuf, + }, + schema_version: Some("2.0".into()), }), agent: None, }), @@ -903,7 +971,9 @@ mod tests { args, [ "--otlp-ocsf-endpoint", - "http://siem.internal:4317", + "http://siem.internal:4318", + "--otlp-ocsf-protocol", + "http/protobuf", "--gateway-name", "production-us-west" ] @@ -919,8 +989,11 @@ mod tests { service_name: None, traces: None, ocsf: None, - agent: Some(openshell_server::config_file::AgentOtlpConfig { - endpoint: "http://agents.internal:4317".to_string(), + agent: Some(AgentOtlpConfig { + collector: OtlpCollectorConfig { + endpoint: "http://agents.internal:4318".to_string(), + protocol: OtlpProtocol::HttpProtobuf, + }, }), }), "production-us-west", @@ -935,7 +1008,9 @@ mod tests { args, [ "--otlp-agent-endpoint", - "http://agents.internal:4317", + "http://agents.internal:4318", + "--otlp-agent-protocol", + "http/protobuf", "--gateway-name", "production-us-west" ] diff --git a/crates/openshell-otel-test-support/Cargo.toml b/crates/openshell-otel-test-support/Cargo.toml index b9f242f0f1..05b67b233d 100644 --- a/crates/openshell-otel-test-support/Cargo.toml +++ b/crates/openshell-otel-test-support/Cargo.toml @@ -11,6 +11,11 @@ license.workspace = true repository.workspace = true [dependencies] +bytes = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true, features = ["tokio"] } openshell-otel = { path = "../openshell-otel" } opentelemetry-proto = { version = "0.33", default-features = false, features = ["gen-tonic", "logs", "trace"] } prost = { workspace = true } diff --git a/crates/openshell-otel-test-support/src/http.rs b/crates/openshell-otel-test-support/src/http.rs new file mode 100644 index 0000000000..fe2db25909 --- /dev/null +++ b/crates/openshell-otel-test-support/src/http.rs @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Loopback OTLP/HTTP collector that records every request. + +use std::time::Duration; + +use bytes::Bytes; +use http_body_util::BodyExt as _; + +/// One request received by [`OtlpHttpTestServer`]. +#[derive(Clone, Debug)] +pub struct HttpExport { + pub method: http::Method, + pub path: String, + pub headers: http::HeaderMap, + pub body: Bytes, +} + +/// A loopback OTLP/HTTP collector that records every request and answers +/// each with a fixed status. +pub struct OtlpHttpTestServer { + endpoint: String, + exports: tokio::sync::mpsc::UnboundedReceiver, + task: tokio::task::JoinHandle<()>, +} + +impl OtlpHttpTestServer { + /// Starts the collector on an ephemeral loopback port, answering 200. + pub async fn start() -> Self { + Self::start_with_status(http::StatusCode::OK).await + } + + /// Starts the collector, answering every request with `status`. + pub async fn start_with_status(status: http::StatusCode) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("OTLP/HTTP test collector should bind a loopback listener"); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let (tx, exports) = tokio::sync::mpsc::unbounded_channel(); + let task = tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + continue; + }; + let tx = tx.clone(); + tokio::spawn(async move { + let service = hyper::service::service_fn( + move |request: http::Request| { + let tx = tx.clone(); + async move { + let (parts, body) = request.into_parts(); + let body = body.collect().await?.to_bytes(); + let _ = tx.send(HttpExport { + method: parts.method, + path: parts.uri.path().to_string(), + headers: parts.headers, + body, + }); + let mut response = + http::Response::new(http_body_util::Full::new(Bytes::new())); + *response.status_mut() = status; + Ok::<_, hyper::Error>(response) + } + }, + ); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(hyper_util::rt::TokioIo::new(stream), service) + .await; + }); + } + }); + Self { + endpoint, + exports, + task, + } + } + + #[must_use] + pub fn endpoint(&self) -> &str { + &self.endpoint + } + + /// The next request, waiting up to five seconds for it. + pub async fn next_export(&mut self) -> HttpExport { + tokio::time::timeout(Duration::from_secs(5), self.exports.recv()) + .await + .expect("OTLP/HTTP export should arrive") + .expect("OTLP/HTTP test collector is running") + } +} + +impl Drop for OtlpHttpTestServer { + fn drop(&mut self) { + self.task.abort(); + } +} diff --git a/crates/openshell-otel-test-support/src/lib.rs b/crates/openshell-otel-test-support/src/lib.rs index 4058b5a190..ea34843649 100644 --- a/crates/openshell-otel-test-support/src/lib.rs +++ b/crates/openshell-otel-test-support/src/lib.rs @@ -3,6 +3,10 @@ //! Shared OTLP collector fixture for `OpenShell` tracing tests. +mod http; + +pub use http::{HttpExport, OtlpHttpTestServer}; + /// Helpers for tests that exercise the supervisor's agent trace relay from /// the agent side: the attribution keys the relay sets and a builder for the /// protobuf export request an SDK would send. @@ -104,6 +108,7 @@ pub async fn assert_compute_driver_tracing( let collector = OtlpTestServer::start().await; let (provider, error) = descriptor.provider_for( Some(collector.endpoint()), + openshell_otel::Protocol::Grpc, service_version, Some("test-gateway"), Some(descriptor.compute_driver()), diff --git a/crates/openshell-otel/Cargo.toml b/crates/openshell-otel/Cargo.toml index 3a85893cfb..b174f65871 100644 --- a/crates/openshell-otel/Cargo.toml +++ b/crates/openshell-otel/Cargo.toml @@ -11,13 +11,18 @@ license.workspace = true repository.workspace = true [dependencies] +async-trait = "0.1" +bytes = { workspace = true } futures = { workspace = true } http = { workspace = true } +http-body-util = { workspace = true } +hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } metrics = { workspace = true } openshell-ocsf = { path = "../openshell-ocsf" } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["logs"] } -opentelemetry-otlp = { workspace = true, features = ["logs"] } +opentelemetry-http = { workspace = true } +opentelemetry-otlp = { workspace = true, features = ["gzip-http", "http-proto", "logs"] } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } @@ -28,7 +33,10 @@ tonic = { workspace = true } tower-http = { workspace = true } [dev-dependencies] +openshell-otel-test-support = { path = "../openshell-otel-test-support" } +opentelemetry-proto = { workspace = true, features = ["logs"] } opentelemetry_sdk = { workspace = true, features = ["testing"] } +prost = { workspace = true } tokio = { workspace = true, features = ["test-util"] } [lints] diff --git a/crates/openshell-otel/src/driver.rs b/crates/openshell-otel/src/driver.rs index b9c19298d5..b5e9cca555 100644 --- a/crates/openshell-otel/src/driver.rs +++ b/crates/openshell-otel/src/driver.rs @@ -73,16 +73,29 @@ impl ComputeDriverTracing { pub fn provider_for( self, endpoint: Option<&str>, + protocol: crate::Protocol, service_version: &'static str, gateway_name: Option<&str>, compute_driver: Option<&str>, ) -> (Option, Option) { - crate::provider_for(endpoint.map(|endpoint| OtlpExportConfig { - endpoint, - service_name: ServiceName::Fixed(self.service_name), - service_version: Some(service_version), - resource_attributes: crate::gateway_resource_attributes(gateway_name, compute_driver), - })) + let provider = endpoint.map(|endpoint| { + crate::OtlpTarget::new(&OtlpExportConfig { + endpoint, + service_name: ServiceName::Fixed(self.service_name), + service_version: Some(service_version), + resource_attributes: crate::gateway_resource_attributes( + gateway_name, + compute_driver, + ), + })? + .with_protocol(protocol) + .tracer_provider() + }); + match provider { + None => (None, None), + Some(Ok(provider)) => (Some(provider), None), + Some(Err(error)) => (None, Some(error)), + } } pub fn layer(self, provider: &SdkTracerProvider) -> crate::OtlpLayer @@ -187,6 +200,7 @@ pub struct DriverTracingHandle { #[derive(Debug, Clone, Copy)] pub struct DriverTracingConfig<'a> { pub endpoint: Option<&'a str>, + pub protocol: crate::Protocol, pub gateway_name: Option<&'a str>, pub service_version: &'static str, pub log_level: &'a str, @@ -210,6 +224,7 @@ pub fn install_driver_tracing( ) -> DriverTracingHandle { let (provider, setup_error) = descriptor.provider_for( config.endpoint, + config.protocol, config.service_version, config.gateway_name, Some(descriptor.compute_driver()), @@ -243,6 +258,28 @@ mod tests { assert_eq!(TRACING.in_process_targets()[1], "openshell_otel"); } + #[tokio::test(flavor = "multi_thread")] + async fn driver_provider_exports_over_the_configured_protocol() { + use opentelemetry::trace::{Tracer as _, TracerProvider as _}; + + const TRACING: ComputeDriverTracing = crate::compute_driver_tracing!(); + let mut collector = openshell_otel_test_support::OtlpHttpTestServer::start().await; + let (provider, error) = TRACING.provider_for( + Some(collector.endpoint()), + crate::Protocol::HttpProtobuf, + "1.2.3", + Some("test-gateway"), + Some("docker"), + ); + assert!(error.is_none(), "unexpected setup error: {error:?}"); + let provider = provider.expect("provider"); + provider.tracer("test").in_span("driver-span", |_| {}); + let flush = tokio::task::spawn_blocking(move || provider.force_flush()); + + assert_eq!(collector.next_export().await.path, "/v1/traces"); + flush.await.unwrap().expect("flush succeeds"); + } + #[tokio::test] async fn disabled_stream_tracing_returns_the_original_box() { let stream: BoxGrpcStream<()> = Box::pin(futures::stream::empty()); diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index 81f0f7ff08..47332df7f5 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -8,6 +8,7 @@ mod grpc; mod logs; mod ocsf; mod propagation; +mod transport; pub use driver::{ BoxGrpcStream, ComputeDriverTracing, DriverTracingConfig, DriverTracingHandle, @@ -25,12 +26,16 @@ pub use propagation::{ TraceContextInterceptor, current_trace_context_carrier, current_trace_context_environment, set_parent_from_environment, set_parent_from_trace_context, trace_context_environment, }; +pub use transport::{ + CONNECT_TIMEOUT, OtlpHttpClient, TCP_KEEPALIVE_IDLE, TCP_KEEPALIVE_INTERVAL, + TCP_KEEPALIVE_RETRIES, signal_url, +}; use opentelemetry::KeyValue; use opentelemetry::trace::TracerProvider as _; pub use opentelemetry_otlp::LogExporter; use opentelemetry_otlp::tonic_types::transport::ClientTlsConfig; -use opentelemetry_otlp::{SpanExporter, WithExportConfig, WithTonicConfig}; +use opentelemetry_otlp::{SpanExporter, WithExportConfig, WithHttpConfig, WithTonicConfig}; use opentelemetry_sdk::Resource; use opentelemetry_sdk::trace::SdkTracer; pub use opentelemetry_sdk::trace::SdkTracerProvider; @@ -123,7 +128,15 @@ pub enum ServiceName<'a> { EnvironmentOr(&'a str), } -/// Inputs for OTLP/gRPC export. +/// OTLP transport to a collector. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Protocol { + #[default] + Grpc, + HttpProtobuf, +} + +/// Inputs for OTLP export. #[derive(Debug, Clone)] pub struct OtlpExportConfig<'a> { pub endpoint: &'a str, @@ -146,6 +159,18 @@ pub enum SetupError { #[error("failed to build the OTLP exporter: {0}")] Exporter(#[from] opentelemetry_otlp::ExporterBuildError), + + #[error("invalid OTLP endpoint {endpoint:?}: {source}")] + InvalidSignalUrl { + endpoint: String, + source: http::Error, + }, + + #[error("OTLP/HTTP export over https is not supported")] + HttpsUnsupported, + + #[error("OTLP/HTTP export needs a tokio runtime")] + NoRuntime, } fn resource_attributes(config: &OtlpExportConfig<'_>) -> Vec { @@ -206,12 +231,13 @@ pub fn validated_endpoint(endpoint: &str) -> Result<(&str, http::Uri), SetupErro Ok((endpoint, uri)) } -/// A validated OTLP/gRPC collector endpoint and the resource that every -/// signal exported to it describes. +/// A validated OTLP collector endpoint and the resource that every signal +/// exported to it describes. #[derive(Debug, Clone)] pub struct OtlpTarget { endpoint: String, tls: bool, + protocol: Protocol, resource: Resource, } @@ -222,10 +248,31 @@ impl OtlpTarget { Ok(Self { endpoint: endpoint.to_string(), tls: uri.scheme() == Some(&http::uri::Scheme::HTTPS), + protocol: Protocol::Grpc, resource: resource_for(config), }) } + /// Export over `protocol` instead of gRPC. + #[must_use] + pub fn with_protocol(mut self, protocol: Protocol) -> Self { + self.protocol = protocol; + self + } + + /// The URL of `signal_path` under the endpoint, for OTLP/HTTP export. + pub(crate) fn http_signal_url(&self, signal_path: &str) -> Result { + if self.tls { + return Err(SetupError::HttpsUnsupported); + } + signal_url(&self.endpoint, signal_path) + .map(|url| url.to_string()) + .map_err(|source| SetupError::InvalidSignalUrl { + endpoint: self.endpoint.clone(), + source, + }) + } + /// Point an exporter builder at this target, over TLS for `https://`. pub(crate) fn configure(&self, builder: B) -> B where @@ -252,9 +299,19 @@ impl OtlpTarget { /// Build a trace provider that exports spans to this target in batches. pub fn tracer_provider(&self) -> Result { - let exporter = self - .configure(SpanExporter::builder().with_tonic()) - .build()?; + let exporter = match self.protocol { + Protocol::Grpc => self + .configure(SpanExporter::builder().with_tonic()) + .build()?, + Protocol::HttpProtobuf => SpanExporter::builder() + .with_http() + .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) + .with_endpoint(self.http_signal_url("v1/traces")?) + .with_http_client(OtlpHttpClient::new(transport::export_timeout( + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + ))?) + .build()?, + }; Ok(SdkTracerProvider::builder() .with_batch_exporter(exporter) .with_resource(self.resource.clone()) @@ -262,7 +319,7 @@ impl OtlpTarget { } } -/// Build an OTLP/gRPC trace provider. +/// Build an OTLP trace provider. pub fn build_provider(config: &OtlpExportConfig<'_>) -> Result { OtlpTarget::new(config)?.tracer_provider() } diff --git a/crates/openshell-otel/src/logs.rs b/crates/openshell-otel/src/logs.rs index cd67d4034c..09eab68446 100644 --- a/crates/openshell-otel/src/logs.rs +++ b/crates/openshell-otel/src/logs.rs @@ -1,13 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! OTLP/gRPC log export support. +//! OTLP log export support. use http::{HeaderMap, HeaderName, HeaderValue}; use opentelemetry_otlp::tonic_types::metadata::MetadataMap; -use opentelemetry_otlp::{LogExporter, WithTonicConfig}; +use opentelemetry_otlp::{LogExporter, WithExportConfig, WithHttpConfig, WithTonicConfig}; -use crate::{OtlpTarget, SetupError}; +use crate::{OtlpHttpClient, OtlpTarget, Protocol, SetupError}; impl OtlpTarget { /// Build a log exporter for this target. Callers attach @@ -23,19 +23,38 @@ impl OtlpTarget { &self, headers: Option<&str>, ) -> Result { - let mut builder = self.configure(LogExporter::builder().with_tonic()); - if let Some(headers) = headers { - builder = builder.with_metadata(header_metadata(headers)); + let headers = headers.map(parse_headers).unwrap_or_default(); + match self.protocol { + Protocol::Grpc => Ok(self + .configure(LogExporter::builder().with_tonic()) + .with_metadata(MetadataMap::from_headers(headers)) + .build()?), + Protocol::HttpProtobuf => Ok(LogExporter::builder() + .with_http() + .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) + .with_endpoint(self.http_signal_url("v1/logs")?) + .with_http_client(OtlpHttpClient::new(crate::transport::export_timeout( + "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", + ))?) + // The SDK percent-decodes these values again. + .with_headers( + headers + .iter() + .filter_map(|(name, value)| { + Some((name.to_string(), value.to_str().ok()?.replace('%', "%25"))) + }) + .collect(), + ) + .build()?), } - Ok(builder.build()?) } } /// Parse headers the way the SDK parses `OTEL_EXPORTER_OTLP_HEADERS`: /// comma-separated `key=value` pairs with percent-encoded values. Entries -/// that are empty or not valid gRPC metadata are skipped. -fn header_metadata(headers: &str) -> MetadataMap { - let mut metadata = HeaderMap::new(); +/// that are empty or not valid headers are skipped. +fn parse_headers(headers: &str) -> HeaderMap { + let mut map = HeaderMap::new(); for (key, value) in headers.split_terminator(',').filter_map(|entry| { let (key, value) = entry.trim().split_once('=')?; let (key, value) = (key.trim(), value.trim()); @@ -46,10 +65,10 @@ fn header_metadata(headers: &str) -> MetadataMap { HeaderName::try_from(key), HeaderValue::try_from(value.as_str()), ) { - metadata.insert(key, value); + map.insert(key, value); } } - MetadataMap::from_headers(metadata) + map } fn percent_decode(value: &str) -> Option { @@ -100,7 +119,7 @@ mod tests { #[test] fn headers_parse_like_the_sdk_environment_variable() { - let metadata = super::header_metadata( + let metadata = super::parse_headers( "authorization=Bearer%20secret, x-tenant = blue ,=empty-key,no-value=,bad", ); assert_eq!( @@ -113,7 +132,7 @@ mod tests { #[test] fn malformed_percent_encoding_is_kept_verbatim() { - let metadata = super::header_metadata("token=abc%zz"); + let metadata = super::parse_headers("token=abc%zz"); assert_eq!(metadata.get("token").unwrap().to_str().unwrap(), "abc%zz"); } diff --git a/crates/openshell-otel/src/transport.rs b/crates/openshell-otel/src/transport.rs new file mode 100644 index 0000000000..c14e6c10f1 --- /dev/null +++ b/crates/openshell-otel/src/transport.rs @@ -0,0 +1,362 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP transport shared by OTLP/HTTP exporters, and the TCP settings that +//! every `OpenShell` collector connection uses. + +use std::time::Duration; + +use bytes::Bytes; +use http::uri::PathAndQuery; +use http_body_util::{BodyExt as _, Full, Limited}; +use hyper_util::client::legacy::Client; +use hyper_util::client::legacy::connect::HttpConnector; +use hyper_util::rt::TokioExecutor; +use opentelemetry_http::{HttpClient, HttpError}; + +/// TCP connect deadline for a collector, applied per resolved address. +pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(2); +/// TCP keepalive idle time, probe interval, and probe count, so a collector +/// that vanished without a FIN is noticed on an idle connection, in about 30 +/// seconds on Linux. +pub const TCP_KEEPALIVE_IDLE: Duration = Duration::from_secs(15); +pub const TCP_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5); +pub const TCP_KEEPALIVE_RETRIES: u32 = 3; +/// Largest collector response body read. +const MAX_RESPONSE_BYTES: usize = 64 * 1024; + +/// Per-request export deadline when the environment sets none, as for the +/// SDK's own HTTP clients. +const DEFAULT_EXPORT_TIMEOUT: Duration = Duration::from_secs(10); + +/// The export deadline from `signal_var` (such as +/// `OTEL_EXPORTER_OTLP_TRACES_TIMEOUT`), else `OTEL_EXPORTER_OTLP_TIMEOUT`, +/// in milliseconds. +pub fn export_timeout(signal_var: &str) -> Duration { + export_timeout_from(signal_var, |name| std::env::var(name).ok()) +} + +fn export_timeout_from(signal_var: &str, var: impl Fn(&str) -> Option) -> Duration { + [signal_var, "OTEL_EXPORTER_OTLP_TIMEOUT"] + .into_iter() + .find_map(|name| var(name)?.trim().parse().ok()) + .map_or(DEFAULT_EXPORT_TIMEOUT, Duration::from_millis) +} + +/// The URL of `signal_path` (such as `v1/traces`) under `endpoint`, treated +/// as a base URL the way SDKs treat `OTEL_EXPORTER_OTLP_ENDPOINT`. A query +/// string is dropped. +pub fn signal_url(endpoint: &str, signal_path: &str) -> Result { + let mut parts = endpoint.parse::()?.into_parts(); + let base = parts + .path_and_query + .as_ref() + .map_or("", PathAndQuery::path) + .trim_end_matches('/'); + parts.path_and_query = Some(format!("{base}/{signal_path}").parse()?); + Ok(http::Uri::from_parts(parts)?) +} + +/// HTTP client for OTLP/HTTP collectors. +/// +/// Requests run on the tokio runtime the client was created in, so SDK +/// exporters can use it from their own export threads. +#[derive(Clone, Debug)] +pub struct OtlpHttpClient { + client: Client>, + runtime: tokio::runtime::Handle, + timeout: Duration, +} + +impl OtlpHttpClient { + /// Creates a client bound to the current tokio runtime whose requests + /// fail after `timeout`. Fails outside a runtime. + pub fn new(timeout: Duration) -> Result { + let runtime = + tokio::runtime::Handle::try_current().map_err(|_| crate::SetupError::NoRuntime)?; + let mut connector = HttpConnector::new(); + connector.set_connect_timeout(Some(CONNECT_TIMEOUT)); + connector.set_keepalive(Some(TCP_KEEPALIVE_IDLE)); + connector.set_keepalive_interval(Some(TCP_KEEPALIVE_INTERVAL)); + connector.set_keepalive_retries(Some(TCP_KEEPALIVE_RETRIES)); + connector.set_nodelay(true); + Ok(Self { + client: Client::builder(TokioExecutor::new()).build(connector), + runtime, + timeout, + }) + } + + /// Sends `request` and returns the response with its body, which fails + /// the request when it is incomplete or larger than 64 KiB. The request + /// stops when the timeout passes or the returned future is dropped. + pub async fn send( + &self, + request: http::Request, + ) -> Result, HttpError> { + let (client, timeout) = (self.client.clone(), self.timeout); + let mut task = AbortOnDrop(self.runtime.spawn(async move { + tokio::time::timeout(timeout, async move { + let response = client.request(request.map(Full::new)).await?; + let (parts, body) = response.into_parts(); + let body = Limited::new(body, MAX_RESPONSE_BYTES) + .collect() + .await? + .to_bytes(); + Ok(http::Response::from_parts(parts, body)) + }) + .await + .map_err(|_| HttpError::from(format!("no response within {timeout:?}")))? + })); + (&mut task.0).await? + } +} + +/// Aborts the task when dropped, unlike a bare `JoinHandle`, which detaches. +struct AbortOnDrop(tokio::task::JoinHandle); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +#[async_trait::async_trait] +impl HttpClient for OtlpHttpClient { + async fn send_bytes( + &self, + request: http::Request, + ) -> Result, HttpError> { + self.send(request).await + } +} + +#[cfg(test)] +mod tests { + use openshell_otel_test_support::OtlpHttpTestServer; + use opentelemetry::logs::{AnyValue, LogRecord as _, Logger as _, LoggerProvider as _}; + use opentelemetry::trace::{Tracer as _, TracerProvider as _}; + use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; + use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; + use prost::Message as _; + + use super::*; + use crate::{OtlpExportConfig, OtlpTarget, Protocol, ServiceName, SetupError}; + + /// A loopback listener that accepts connections and never answers. + async fn silent_collector() -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + #[allow(clippy::collection_is_never_read)] + let mut held = Vec::new(); + while let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + }); + (endpoint, task) + } + + /// A collector that answers each request with `head` and `body`, then + /// closes the connection. + async fn answering_collector( + head: &'static str, + body: Vec, + ) -> (String, tokio::task::JoinHandle<()>) { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + let mut request = [0; 4096]; + let _ = stream.read(&mut request).await; + let _ = stream.write_all(head.as_bytes()).await; + let _ = stream.write_all(&body).await; + } + }); + (endpoint, task) + } + + fn post(endpoint: &str) -> http::Request { + http::Request::post(signal_url(endpoint, "v1/traces").unwrap()) + .body(Bytes::new()) + .unwrap() + } + + #[tokio::test(flavor = "multi_thread")] + async fn requests_fail_after_the_timeout() { + let (endpoint, collector) = silent_collector().await; + let client = OtlpHttpClient::new(Duration::from_millis(200)).unwrap(); + let result = tokio::time::timeout(Duration::from_secs(5), client.send(post(&endpoint))) + .await + .expect("the client gives up on its own"); + assert!(result.is_err()); + collector.abort(); + } + + #[tokio::test(flavor = "multi_thread")] + async fn truncated_responses_fail() { + let (endpoint, collector) = answering_collector( + "HTTP/1.1 200 OK\r\ncontent-length: 100\r\n\r\n", + b"partial".to_vec(), + ) + .await; + let client = OtlpHttpClient::new(Duration::from_secs(5)).unwrap(); + assert!(client.send(post(&endpoint)).await.is_err()); + collector.abort(); + } + + #[tokio::test(flavor = "multi_thread")] + async fn oversized_responses_fail() { + let body = vec![b'x'; MAX_RESPONSE_BYTES + 1]; + let head = format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\n\r\n", body.len()); + let (endpoint, collector) = answering_collector(head.leak(), body).await; + let client = OtlpHttpClient::new(Duration::from_secs(5)).unwrap(); + assert!(client.send(post(&endpoint)).await.is_err()); + collector.abort(); + } + + #[tokio::test(flavor = "multi_thread")] + async fn abandoned_requests_stop() { + let (endpoint, collector) = silent_collector().await; + let client = OtlpHttpClient::new(Duration::from_mins(1)).unwrap(); + let alive = || { + tokio::runtime::Handle::current() + .metrics() + .num_alive_tasks() + }; + let before = alive(); + for _ in 0..3 { + let abandoned = + tokio::time::timeout(Duration::from_millis(100), client.send(post(&endpoint))) + .await; + assert!(abandoned.is_err(), "the collector never answers"); + } + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + alive() <= before, + "{} tasks outlived their callers", + alive() - before + ); + collector.abort(); + } + + fn http_target(endpoint: &str) -> OtlpTarget { + OtlpTarget::new(&OtlpExportConfig { + endpoint, + service_name: ServiceName::Fixed("openshell-test"), + service_version: None, + resource_attributes: Vec::new(), + }) + .unwrap() + .with_protocol(Protocol::HttpProtobuf) + } + + #[test] + fn export_timeouts_prefer_the_signal_variable() { + const SIGNAL: &str = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT"; + const GENERIC: &str = "OTEL_EXPORTER_OTLP_TIMEOUT"; + for (vars, expected) in [ + (vec![(SIGNAL, "30000")], Duration::from_secs(30)), + (vec![(GENERIC, "20000")], Duration::from_secs(20)), + ( + vec![(SIGNAL, "30000"), (GENERIC, "20000")], + Duration::from_secs(30), + ), + ( + vec![(SIGNAL, "soon"), (GENERIC, "20000")], + Duration::from_secs(20), + ), + (vec![], DEFAULT_EXPORT_TIMEOUT), + (vec![(SIGNAL, " 1500 ")], Duration::from_millis(1500)), + ] { + let lookup = |name: &str| { + vars.iter() + .find(|(var, _)| *var == name) + .map(|(_, value)| (*value).to_string()) + }; + assert_eq!(export_timeout_from(SIGNAL, lookup), expected, "{vars:?}"); + } + } + + #[test] + fn signal_urls_extend_the_endpoint_path() { + for (endpoint, url) in [ + ("http://collector:4318", "http://collector:4318/v1/traces"), + ("http://collector:4318/", "http://collector:4318/v1/traces"), + ("https://host/prefix", "https://host/prefix/v1/traces"), + ("https://host/prefix/", "https://host/prefix/v1/traces"), + ("http://host:4318/p?key=v", "http://host:4318/p/v1/traces"), + ] { + assert_eq!(signal_url(endpoint, "v1/traces").unwrap().to_string(), url); + } + assert!(signal_url("not a uri", "v1/traces").is_err()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn https_endpoints_fail_setup() { + for endpoint in ["https://collector:4318", "HTTPS://collector:4318"] { + assert!( + matches!( + http_target(endpoint).tracer_provider(), + Err(SetupError::HttpsUnsupported) + ), + "{endpoint}" + ); + assert!( + matches!( + http_target(endpoint).log_exporter_with_headers(None), + Err(SetupError::HttpsUnsupported) + ), + "{endpoint}" + ); + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn trace_provider_exports_over_http_from_the_batch_thread() { + let mut collector = OtlpHttpTestServer::start().await; + let provider = http_target(collector.endpoint()).tracer_provider().unwrap(); + provider.tracer("test").in_span("http-span", |_| {}); + let flush = tokio::task::spawn_blocking(move || provider.force_flush()); + + let export = collector.next_export().await; + assert_eq!(export.method, http::Method::POST); + assert_eq!(export.path, "/v1/traces"); + assert_eq!( + export.headers[http::header::CONTENT_TYPE], + "application/x-protobuf" + ); + let request = ExportTraceServiceRequest::decode(export.body).unwrap(); + let span = &request.resource_spans[0].scope_spans[0].spans[0]; + assert_eq!(span.name, "http-span"); + flush.await.unwrap().expect("flush succeeds"); + } + + #[tokio::test(flavor = "multi_thread")] + async fn log_exporter_sends_headers_over_http_from_a_blocking_thread() { + let mut collector = OtlpHttpTestServer::start().await; + let exporter = http_target(collector.endpoint()) + .log_exporter_with_headers(Some("authorization=Bearer%20token,x-token=abc%252F")) + .unwrap(); + let provider = opentelemetry_sdk::logs::SdkLoggerProvider::builder() + .with_simple_exporter(exporter) + .build(); + let emit = tokio::task::spawn_blocking(move || { + let logger = provider.logger("test"); + let mut record = logger.create_log_record(); + record.set_body(AnyValue::from("ocsf-event")); + logger.emit(record); + }); + + let export = collector.next_export().await; + assert_eq!(export.path, "/v1/logs"); + assert_eq!(export.headers["authorization"], "Bearer token"); + assert_eq!(export.headers["x-token"], "abc%2F", "decoded exactly once"); + let request = ExportLogsServiceRequest::decode(export.body).unwrap(); + let record = &request.resource_logs[0].scope_logs[0].log_records[0]; + assert!(format!("{:?}", record.body).contains("ocsf-event")); + emit.await.unwrap(); + } +} diff --git a/crates/openshell-otel/tests/http_gzip.rs b/crates/openshell-otel/tests/http_gzip.rs new file mode 100644 index 0000000000..7eafeb5862 --- /dev/null +++ b/crates/openshell-otel/tests/http_gzip.rs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The only test in its binary, since it sets the process environment. + +use openshell_otel::{OtlpExportConfig, OtlpTarget, Protocol, ServiceName}; +use openshell_otel_test_support::OtlpHttpTestServer; +use opentelemetry::trace::{Tracer as _, TracerProvider as _}; + +#[tokio::test(flavor = "multi_thread")] +#[allow(unsafe_code)] +async fn http_exports_honor_the_compression_variable() { + // Nothing else in this binary reads the environment. + unsafe { std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", "gzip") }; + let mut collector = OtlpHttpTestServer::start().await; + let provider = OtlpTarget::new(&OtlpExportConfig { + endpoint: collector.endpoint(), + service_name: ServiceName::Fixed("gzip-test"), + service_version: None, + resource_attributes: Vec::new(), + }) + .unwrap() + .with_protocol(Protocol::HttpProtobuf) + .tracer_provider() + .expect("gzip is supported over HTTP"); + provider.tracer("test").in_span("gzip-span", |_| {}); + let flush = tokio::task::spawn_blocking(move || provider.force_flush()); + + assert_eq!( + collector.next_export().await.headers["content-encoding"], + "gzip" + ); + flush.await.unwrap().expect("flush succeeds"); +} diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 9e681d70c9..9b9462cd67 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -13,7 +13,7 @@ repository.workspace = true [dependencies] openshell-crypto = { workspace = true, features = ["tls-kube", "tls-sqlx"] } openshell-bootstrap = { path = "../openshell-bootstrap" } -openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt", "oauth"] } +openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt", "oauth", "otel"] } openshell-driver-db-credstore = { path = "../openshell-driver-db-credstore" } openshell-driver-kubernetes-secrets = { path = "../openshell-driver-kubernetes-secrets" } openshell-driver-vault = { path = "../openshell-driver-vault" } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index c750794c1f..6a672c5aa5 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -851,8 +851,13 @@ async fn run_from_args( } else if let Some(otlp) = &otlp_export { info!( traces_endpoint = otlp.traces_endpoint().map(tracing::field::display), + traces_protocol = otlp + .traces_endpoint() + .map(|_| tracing::field::display(otlp.traces_protocol())), ocsf_endpoint = otlp.ocsf_endpoint().map(tracing::field::display), + ocsf_protocol = otlp.ocsf_protocol().map(tracing::field::display), agent_endpoint = otlp.agent_endpoint().map(tracing::field::display), + agent_protocol = otlp.agent_protocol().map(tracing::field::display), "OTLP exporting enabled" ); } diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index bbe45b267e..2c74c19360 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -29,6 +29,7 @@ use base64::Engine as _; use openshell_core::proto::{ SupervisorMiddlewareBinding, SupervisorMiddlewareBindingPolicy, SupervisorMiddlewareService, }; +use openshell_core::sandbox_env::OtlpProtocol; use openshell_core::{ ExtensionBindingPolicy, GatewayAuthConfig, GatewayInterceptorConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, @@ -380,29 +381,90 @@ fn serialize_enabled( impl OtlpConfig { /// The collector that receives traces, when trace export is enabled. pub fn traces_endpoint(&self) -> Option<&str> { - self.traces.as_ref().map(|traces| traces.endpoint.as_str()) + self.traces + .as_ref() + .map(|traces| traces.collector.endpoint.as_str()) } /// The collector that receives relayed workload traces, when the agent /// trace relay is enabled. pub fn agent_endpoint(&self) -> Option<&str> { - self.agent.as_ref().map(|agent| agent.endpoint.as_str()) + self.agent + .as_ref() + .map(|agent| agent.collector.endpoint.as_str()) + } + + /// The exports that compute drivers pass to supervisors. + pub fn supervisor_otlp(&self) -> openshell_core::sandbox_env::SupervisorOtlp { + openshell_core::sandbox_env::SupervisorOtlp { + traces: self.traces.as_ref().map(|traces| traces.collector.export()), + ocsf: self.ocsf.as_ref().map(|ocsf| ocsf.collector.export()), + agent: self.agent.as_ref().map(|agent| agent.collector.export()), + } } } -/// `[openshell.gateway.otlp.traces]`. +/// One OTLP collector. A lane table with its own `endpoint` describes its +/// own collector; one without inherits the whole `[openshell.gateway.otlp]` +/// collector. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct OtlpTracesConfig { - /// OTLP/gRPC collector endpoint, e.g. +pub struct OtlpCollectorConfig { + /// Collector endpoint, e.g. /// `http://otel-collector.observability.svc:4317`. pub endpoint: String, + + /// Transport to the collector. Defaults to gRPC. + #[serde(default, skip_serializing_if = "is_grpc")] + pub protocol: OtlpProtocol, +} + +impl OtlpCollectorConfig { + /// Rejects an endpoint with a query string, which no exporter sends, and + /// an `https://` endpoint for `http/protobuf`, which has no TLS. + fn checked(self, table: &str) -> Result { + let endpoint = self.endpoint.trim(); + if endpoint.contains('?') { + return Err(format!( + "{table} endpoint has a query string, which OTLP export does not send; set collector parameters with headers" + )); + } + let https = endpoint + .get(..8) + .is_some_and(|scheme| scheme.eq_ignore_ascii_case("https://")); + if https && self.protocol == OtlpProtocol::HttpProtobuf { + return Err(format!( + "{table} uses an https:// endpoint, which http/protobuf does not support" + )); + } + Ok(self) + } + + fn export(&self) -> openshell_core::sandbox_env::OtlpExport { + openshell_core::sandbox_env::OtlpExport { + endpoint: self.endpoint.clone(), + protocol: self.protocol, + } + } +} + +// Signature dictated by serde's `skip_serializing_if`. +#[allow(clippy::trivially_copy_pass_by_ref)] +fn is_grpc(protocol: &OtlpProtocol) -> bool { + *protocol == OtlpProtocol::Grpc +} + +/// `[openshell.gateway.otlp.traces]`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OtlpTracesConfig { + #[serde(flatten)] + pub collector: OtlpCollectorConfig, } /// `[openshell.gateway.otlp.ocsf]`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OcsfOtlpConfig { - /// OTLP/gRPC collector endpoint for OCSF log records. - pub endpoint: String, + #[serde(flatten)] + pub collector: OtlpCollectorConfig, /// Deprecated and ignored: `OpenShell` emits OCSF 1.8.0 only. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -412,9 +474,9 @@ pub struct OcsfOtlpConfig { /// `[openshell.gateway.otlp.agent]`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AgentOtlpConfig { - /// OTLP/gRPC collector endpoint that sandbox supervisors forward - /// workload traces to. - pub endpoint: String, + /// The collector that sandbox supervisors forward workload traces to. + #[serde(flatten)] + pub collector: OtlpCollectorConfig, } #[derive(Deserialize)] @@ -423,6 +485,8 @@ struct RawOtlpConfig { #[serde(default)] endpoint: Option, #[serde(default)] + protocol: Option, + #[serde(default)] service_name: Option, #[serde(default)] traces: Option, @@ -439,6 +503,8 @@ struct RawAgentOtlp { enabled: bool, #[serde(default)] endpoint: Option, + #[serde(default)] + protocol: Option, } #[derive(Deserialize)] @@ -446,6 +512,8 @@ struct RawAgentOtlp { struct RawOtlpTraces { #[serde(default)] endpoint: Option, + #[serde(default)] + protocol: Option, } #[derive(Deserialize)] @@ -456,6 +524,8 @@ struct RawOcsfOtlp { #[serde(default)] endpoint: Option, #[serde(default)] + protocol: Option, + #[serde(default)] schema_version: Option, } @@ -463,22 +533,52 @@ impl TryFrom for OtlpConfig { type Error = String; fn try_from(raw: RawOtlpConfig) -> Result { - let endpoint_for = |table: &str, endpoint: Option| { - endpoint.or_else(|| raw.endpoint.clone()).ok_or_else(|| { - format!("otlp.{table} needs an endpoint, or otlp.endpoint to default to") + if raw.endpoint.is_none() && raw.protocol.is_some() { + return Err( + "otlp sets protocol without its own endpoint; a table with its own \ + endpoint takes nothing from otlp" + .to_string(), + ); + } + let shared = raw + .endpoint + .clone() + .map(|endpoint| { + OtlpCollectorConfig { + endpoint, + protocol: raw.protocol.unwrap_or_default(), + } + .checked("otlp") }) - }; + .transpose()?; + let collector_for = + |table: &str, endpoint: Option, protocol: Option| match ( + endpoint, protocol, + ) { + (Some(endpoint), protocol) => OtlpCollectorConfig { + endpoint, + protocol: protocol.unwrap_or_default(), + } + .checked(&format!("otlp.{table}")), + (None, Some(_)) => Err(format!( + "otlp.{table} sets protocol without its own endpoint; a table without \ + an endpoint inherits the whole otlp collector" + )), + (None, None) => shared.clone().ok_or_else(|| { + format!("otlp.{table} needs an endpoint, or otlp.endpoint to default to") + }), + }; let traces = match raw.traces { - Some(traces) => Some(endpoint_for("traces", traces.endpoint)?), - None => raw.endpoint.clone(), + Some(traces) => Some(collector_for("traces", traces.endpoint, traces.protocol)?), + None => shared.clone(), } - .map(|endpoint| OtlpTracesConfig { endpoint }); + .map(|collector| OtlpTracesConfig { collector }); let ocsf = raw .ocsf .filter(|ocsf| ocsf.enabled) .map(|ocsf| { Ok::<_, String>(OcsfOtlpConfig { - endpoint: endpoint_for("ocsf", ocsf.endpoint)?, + collector: collector_for("ocsf", ocsf.endpoint, ocsf.protocol)?, schema_version: ocsf.schema_version, }) }) @@ -488,7 +588,7 @@ impl TryFrom for OtlpConfig { .filter(|agent| agent.enabled) .map(|agent| { Ok::<_, String>(AgentOtlpConfig { - endpoint: endpoint_for("agent", agent.endpoint)?, + collector: collector_for("agent", agent.endpoint, agent.protocol)?, }) }) .transpose()?; @@ -1498,7 +1598,10 @@ service_name = "openshell-gateway-dev" assert_eq!( otlp.ocsf, Some(OcsfOtlpConfig { - endpoint: "http://127.0.0.1:4317".into(), + collector: OtlpCollectorConfig { + endpoint: "http://127.0.0.1:4317".into(), + protocol: OtlpProtocol::Grpc, + }, schema_version: None, }) ); @@ -1515,7 +1618,10 @@ service_name = "openshell-gateway-dev" assert_eq!( otlp.ocsf, Some(OcsfOtlpConfig { - endpoint: "http://siem:4317".into(), + collector: OtlpCollectorConfig { + endpoint: "http://siem:4317".into(), + protocol: OtlpProtocol::Grpc, + }, schema_version: None, }) ); @@ -1538,7 +1644,7 @@ service_name = "openshell-gateway-dev" otlp("[openshell.gateway.otlp.ocsf]\nenabled = true\nendpoint = 'http://siem:4317'\n"); assert_eq!(otlp.traces_endpoint(), None); assert_eq!( - otlp.ocsf.map(|ocsf| ocsf.endpoint).as_deref(), + otlp.ocsf.map(|ocsf| ocsf.collector.endpoint).as_deref(), Some("http://siem:4317") ); } @@ -1595,12 +1701,120 @@ service_name = "openshell-gateway-dev" assert!(format!("{error:#}").contains("otlp.agent needs an endpoint")); } + #[test] + fn otlp_lanes_inherit_the_whole_shared_collector_or_none_of_it() { + use openshell_core::sandbox_env::{OtlpExport, SupervisorOtlp}; + + let export = |endpoint: &str, protocol| { + Some(OtlpExport { + endpoint: endpoint.into(), + protocol, + }) + }; + let shared = otlp( + "[openshell.gateway.otlp]\nendpoint = 'http://shared:4318'\n\ + protocol = 'http/protobuf'\n\n\ + [openshell.gateway.otlp.ocsf]\nenabled = true\n\n\ + [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'http://agents:4317'\n", + ); + assert_eq!( + shared.supervisor_otlp(), + SupervisorOtlp { + traces: export("http://shared:4318", OtlpProtocol::HttpProtobuf), + ocsf: export("http://shared:4318", OtlpProtocol::HttpProtobuf), + agent: export("http://agents:4317", OtlpProtocol::Grpc), + }, + "lanes without an endpoint inherit the shared protocol; a lane \ + with its own endpoint does not" + ); + + let separate = otlp( + "[openshell.gateway.otlp.traces]\nendpoint = 'http://traces:4318'\n\ + protocol = 'http/protobuf'\n\n\ + [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'http://agents:4318'\n\ + protocol = 'http/protobuf'\n", + ); + assert_eq!( + separate.supervisor_otlp(), + SupervisorOtlp { + traces: export("http://traces:4318", OtlpProtocol::HttpProtobuf), + agent: export("http://agents:4318", OtlpProtocol::HttpProtobuf), + ..SupervisorOtlp::default() + } + ); + } + + #[test] + fn otlp_lane_protocol_needs_its_own_endpoint() { + for table in [ + "[openshell.gateway.otlp.traces]\nprotocol = 'http/protobuf'\n", + "[openshell.gateway.otlp.ocsf]\nenabled = true\nprotocol = 'http/protobuf'\n", + "[openshell.gateway.otlp.agent]\nenabled = true\nprotocol = 'http/protobuf'\n", + ] { + let tmp = write_tmp(&format!( + "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317'\n\n{table}" + )); + let error = load(tmp.path()).expect_err("protocol without endpoint"); + assert!( + format!("{error:#}").contains("protocol without its own endpoint"), + "{table}: {error:#}" + ); + } + let tmp = write_tmp( + "[openshell.gateway.otlp]\nprotocol = 'http/protobuf'\n\n\ + [openshell.gateway.otlp.traces]\nendpoint = 'http://traces:4318'\n", + ); + let error = load(tmp.path()).expect_err("a shared protocol without a shared endpoint"); + assert!( + format!("{error:#}").contains("otlp sets protocol without its own endpoint"), + "{error:#}" + ); + let tmp = write_tmp( + "[openshell.gateway.otlp.agent]\nenabled = true\n\ + endpoint = 'http://agents:4318'\nprotocol = 'http/json'\n", + ); + let error = load(tmp.path()).expect_err("http/json is not supported"); + assert!(format!("{error:#}").contains("http/json"), "{error:#}"); + + for table in [ + "[openshell.gateway.otlp]\nendpoint = 'https://shared:4318'\nprotocol = 'http/protobuf'\n", + "[openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'HTTPS://agents:4318'\n\ + protocol = 'http/protobuf'\n", + ] { + let tmp = write_tmp(table); + let error = load(tmp.path()).expect_err("https is not supported over http/protobuf"); + assert!( + format!("{error:#}").contains("which http/protobuf does not support"), + "{table}: {error:#}" + ); + } + + for (table, expected) in [ + ( + "[openshell.gateway.otlp.agent]\nenabled = true\n\ + endpoint = 'http://agents:4318/otlp?api_key=secret'\nprotocol = 'http/protobuf'\n", + "otlp.agent endpoint has a query string", + ), + ( + "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317/?api_key=secret'\n", + "otlp endpoint has a query string", + ), + ] { + let tmp = write_tmp(table); + let error = load(tmp.path()).expect_err("a query string is not supported"); + let message = format!("{error:#}"); + assert!(message.contains(expected), "{table}: {message}"); + assert!(!message.contains("secret"), "{table}: {message}"); + } + } + #[test] fn otlp_config_round_trips() { let tmp = write_tmp( "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317'\nservice_name = 'gw'\n\n\ [openshell.gateway.otlp.ocsf]\nenabled = true\n\n\ - [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'http://agents:4317'\n", + [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'http://agents:4318'\n\ + protocol = 'http/protobuf'\n", ); let file = load(tmp.path()).unwrap(); let encoded = toml::to_string(&file).unwrap(); diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index a50ba1e217..426c283f83 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -4,8 +4,8 @@ //! OpenTelemetry export for the gateway. //! //! Converts selected Rust `tracing` spans into OpenTelemetry traces and -//! gateway OCSF events into log records, and exports each over OTLP/gRPC when -//! configured. +//! gateway OCSF events into log records, and exports each over OTLP, gRPC or +//! HTTP/protobuf per collector, when configured. //! //! # Configuration split //! @@ -24,6 +24,7 @@ //! surfaces (the gateway log stream and the Prometheus `/metrics` endpoint). use openshell_core::jwt::SecretOtlpHeaders; +use openshell_core::sandbox_env::OtlpProtocol; use openshell_otel::{OcsfOtlp, OtlpExportConfig, OtlpTarget, ServiceName}; pub use openshell_otel::{SetupError, TraceContextInterceptor, mark_error}; #[cfg(test)] @@ -32,7 +33,7 @@ use opentelemetry_sdk::trace::SdkTracerProvider; use tracing::Subscriber; use tracing_subscriber::registry::LookupSpan; -use crate::config_file::OtlpConfig; +use crate::config_file::{OtlpCollectorConfig, OtlpConfig}; /// `service.name` reported when the config file does not override it. const DEFAULT_SERVICE_NAME: &str = "openshell-gateway"; @@ -120,12 +121,13 @@ pub struct OtlpExport { ocsf: Option>, /// Validated collector that supervisors relay workload traces to. The /// gateway itself exports nothing for this signal. - agent_endpoint: Option, + agent: Option>, } /// One signal's exporter and the endpoint it sends to. struct Signal { endpoint: String, + protocol: OtlpProtocol, exporter: E, } @@ -152,14 +154,18 @@ impl OtlpExport { cfg: &OtlpConfig, gateway: GatewayResourceAttributes<'_>, ) -> Result { - let target = |endpoint| OtlpTarget::new(&export_config(cfg, endpoint, gateway)); + let target = |collector: &OtlpCollectorConfig| { + OtlpTarget::new(&export_config(cfg, &collector.endpoint, gateway)) + .map(|target| target.with_protocol(collector.protocol.into())) + }; let traces = cfg .traces .as_ref() .map(|traces| { - let target = target(&traces.endpoint)?; + let target = target(&traces.collector)?; Ok::<_, SetupError>(Signal { endpoint: target.endpoint().to_string(), + protocol: traces.collector.protocol, exporter: target.tracer_provider()?, }) }) @@ -168,28 +174,47 @@ impl OtlpExport { .ocsf .as_ref() .map(|ocsf| { - let target = target(&ocsf.endpoint)?; + let target = target(&ocsf.collector)?; Ok::<_, SetupError>(Signal { endpoint: target.endpoint().to_string(), + protocol: ocsf.collector.protocol, exporter: OcsfOtlp::start(target.log_exporter()?, target.resource().clone()), }) }) .transpose()?; - let agent_endpoint = cfg - .agent_endpoint() - .map(|endpoint| target(endpoint).map(|target| target.endpoint().to_string())) + let agent = cfg + .agent + .as_ref() + .map(|agent| { + target(&agent.collector).map(|target| Signal { + endpoint: target.endpoint().to_string(), + protocol: agent.collector.protocol, + exporter: (), + }) + }) .transpose()?; Ok(Self { traces, ocsf, - agent_endpoint, + agent, }) } /// The collector that supervisors relay workload traces to, when the /// agent trace relay is enabled. pub fn agent_endpoint(&self) -> Option<&str> { - self.agent_endpoint.as_deref() + self.agent.as_ref().map(|agent| agent.endpoint.as_str()) + } + + /// The transport to the OCSF collector, when OCSF export is enabled. + pub fn ocsf_protocol(&self) -> Option { + self.ocsf.as_ref().map(|ocsf| ocsf.protocol) + } + + /// The transport to the agent relay's collector, when the relay is + /// enabled. + pub fn agent_protocol(&self) -> Option { + self.agent.as_ref().map(|agent| agent.protocol) } /// The collector receiving traces, when trace export is enabled. @@ -197,6 +222,13 @@ impl OtlpExport { self.traces.as_ref().map(|traces| traces.endpoint.as_str()) } + /// The transport to the trace collector. + pub fn traces_protocol(&self) -> OtlpProtocol { + self.traces + .as_ref() + .map_or(OtlpProtocol::Grpc, |traces| traces.protocol) + } + /// The trace provider, when trace export is enabled. pub fn tracer_provider(&self) -> Option<&SdkTracerProvider> { self.traces.as_ref().map(|traces| &traces.exporter) @@ -584,7 +616,10 @@ mod tests { OtlpConfig { service_name: None, traces: Some(OtlpTracesConfig { - endpoint: endpoint.into(), + collector: OtlpCollectorConfig { + endpoint: endpoint.into(), + protocol: OtlpProtocol::Grpc, + }, }), ocsf: None, agent: None, @@ -593,7 +628,10 @@ mod tests { fn ocsf_config(endpoint: &str) -> OcsfOtlpConfig { OcsfOtlpConfig { - endpoint: endpoint.into(), + collector: OtlpCollectorConfig { + endpoint: endpoint.into(), + protocol: OtlpProtocol::Grpc, + }, schema_version: None, } } @@ -703,6 +741,51 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread")] + async fn gateway_signals_export_over_their_configured_protocol() { + use openshell_ocsf::{ConfigStateChangeBuilder, ocsf_emit}; + use opentelemetry::trace::{Tracer as _, TracerProvider as _}; + use tracing_subscriber::prelude::*; + + let mut collector = openshell_otel_test_support::OtlpHttpTestServer::start().await; + let http = |endpoint: &str| OtlpCollectorConfig { + endpoint: endpoint.into(), + protocol: OtlpProtocol::HttpProtobuf, + }; + let mut cfg = config(); + cfg.traces = Some(OtlpTracesConfig { + collector: http(collector.endpoint()), + }); + cfg.ocsf = Some(OcsfOtlpConfig { + collector: http(collector.endpoint()), + schema_version: None, + }); + let (export, error) = OtlpExport::start(Some(&cfg), GatewayResourceAttributes::default()); + assert!(error.is_none(), "unexpected setup error: {error:?}"); + let export = export.expect("OTLP export"); + assert_eq!(export.traces_protocol(), OtlpProtocol::HttpProtobuf); + assert_eq!(export.ocsf_protocol(), Some(OtlpProtocol::HttpProtobuf)); + assert_eq!(export.agent_protocol(), None); + + let provider = export.tracer_provider().expect("trace export").clone(); + provider.tracer("test").in_span("gateway-span", |_| {}); + let event = + ConfigStateChangeBuilder::new(&crate::gateway_ocsf::context("sandbox-1", "sandbox")) + .message("policy approved") + .build(); + let subscriber = + tracing_subscriber::registry().with(export.ocsf().expect("OCSF log export").layer()); + tracing::subscriber::with_default(subscriber, || ocsf_emit!(event)); + export.shutdown().await; + + let mut paths = vec![ + collector.next_export().await.path, + collector.next_export().await.path, + ]; + paths.sort(); + assert_eq!(paths, ["/v1/logs", "/v1/traces"]); + } + /// Supervisors authenticate to the collector with the headers from their /// launch bundle. #[tokio::test] @@ -902,7 +985,10 @@ mod tests { bad_ocsf.ocsf = Some(ocsf_config("definitely not a url")); let mut bad_agent = config(); bad_agent.agent = Some(crate::config_file::AgentOtlpConfig { - endpoint: "definitely not a url".into(), + collector: OtlpCollectorConfig { + endpoint: "definitely not a url".into(), + protocol: OtlpProtocol::Grpc, + }, }); for cfg in [ with_traces_endpoint("definitely not a url"), diff --git a/crates/openshell-server/src/tracing_setup.rs b/crates/openshell-server/src/tracing_setup.rs index c65752d7ac..fd48c751c9 100644 --- a/crates/openshell-server/src/tracing_setup.rs +++ b/crates/openshell-server/src/tracing_setup.rs @@ -69,16 +69,17 @@ pub fn install( driver: Option, gateway: GatewayResourceAttributes<'_>, ) -> (TracingHandle, Option) { - let driver_endpoint = driver - .is_some() - .then_some(otlp) - .flatten() - .and_then(OtlpExport::traces_endpoint); + let driver_otlp = driver.is_some().then_some(otlp).flatten(); + let driver_endpoint = driver_otlp.and_then(OtlpExport::traces_endpoint); + let driver_protocol = driver_otlp.map_or(openshell_otel::Protocol::Grpc, |otlp| { + otlp.traces_protocol().into() + }); let (driver_tracer_provider, driver_setup_error) = driver.map_or_else( || (None, None), |descriptor| { descriptor.provider_for( driver_endpoint, + driver_protocol, openshell_core::VERSION, gateway.name(), gateway.compute_driver(), diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 5956ef43f7..813cd2b679 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -117,7 +117,7 @@ Console JSON is installed independently of optional file appenders and uses an I ## OTLP agent trace relay -The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_AGENT_ENDPOINT`. The gateway sets that variable only when `[openshell.gateway.otlp.agent]` is enabled, so trace export alone leaves the relay inactive. Metrics and logs exports to the relay are acknowledged and discarded. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards batches over OTLP/gRPC with a lazy connection. +The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_AGENT_ENDPOINT`, over the transport named by `OPENSHELL_OTLP_AGENT_PROTOCOL` (`grpc` when unset, or `http/protobuf`). The gateway sets those variables only when `[openshell.gateway.otlp.agent]` is enabled, so trace export alone leaves the relay inactive. Metrics and logs exports to the relay are acknowledged and discarded. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards the encoded batches to the collector with a lazy connection. Limits are fixed: 2 MiB per request body, at most 512 `ResourceSpans` entries, 256 KiB of encoded `Resource` fields, and 8192 resource attributes per request (each of these answers 413), a 10 second header-read timeout, 16 concurrent connections, 32 buffered batches, and a 5 second deadline per export attempt. Only the resources are decoded; spans are copied byte for byte, at most two requests at a time. Worst-case memory attributable to the agent is about 67 MiB of buffered batches, plus 32 MiB of bodies awaiting enrichment across the 16 connections, plus about 12 MiB for the two enrichments in progress, plus about 4 MiB for the batch being exported and its encoded copy. A full buffer answers 503 with `Retry-After`. Without an agent endpoint nothing starts and the proxy refuses the relay address. After the agent exits, `run_sandbox_with_backend` flushes buffered batches for up to 5 seconds before it reports the exit to the gateway, then logs a final summary of rejected and failed counts. Flushes skip the wait while the most recent export failed, and batches still pending when the relay is dropped without a flush count as failed. The supervisor tells the sandbox runtime through `AgentSpec::otlp_relay_active` whether the relay runs, and the runtime injects the generic `OTEL_EXPORTER_OTLP_*` defaults only then. diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index bdf4035c61..4085adda2e 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -851,6 +851,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { takeover: None, supervisor_recovery_timeout_secs: None, otlp_agent_endpoint: None, + otlp_agent_protocol: openshell_core::sandbox_env::OtlpProtocol::Grpc, }; let bindings = crate::SandboxInstanceBindings::dedicated( &config, @@ -1055,6 +1056,7 @@ async fn suspend_signal_exits_without_terminating_boundary() { takeover: None, supervisor_recovery_timeout_secs: None, otlp_agent_endpoint: None, + otlp_agent_protocol: openshell_core::sandbox_env::OtlpProtocol::Grpc, }; let bindings = crate::SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index 2d3894d563..9e2a5fec64 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -307,6 +307,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl takeover: None, supervisor_recovery_timeout_secs: None, otlp_agent_endpoint: None, + otlp_agent_protocol: openshell_core::sandbox_env::OtlpProtocol::Grpc, }; let bindings = SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 8b1035beca..c96d7258d7 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -919,9 +919,11 @@ pub struct SandboxInstanceConfig { /// attach: how long it keeps the workload frozen after losing this /// supervisor. `None` leaves the runtime default. pub supervisor_recovery_timeout_secs: Option, - /// OTLP/gRPC collector for relayed agent traces; `None` keeps the relay + /// OTLP collector for relayed agent traces; `None` keeps the relay /// inactive. pub otlp_agent_endpoint: Option, + /// Transport to the relayed agent trace collector. + pub otlp_agent_protocol: openshell_core::sandbox_env::OtlpProtocol, } impl SandboxInstanceConfig { @@ -1076,6 +1078,7 @@ pub async fn run_sandbox_instance( takeover, supervisor_recovery_timeout_secs, otlp_agent_endpoint, + otlp_agent_protocol, } = config; // The owner binds the event context and log sink to the instance's // threads; the instance itself uses the rest. @@ -1296,7 +1299,7 @@ pub async fn run_sandbox_instance( // before attach so the agent spec can tell the boundary whether to point // workload SDKs at it. let (mut otlp_relay, otlp_reserved_destination) = - otlp_relay::activate(otlp_agent_endpoint, sandbox_id.clone()); + otlp_relay::activate(otlp_agent_endpoint, otlp_agent_protocol, sandbox_id.clone()); // Construct the selected client only after shared policy and credential // setup has completed. Keep live credentials and later CA publication in @@ -4648,6 +4651,7 @@ mod tests { takeover: None, supervisor_recovery_timeout_secs: None, otlp_agent_endpoint: None, + otlp_agent_protocol: openshell_core::sandbox_env::OtlpProtocol::Grpc, }; let bindings = SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 7c35b96826..a5f426f743 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -11,6 +11,7 @@ use std::sync::atomic::AtomicBool; use clap::{Parser, ValueEnum}; use miette::{IntoDiagnostic, Result}; +use openshell_core::sandbox_env::OtlpProtocol; use openshell_isolation_interface::contract::BackendDescriptor; use tracing::{error, info, warn}; use tracing_subscriber::EnvFilter; @@ -134,19 +135,34 @@ struct Args { #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] log_level: String, - /// OTLP/gRPC collector endpoint for supervisor trace export. + /// OTLP collector endpoint for supervisor trace export. #[arg(long, env = openshell_core::sandbox_env::OTLP_ENDPOINT)] otlp_endpoint: Option, - /// OTLP/gRPC collector endpoint for supervisor OCSF event export. + /// OTLP transport for supervisor trace export: `grpc` or + /// `http/protobuf`. + #[arg(long, env = openshell_core::sandbox_env::OTLP_PROTOCOL, default_value_t)] + otlp_protocol: OtlpProtocol, + + /// OTLP collector endpoint for supervisor OCSF event export. #[arg(long, env = openshell_core::sandbox_env::OTLP_OCSF_ENDPOINT)] otlp_ocsf_endpoint: Option, - /// OTLP/gRPC collector endpoint that workload traces are relayed to. - /// Absent keeps the relay inactive. + /// OTLP transport for supervisor OCSF event export: `grpc` or + /// `http/protobuf`. + #[arg(long, env = openshell_core::sandbox_env::OTLP_OCSF_PROTOCOL, default_value_t)] + otlp_ocsf_protocol: OtlpProtocol, + + /// OTLP collector endpoint that workload traces are relayed to. Absent + /// keeps the relay inactive. #[arg(long, env = openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT)] otlp_agent_endpoint: Option, + /// OTLP transport to the relayed trace collector: `grpc` or + /// `http/protobuf`. + #[arg(long, env = openshell_core::sandbox_env::OTLP_AGENT_PROTOCOL, default_value_t)] + otlp_agent_protocol: OtlpProtocol, + /// Name of the gateway that launched this sandbox, for telemetry. #[arg(long, env = openshell_core::sandbox_env::GATEWAY_NAME)] gateway_name: Option, @@ -480,9 +496,12 @@ fn main() -> Result<()> { args.gateway_name.as_deref(), ), }; - // The tonic exporters must be created inside the runtime that drives them. + // The exporters must be created inside the runtime that drives them. let (otlp_provider, otlp_setup_error) = runtime.block_on(async { - openshell_otel::provider_for(args.otlp_endpoint.as_deref().map(export_config)) + start_trace_export( + args.otlp_endpoint.as_deref().map(export_config), + args.otlp_protocol, + ) }); let ocsf_headers = isolation_inputs .as_ref() @@ -491,6 +510,7 @@ fn main() -> Result<()> { let (ocsf_export, ocsf_setup_error) = runtime.block_on(async { start_ocsf_export( args.otlp_ocsf_endpoint.as_deref().map(export_config), + args.otlp_ocsf_protocol, ocsf_headers, ) }); @@ -546,12 +566,12 @@ fn main() -> Result<()> { if let Some(error) = otlp_setup_error { warn!(%error, "OTLP exporting could not be started; continuing without it"); } else if let Some(endpoint) = &args.otlp_endpoint { - info!(endpoint, "OTLP exporting enabled"); + info!(endpoint, protocol = %args.otlp_protocol, "OTLP exporting enabled"); } if let Some(error) = ocsf_setup_error { warn!(%error, "OTLP OCSF export could not be started; continuing without it"); } else if let Some(endpoint) = &args.otlp_ocsf_endpoint { - info!(endpoint, "OTLP OCSF export enabled"); + info!(endpoint, protocol = %args.otlp_ocsf_protocol, "OTLP OCSF export enabled"); } let workdir = args.workdir.clone(); @@ -627,6 +647,7 @@ fn main() -> Result<()> { takeover: None, supervisor_recovery_timeout_secs: None, otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), + otlp_agent_protocol: args.otlp_agent_protocol, }; // Log push already shares these credentials and the logging // layers these OCSF JSON settings. @@ -686,6 +707,7 @@ fn main() -> Result<()> { /// once its tracing subscriber is installed. fn start_ocsf_export( config: Option>, + protocol: OtlpProtocol, headers: Option<&str>, ) -> ( Option, @@ -694,7 +716,7 @@ fn start_ocsf_export( let Some(config) = config else { return (None, None); }; - let started = openshell_otel::OtlpTarget::new(&config).and_then(|target| { + let started = target(&config, protocol).and_then(|target| { Ok(openshell_otel::OcsfOtlp::start( target.log_exporter_with_headers(headers)?, target.resource().clone(), @@ -706,6 +728,30 @@ fn start_ocsf_export( } } +/// Starts supervisor trace export to `config` over `protocol`. A setup +/// failure disables export and is returned for the caller to report once its +/// tracing subscriber is installed. +fn start_trace_export( + config: Option>, + protocol: OtlpProtocol, +) -> ( + Option, + Option, +) { + match config.map(|config| target(&config, protocol)?.tracer_provider()) { + None => (None, None), + Some(Ok(provider)) => (Some(provider), None), + Some(Err(error)) => (None, Some(error)), + } +} + +fn target( + config: &openshell_otel::OtlpExportConfig<'_>, + protocol: OtlpProtocol, +) -> Result { + Ok(openshell_otel::OtlpTarget::new(config)?.with_protocol(protocol.into())) +} + /// Resource attributes identifying this supervisor's sandbox and the gateway /// that launched it. fn supervisor_resource_attributes( @@ -936,6 +982,80 @@ mod tests { assert_eq!(args.otlp_endpoint, None); } + #[test] + fn otlp_protocols_default_to_grpc() { + let args = Args::try_parse_from(["openshell-supervisor"]).expect("parse defaults"); + assert_eq!( + [ + args.otlp_protocol, + args.otlp_ocsf_protocol, + args.otlp_agent_protocol + ], + [OtlpProtocol::Grpc; 3] + ); + let args = Args::try_parse_from([ + "openshell-supervisor", + "--otlp-protocol", + "http/protobuf", + "--otlp-ocsf-protocol", + "http/protobuf", + "--otlp-agent-protocol", + "http/protobuf", + ]) + .expect("parse HTTP protocols"); + assert_eq!( + [ + args.otlp_protocol, + args.otlp_ocsf_protocol, + args.otlp_agent_protocol + ], + [OtlpProtocol::HttpProtobuf; 3] + ); + assert!( + Args::try_parse_from(["openshell-supervisor", "--otlp-protocol", "http/json"]).is_err() + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn supervisor_signals_export_over_their_configured_protocol() { + use openshell_ocsf::{ConfigStateChangeBuilder, ocsf_emit}; + use opentelemetry::trace::{Tracer as _, TracerProvider as _}; + use tracing_subscriber::prelude::*; + + let mut collector = openshell_otel_test_support::OtlpHttpTestServer::start().await; + let config = || openshell_otel::OtlpExportConfig { + endpoint: collector.endpoint(), + service_name: openshell_otel::ServiceName::Fixed(SERVICE_NAME), + service_version: None, + resource_attributes: Vec::new(), + }; + let (provider, error) = start_trace_export(Some(config()), OtlpProtocol::HttpProtobuf); + assert!(error.is_none(), "unexpected setup error: {error:?}"); + let provider = provider.expect("trace export"); + let (ocsf, error) = start_ocsf_export(Some(config()), OtlpProtocol::HttpProtobuf, None); + assert!(error.is_none(), "unexpected setup error: {error:?}"); + let ocsf = ocsf.expect("OCSF export"); + + provider.tracer("test").in_span("supervisor-span", |_| {}); + let event = ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .message("policy loaded") + .build(); + let subscriber = tracing_subscriber::registry().with(ocsf.layer()); + tracing::subscriber::with_default(subscriber, || ocsf_emit!(event)); + tokio::task::spawn_blocking(move || provider.force_flush()) + .await + .unwrap() + .expect("trace flush"); + ocsf.shutdown().await; + + let mut paths = vec![ + collector.next_export().await.path, + collector.next_export().await.path, + ]; + paths.sort(); + assert_eq!(paths, ["/v1/logs", "/v1/traces"]); + } + #[tokio::test] async fn ocsf_export_starts_only_with_a_valid_endpoint() { let config = |endpoint| openshell_otel::OtlpExportConfig { @@ -944,15 +1064,17 @@ mod tests { service_version: None, resource_attributes: Vec::new(), }; - let (export, error) = start_ocsf_export(None, None); + let (export, error) = start_ocsf_export(None, OtlpProtocol::Grpc, None); assert!(export.is_none() && error.is_none()); - let (export, error) = start_ocsf_export(Some(config("not a url")), None); + let (export, error) = + start_ocsf_export(Some(config("not a url")), OtlpProtocol::Grpc, None); assert!(export.is_none()); assert!(error.is_some()); let (export, error) = start_ocsf_export( Some(config("http://127.0.0.1:4317")), + OtlpProtocol::Grpc, Some("authorization=Bearer%20t"), ); assert!(error.is_none(), "unexpected setup error: {error:?}"); diff --git a/crates/openshell-supervisor/src/otlp_relay/exporter.rs b/crates/openshell-supervisor/src/otlp_relay/exporter.rs index 82eb45ffbb..bc5b0e2d6c 100644 --- a/crates/openshell-supervisor/src/otlp_relay/exporter.rs +++ b/crates/openshell-supervisor/src/otlp_relay/exporter.rs @@ -1,17 +1,22 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! OTLP/gRPC forwarder for enriched agent batches. +//! OTLP forwarder for enriched agent batches, over gRPC or HTTP/protobuf. //! -//! The channel connects lazily so an unreachable collector never fails -//! startup, and tonic re-establishes it on later attempts. Batches are sent -//! as the already-encoded bytes the receiver produced: the agent-controlled +//! Connections are made lazily so an unreachable collector never fails +//! startup, and are re-established on later attempts. Batches are sent as +//! the already-encoded bytes the receiver produced: the agent-controlled //! payload is never decoded into a span tree on the export path. use std::time::Duration; use bytes::{BufMut as _, Bytes}; use http::uri::PathAndQuery; +use openshell_core::sandbox_env::OtlpProtocol; +use openshell_otel::{ + CONNECT_TIMEOUT, OtlpHttpClient, TCP_KEEPALIVE_IDLE, TCP_KEEPALIVE_INTERVAL, + TCP_KEEPALIVE_RETRIES, +}; use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceResponse; use prost::Message; use tonic::client::Grpc; @@ -21,11 +26,6 @@ use tonic::{GrpcMethod, Status}; use super::EXPORT_TIMEOUT; -/// TCP connect deadline, applied per resolved address by hyper, kept well -/// under [`EXPORT_TIMEOUT`] so the batch that triggers a failed connect -/// usually observes the connect error rather than a request timeout. DNS -/// resolution is bounded only by [`EXPORT_TIMEOUT`]. -const CONNECT_TIMEOUT: Duration = Duration::from_secs(2); /// HTTP/2 PING cadence while an export is in flight. Pings are never sent on /// an idle connection: stock collectors (grpc-go) close connections that /// ping without an open stream. @@ -33,12 +33,10 @@ const HTTP2_PING_INTERVAL: Duration = Duration::from_secs(2); /// Unanswered-PING deadline; together with the interval it closes a dead /// connection inside [`EXPORT_TIMEOUT`] so the next export redials. const HTTP2_PING_TIMEOUT: Duration = Duration::from_secs(2); -/// TCP keepalive idle time, probe interval, and probe count for idle -/// connections, where HTTP/2 pings are not allowed. Detects a vanished peer -/// in about 30 seconds on Linux. -const TCP_KEEPALIVE_IDLE: Duration = Duration::from_secs(15); -const TCP_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5); -const TCP_KEEPALIVE_RETRIES: u32 = 3; + +/// Added to the client's own deadline so that [`EXPORT_TIMEOUT`] is the one +/// that ends a slow export. +const CLIENT_TIMEOUT_MARGIN: Duration = Duration::from_secs(1); const SERVICE: &str = "opentelemetry.proto.collector.trace.v1.TraceService"; const METHOD: &str = "Export"; @@ -47,8 +45,12 @@ const PATH: &str = "/opentelemetry.proto.collector.trace.v1.TraceService/Export" /// One export attempt failed. The batch is discarded by the caller. #[derive(Debug)] pub enum ExportError { - /// The collector answered with an error or the transport failed. + /// The gRPC collector answered with an error or the transport failed. Status(Status), + /// The HTTP collector answered with a non-success status. + HttpStatus(http::StatusCode), + /// The HTTP request could not be completed. + Http(String), /// No answer within [`EXPORT_TIMEOUT`]. Timeout, } @@ -57,6 +59,8 @@ impl std::fmt::Display for ExportError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Status(status) => write!(f, "{}: {}", status.code(), status.message()), + Self::HttpStatus(status) => write!(f, "collector answered HTTP {status}"), + Self::Http(error) => f.write_str(error), Self::Timeout => write!(f, "no response within {EXPORT_TIMEOUT:?}"), } } @@ -64,38 +68,61 @@ impl std::fmt::Display for ExportError { impl std::error::Error for ExportError {} +/// The collector endpoint could not be turned into an export target. +#[derive(Debug)] +pub enum EndpointError { + Grpc(tonic::transport::Error), + Http(http::Error), + Runtime(openshell_otel::SetupError), +} + +impl std::fmt::Display for EndpointError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Grpc(error) => error.fmt(f), + Self::Http(error) => error.fmt(f), + Self::Runtime(error) => error.fmt(f), + } + } +} + +impl std::error::Error for EndpointError {} + +enum Transport { + Grpc(Grpc), + Http { + client: Box, + uri: http::Uri, + }, +} + /// Forwards encoded `ExportTraceServiceRequest` batches to one collector. pub struct Exporter { - client: Grpc, + transport: Transport, endpoint: String, } impl Exporter { - /// Prepares a lazy channel to `endpoint`. Fails only when the endpoint is - /// not a valid URI; no I/O happens here. - pub fn new(endpoint: &str) -> Result { - // Liveness detection for a collector that accepted the TCP connection - // and then went silent or vanished without a FIN, so the lazy channel - // redials instead of timing out every batch on a dead connection for - // the sandbox lifetime. HTTP/2 pings run only while an export is in - // flight; TCP keepalive covers idle connections. Detection of a dead - // connection during an export is covered by the silent-collector - // test; idle detection runs on the TCP keepalive timers and has no - // unit test, because those intervals are real-time constants. - // - // Deliberately not built on `openshell_extension_core::transport`: - // that recipe connects eagerly, pings idle connections, and carries a - // first-party TLS trust policy, none of which fit a third-party - // collector that must not gate supervisor startup. - let channel = Endpoint::from_shared(endpoint.to_string())? - .connect_timeout(CONNECT_TIMEOUT) - .tcp_keepalive(Some(TCP_KEEPALIVE_IDLE)) - .tcp_keepalive_interval(Some(TCP_KEEPALIVE_INTERVAL)) - .tcp_keepalive_retries(Some(TCP_KEEPALIVE_RETRIES)) - .http2_keep_alive_interval(HTTP2_PING_INTERVAL) - .keep_alive_timeout(HTTP2_PING_TIMEOUT) - .connect_lazy(); - Ok(Self::with_channel(channel, endpoint)) + /// Prepares a lazy connection to `endpoint` over `protocol`. Fails only + /// when the endpoint is not a valid URI; no I/O happens here. + pub fn new(endpoint: &str, protocol: OtlpProtocol) -> Result { + match protocol { + OtlpProtocol::Grpc => Ok(Self::with_channel( + grpc_channel(endpoint).map_err(EndpointError::Grpc)?, + endpoint, + )), + OtlpProtocol::HttpProtobuf => Ok(Self { + transport: Transport::Http { + client: Box::new( + OtlpHttpClient::new(EXPORT_TIMEOUT + CLIENT_TIMEOUT_MARGIN) + .map_err(EndpointError::Runtime)?, + ), + uri: openshell_otel::signal_url(endpoint, "v1/traces") + .map_err(EndpointError::Http)?, + }, + endpoint: endpoint.to_string(), + }), + } } /// Forwards over a channel prepared elsewhere. This is the seam for a @@ -105,7 +132,7 @@ impl Exporter { #[must_use] pub fn with_channel(channel: Channel, endpoint: &str) -> Self { Self { - client: Grpc::new(channel), + transport: Transport::Grpc(Grpc::new(channel)), endpoint: endpoint.to_string(), } } @@ -118,27 +145,90 @@ impl Exporter { /// Sends one encoded batch, bounded by [`EXPORT_TIMEOUT`]. pub async fn export(&self, batch: Bytes) -> Result<(), ExportError> { - let mut client = self.client.clone(); - let send = async move { - client.ready().await.map_err(|error| { - Status::unavailable(format!("collector channel not ready: {error}")) - })?; - let mut request = tonic::Request::new(batch); - request - .extensions_mut() - .insert(GrpcMethod::new(SERVICE, METHOD)); - client - .unary(request, PathAndQuery::from_static(PATH), PassthroughCodec) - .await + let send = async { + match &self.transport { + Transport::Grpc(client) => export_grpc(client.clone(), batch).await, + Transport::Http { client, uri } => export_http(client, uri, batch).await, + } }; - match tokio::time::timeout(EXPORT_TIMEOUT, send).await { - Ok(Ok(_)) => Ok(()), - Ok(Err(status)) => Err(ExportError::Status(status)), - Err(_) => Err(ExportError::Timeout), - } + tokio::time::timeout(EXPORT_TIMEOUT, send) + .await + .unwrap_or(Err(ExportError::Timeout)) } } +fn grpc_channel(endpoint: &str) -> Result { + // Liveness detection for a collector that accepted the TCP connection + // and then went silent or vanished without a FIN, so the lazy channel + // redials instead of timing out every batch on a dead connection for + // the sandbox lifetime. HTTP/2 pings run only while an export is in + // flight; TCP keepalive covers idle connections. Detection of a dead + // connection during an export is covered by the silent-collector + // test; idle detection runs on the TCP keepalive timers and has no + // unit test, because those intervals are real-time constants. + // + // Deliberately not built on `openshell_extension_core::transport`: + // that recipe connects eagerly, pings idle connections, and carries a + // first-party TLS trust policy, none of which fit a third-party + // collector that must not gate supervisor startup. + Ok(Endpoint::from_shared(endpoint.to_string())? + .connect_timeout(CONNECT_TIMEOUT) + .tcp_keepalive(Some(TCP_KEEPALIVE_IDLE)) + .tcp_keepalive_interval(Some(TCP_KEEPALIVE_INTERVAL)) + .tcp_keepalive_retries(Some(TCP_KEEPALIVE_RETRIES)) + .http2_keep_alive_interval(HTTP2_PING_INTERVAL) + .keep_alive_timeout(HTTP2_PING_TIMEOUT) + .connect_lazy()) +} + +async fn export_grpc(mut client: Grpc, batch: Bytes) -> Result<(), ExportError> { + client.ready().await.map_err(|error| { + ExportError::Status(Status::unavailable(format!( + "collector channel not ready: {error}" + ))) + })?; + let mut request = tonic::Request::new(batch); + request + .extensions_mut() + .insert(GrpcMethod::new(SERVICE, METHOD)); + client + .unary(request, PathAndQuery::from_static(PATH), PassthroughCodec) + .await + .map(|_| ()) + .map_err(ExportError::Status) +} + +async fn export_http( + client: &OtlpHttpClient, + uri: &http::Uri, + batch: Bytes, +) -> Result<(), ExportError> { + let request = http::Request::post(uri.clone()) + .header(http::header::CONTENT_TYPE, "application/x-protobuf") + .body(batch) + .map_err(|error| ExportError::Http(error.to_string()))?; + let response = client + .send(request) + .await + .map_err(|error| ExportError::Http(error_chain(error.as_ref())))?; + if response.status().is_success() { + Ok(()) + } else { + Err(ExportError::HttpStatus(response.status())) + } +} + +fn error_chain(error: &dyn std::error::Error) -> String { + let mut message = error.to_string(); + let mut source = error.source(); + while let Some(cause) = source { + message.push_str(": "); + message.push_str(&cause.to_string()); + source = cause.source(); + } + message +} + /// gRPC codec that writes a pre-encoded request body as-is and decodes the /// collector's `ExportTraceServiceResponse`. struct PassthroughCodec; @@ -227,12 +317,140 @@ mod tests { use openshell_otel_test_support::OtlpTestServer; use super::super::enrichment::test_util::encoded_request; + use openshell_core::sandbox_env::OtlpProtocol; + + use openshell_otel_test_support::OtlpHttpTestServer; + use super::test_util::SilentCollector; use super::*; #[test] fn new_rejects_an_invalid_uri() { - assert!(Exporter::new("not a uri").is_err()); + assert!(Exporter::new("not a uri", OtlpProtocol::Grpc).is_err()); + assert!(Exporter::new("not a uri", OtlpProtocol::HttpProtobuf).is_err()); + } + + #[tokio::test] + async fn http_export_posts_the_batch_to_the_traces_path_under_the_endpoint() { + for (suffix, path) in [ + ("", "/v1/traces"), + ("/", "/v1/traces"), + ("/prefix", "/prefix/v1/traces"), + ] { + let mut collector = OtlpHttpTestServer::start().await; + let endpoint = format!("{}{suffix}", collector.endpoint()); + let exporter = Exporter::new(&endpoint, OtlpProtocol::HttpProtobuf).unwrap(); + let batch = Bytes::from(encoded_request("over-http", &[])); + + exporter + .export(batch.clone()) + .await + .expect("export succeeds"); + let export = collector.next_export().await; + + assert_eq!(export.method, http::Method::POST); + assert_eq!(export.path, path, "endpoint {endpoint}"); + assert_eq!( + export.headers[http::header::CONTENT_TYPE], + "application/x-protobuf" + ); + assert_eq!(export.body, batch); + } + } + + #[tokio::test] + async fn http_export_fails_on_a_non_success_status() { + let collector = OtlpHttpTestServer::start_with_status(http::StatusCode::FORBIDDEN).await; + let exporter = Exporter::new(collector.endpoint(), OtlpProtocol::HttpProtobuf).unwrap(); + let error = exporter + .export(Bytes::from(encoded_request("s", &[]))) + .await + .expect_err("403 fails"); + assert!( + matches!(error, ExportError::HttpStatus(http::StatusCode::FORBIDDEN)), + "{error}" + ); + } + + #[tokio::test] + async fn http_export_to_a_closed_port_fails_promptly() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + let exporter = Exporter::new(&endpoint, OtlpProtocol::HttpProtobuf).unwrap(); + + let started = Instant::now(); + let error = exporter + .export(Bytes::from(encoded_request("s", &[]))) + .await + .expect_err("closed port fails"); + assert!(matches!(error, ExportError::Http(_)), "{error}"); + assert!(started.elapsed() < EXPORT_TIMEOUT); + } + + #[tokio::test] + async fn http_export_to_a_silent_collector_times_out_then_reconnects() { + // The collector holds its first connection open without answering + // and answers every later one, so the second export succeeds only + // on a fresh connection. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let collector = tokio::spawn(async move { + let (_silent, _) = listener.accept().await.unwrap(); + loop { + let (stream, _) = listener.accept().await.unwrap(); + tokio::spawn(answer_once(stream)); + } + }); + let exporter = Exporter::new(&endpoint, OtlpProtocol::HttpProtobuf).unwrap(); + + let started = Instant::now(); + let error = exporter + .export(Bytes::from(encoded_request("s", &[]))) + .await + .expect_err("silent collector times out"); + assert!(matches!(error, ExportError::Timeout), "{error}"); + let elapsed = started.elapsed(); + assert!( + elapsed >= EXPORT_TIMEOUT && elapsed < EXPORT_TIMEOUT + CLIENT_TIMEOUT_MARGIN, + "elapsed {elapsed:?}" + ); + exporter + .export(Bytes::from(encoded_request("s", &[]))) + .await + .expect("the next export reconnects"); + collector.abort(); + } + + /// Reads one HTTP/1.1 request from `stream` and answers it with an empty + /// 200. + async fn answer_once(mut stream: tokio::net::TcpStream) { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + let mut request = Vec::new(); + let mut chunk = [0; 4096]; + loop { + let read = stream.read(&mut chunk).await.unwrap(); + assert!(read > 0, "the request ended early"); + request.extend_from_slice(&chunk[..read]); + let text = String::from_utf8_lossy(&request); + if let Some(head_end) = text.find("\r\n\r\n") { + let length = text[..head_end] + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok())? + }) + .unwrap_or(0); + if request.len() >= head_end + 4 + length { + break; + } + } + } + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n") + .await + .unwrap(); } #[tokio::test] @@ -254,7 +472,7 @@ mod tests { #[tokio::test] async fn export_reaches_the_collector_with_attributes_intact() { let collector = OtlpTestServer::start().await; - let exporter = Exporter::new(collector.endpoint()).unwrap(); + let exporter = Exporter::new(collector.endpoint(), OtlpProtocol::Grpc).unwrap(); let batch = Bytes::from(encoded_request( "exported-span", &[("service.name", "unit-agent"), ("custom", "kept")], @@ -284,7 +502,7 @@ mod tests { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); drop(listener); - let exporter = Exporter::new(&endpoint).unwrap(); + let exporter = Exporter::new(&endpoint, OtlpProtocol::Grpc).unwrap(); let started = Instant::now(); let error = exporter @@ -303,7 +521,7 @@ mod tests { // error inside the export deadline rather than a 5 s timeout, and // the lazy channel is left closed for the next export to redial. let collector = SilentCollector::start().await; - let exporter = Exporter::new(&collector.endpoint).unwrap(); + let exporter = Exporter::new(&collector.endpoint, OtlpProtocol::Grpc).unwrap(); let started = Instant::now(); let error = exporter .export(Bytes::from(encoded_request("s", &[]))) diff --git a/crates/openshell-supervisor/src/otlp_relay/mod.rs b/crates/openshell-supervisor/src/otlp_relay/mod.rs index 39eeb299a0..281d23ac76 100644 --- a/crates/openshell-supervisor/src/otlp_relay/mod.rs +++ b/crates/openshell-supervisor/src/otlp_relay/mod.rs @@ -7,9 +7,10 @@ //! address [`openshell_core::sandbox_env::OTLP_RELAY_ADDR`]. The sandbox //! broker stages that connection to the supervisor, the network proxy hands //! the stream to [`receiver::OtlpConnectionServer`], and the accepted batch -//! is enriched with the sandbox id, buffered, and forwarded over OTLP/gRPC to -//! the collector the gateway names in `OPENSHELL_OTLP_AGENT_ENDPOINT`. The -//! gateway sets it only when `[openshell.gateway.otlp.agent]` is enabled. +//! is enriched with the sandbox id, buffered, and forwarded over OTLP/gRPC or +//! OTLP/HTTP to the collector the gateway names in +//! `OPENSHELL_OTLP_AGENT_ENDPOINT`, using `OPENSHELL_OTLP_AGENT_PROTOCOL`. The +//! gateway sets them only when `[openshell.gateway.otlp.agent]` is enabled. pub mod buffer; pub mod enrichment; @@ -21,6 +22,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::{Duration, Instant}; +use openshell_core::sandbox_env::OtlpProtocol; use openshell_isolation_interface::contract::BoundaryDuplexStream; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; use openshell_supervisor_network::proxy::{ @@ -221,8 +223,10 @@ impl RelayCounters { /// Input to [`start`]. #[derive(Debug, Clone)] pub struct OtlpRelayConfig { - /// OTLP/gRPC collector URI (`OPENSHELL_OTLP_AGENT_ENDPOINT`). + /// Collector URI (`OPENSHELL_OTLP_AGENT_ENDPOINT`). pub endpoint: String, + /// Transport to the collector (`OPENSHELL_OTLP_AGENT_PROTOCOL`). + pub protocol: OtlpProtocol, /// Sandbox identifier attached to every forwarded resource. pub sandbox_id: String, } @@ -233,10 +237,10 @@ pub enum StartError { /// The collector endpoint is empty or not a URI, judged the same way as /// for the supervisor's own exporter. InvalidEndpoint(openshell_otel::SetupError), - /// The collector endpoint is a URI tonic does not accept. - Transport(tonic::transport::Error), + /// The collector endpoint is a URI the exporter does not accept. + Transport(exporter::EndpointError), /// The collector endpoint uses a scheme the relay does not speak. The - /// relay forwards over plaintext gRPC only, so an `https` endpoint keeps + /// relay forwards over plaintext only, so an `https` endpoint keeps /// it inactive rather than downgrading; the supervisor's own exporter /// does speak TLS. TLS on the relayed hop is a tracked follow-up. UnsupportedScheme(String), @@ -249,7 +253,7 @@ impl std::fmt::Display for StartError { Self::Transport(error) => write!(f, "invalid collector endpoint: {error}"), Self::UnsupportedScheme(scheme) => write!( f, - "unsupported collector endpoint scheme {scheme:?}: the relay speaks plaintext OTLP/gRPC (http) only" + "unsupported collector endpoint scheme {scheme:?}: the relay speaks plaintext OTLP (http) only" ), } } @@ -323,7 +327,7 @@ pub fn start(config: OtlpRelayConfig) -> Result<(OtlpRelay, ReservedDestination) { return Err(StartError::UnsupportedScheme(scheme.to_string())); } - let exporter = Exporter::new(endpoint).map_err(StartError::Transport)?; + let exporter = Exporter::new(endpoint, config.protocol).map_err(StartError::Transport)?; if config.sandbox_id.len() > enrichment::MAX_SANDBOX_ID_BYTES { // The attribution growth bound assumes a shorter id; batches near // the wire limit with many entries will be refused with 413. @@ -340,6 +344,7 @@ pub fn start(config: OtlpRelayConfig) -> Result<(OtlpRelay, ReservedDestination) .status(StatusId::Success) .state(StateId::Enabled, "enabled") .unmapped("endpoint", endpoint.to_string()) + .unmapped("protocol", config.protocol.to_string()) .message("OTLP agent trace relay enabled") .build() ); @@ -356,6 +361,7 @@ pub fn start(config: OtlpRelayConfig) -> Result<(OtlpRelay, ReservedDestination) /// relay inactive the same way. pub fn activate( endpoint: Option, + protocol: OtlpProtocol, sandbox_id: Option, ) -> (Option, Option) { let Some(endpoint) = endpoint else { @@ -383,6 +389,7 @@ pub fn activate( }; match start(OtlpRelayConfig { endpoint: endpoint.clone(), + protocol, sandbox_id, }) { Ok((relay, reserved)) => (Some(relay), Some(reserved)), @@ -591,7 +598,8 @@ mod tests { use std::sync::{Mutex, MutexGuard}; use std::time::Duration; - use openshell_otel_test_support::OtlpTestServer; + use openshell_otel_test_support::{OtlpHttpTestServer, OtlpTestServer}; + use prost::Message as _; use tokio::io::AsyncWriteExt as _; use tracing_subscriber::layer::SubscriberExt as _; @@ -719,7 +727,7 @@ mod tests { fn gated(endpoint: &str) -> (OtlpRelay, ReservedDestination, watch::Sender) { let (gate_tx, gate_rx) = watch::channel(false); - let exporter = Exporter::new(endpoint).unwrap(); + let exporter = Exporter::new(endpoint, OtlpProtocol::Grpc).unwrap(); let (relay, reserved) = assemble(exporter, "sb-test".into(), Some(gate_rx)); (relay, reserved, gate_tx) } @@ -740,11 +748,53 @@ mod tests { } } + #[tokio::test] + async fn relays_to_an_otlp_http_collector_when_configured() { + let mut collector = OtlpHttpTestServer::start().await; + let (_relay, reserved) = start(OtlpRelayConfig { + endpoint: collector.endpoint().to_string(), + protocol: OtlpProtocol::HttpProtobuf, + sandbox_id: "sb-http".into(), + }) + .unwrap(); + let (mut client, boundary) = tokio::io::duplex(64 * 1024); + tokio::spawn(reserved.handler.serve(Box::new(boundary)).unwrap()); + + let body = encoded_request("http-span", &[]); + client + .write_all(&request("POST", "/v1/traces", PROTOBUF, &body)) + .await + .unwrap(); + assert_eq!(read_head(&mut client).await.status, 200); + + let export = collector.next_export().await; + assert_eq!(export.path, "/v1/traces"); + let decoded = + opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest::decode( + export.body, + ) + .unwrap(); + let resource = decoded.resource_spans[0].resource.as_ref().unwrap(); + assert!(resource.attributes.iter().any(|attribute| { + attribute.key == SANDBOX_ID_KEY + && attribute + .value + .as_ref() + .and_then(|value| value.value.as_ref()) + == Some( + &opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue( + "sb-http".into(), + ), + ) + })); + } + #[tokio::test] async fn http_post_over_duplex_reaches_collector() { let collector = OtlpTestServer::start().await; let (relay, reserved) = start(OtlpRelayConfig { endpoint: collector.endpoint().to_string(), + protocol: OtlpProtocol::Grpc, sandbox_id: "sb-test".into(), }) .unwrap(); @@ -813,6 +863,7 @@ mod tests { .num_alive_tasks(); let error = start(OtlpRelayConfig { endpoint: "not a uri".into(), + protocol: OtlpProtocol::Grpc, sandbox_id: "sb-test".into(), }) .err() @@ -821,6 +872,7 @@ mod tests { for blank in ["", " "] { let error = start(OtlpRelayConfig { endpoint: blank.into(), + protocol: OtlpProtocol::Grpc, sandbox_id: "sb-test".into(), }) .err() @@ -844,6 +896,7 @@ mod tests { let collector = OtlpTestServer::start().await; let (relay, _reserved) = start(OtlpRelayConfig { endpoint: format!(" {}\n", collector.endpoint()), + protocol: OtlpProtocol::Grpc, sandbox_id: "sb-test".into(), }) .expect("a padded endpoint starts the relay"); @@ -859,7 +912,7 @@ mod tests { let before = tokio::runtime::Handle::current() .metrics() .num_alive_tasks(); - let (relay, reserved) = activate(None, Some("sb-test".into())); + let (relay, reserved) = activate(None, OtlpProtocol::Grpc, Some("sb-test".into())); assert!(relay.is_none()); assert!(reserved.is_none()); let after = tokio::runtime::Handle::current() @@ -877,7 +930,11 @@ mod tests { let before = tokio::runtime::Handle::current() .metrics() .num_alive_tasks(); - let (relay, reserved) = activate(Some(collector.endpoint().to_string()), None); + let (relay, reserved) = activate( + Some(collector.endpoint().to_string()), + OtlpProtocol::Grpc, + None, + ); assert!(relay.is_none()); assert!(reserved.is_none()); let after = tokio::runtime::Handle::current() @@ -896,6 +953,7 @@ mod tests { let _enter = runtime.enter(); let error = start(OtlpRelayConfig { endpoint: "https://collector.example:4317".into(), + protocol: OtlpProtocol::Grpc, sandbox_id: "sb-test".into(), }) .err() @@ -908,6 +966,7 @@ mod tests { assert!( start(OtlpRelayConfig { endpoint: "HTTP://collector.example:4317".into(), + protocol: OtlpProtocol::Grpc, sandbox_id: "sb-test".into(), }) .is_ok() diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index 28f49b426b..864701d2ed 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -816,6 +816,7 @@ impl SharedHost { // The shared host has no per-instance collector endpoint yet, so // hosted instances do not relay agent traces. otlp_agent_endpoint: None, + otlp_agent_protocol: openshell_core::sandbox_env::OtlpProtocol::Grpc, }; crate::sandbox_instance_host::sandbox_instance_launch( Box::new(crate::backend_setup::RoutedBackendSetup::new( diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index df46e0a4a1..649d19d4c9 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -657,7 +657,7 @@ discovery endpoint or its TLS CA. | server.oidc.rolesClaim | string | `""` | Dot-separated path to the roles array in the JWT claims. Keycloak: "realm_access.roles", Entra ID: "roles", Okta: "groups". | | server.oidc.scopesClaim | string | `""` | Dot-separated path to the scopes array in the JWT claims. | | server.oidc.userRole | string | `""` | Role name for standard user access. | -| server.otlp.endpoint | string | `""` | OTLP/gRPC collector endpoint, conventionally using port 4317. | +| server.otlp.endpoint | string | `""` | OTLP/gRPC collector endpoint, conventionally port 4317. For an `http/protobuf` collector, also set protocol under gatewayConfig."openshell.gateway.otlp". | | server.otlp.serviceName | string | `""` | Gateway OpenTelemetry service name. Empty uses openshell-gateway. | | server.policyValidationFailureMode | string | `"fail_closed"` | Posture when a candidate sandbox policy fails validation. `fail_closed` deactivates the previous policy; `retain_last_valid` keeps it active. | | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into gateway and sandbox pods for dynamic provider token grants. | diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index fa1519e02c..2a3f8ad822 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -390,9 +390,11 @@ server: name: "" # -- Gateway log level. logLevel: info - # OpenTelemetry trace export over OTLP/gRPC. Leave endpoint empty to disable. + # OpenTelemetry trace export over OTLP. Leave endpoint empty to disable. otlp: - # -- OTLP/gRPC collector endpoint, conventionally using port 4317. + # -- OTLP/gRPC collector endpoint, conventionally port 4317. For an + # `http/protobuf` collector, also set protocol under + # gatewayConfig."openshell.gateway.otlp". endpoint: "" # -- Gateway OpenTelemetry service name. Empty uses openshell-gateway. serviceName: "" diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 3d5680301d..20446834b9 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -600,7 +600,7 @@ Allow enough retention and disk space for shipper outages. Graceful shutdown all ## OTLP Export -`[openshell.gateway.otlp]` enables OpenTelemetry export over OTLP/gRPC. Omit the table to disable export. Its `endpoint` is the collector for every signal the gateway exports and, on its own, enables trace export. `[openshell.gateway.otlp.traces]` sends spans to a different collector. [OCSF log export](#ocsf-log-export) is off until `[openshell.gateway.otlp.ocsf]` sets `enabled = true`, and the [agent trace relay](#agent-trace-relay) is off until `[openshell.gateway.otlp.agent]` sets `enabled = true`. Each can use its own collector. +`[openshell.gateway.otlp]` enables OpenTelemetry export over OTLP. Omit the table to disable export. Its `endpoint` is the collector for every signal the gateway exports and, on its own, enables trace export. `[openshell.gateway.otlp.traces]` sends spans to a different collector. [OCSF log export](#ocsf-log-export) is off until `[openshell.gateway.otlp.ocsf]` sets `enabled = true`, and the [agent trace relay](#agent-trace-relay) is off until `[openshell.gateway.otlp.agent]` sets `enabled = true`. Each can use its own collector. The gateway already uses the Rust `tracing` framework for structured logs sent to stdout and the sandbox log stream. Enabling this section adds an OpenTelemetry layer to the same tracing subscriber. It exports span trees to an OTLP collector without exporting, replacing, or redirecting the existing diagnostic log events. @@ -609,16 +609,17 @@ The gateway already uses the Rust `tracing` framework for structured logs sent t endpoint = "http://otel-collector.observability.svc:4317" service_name = "openshell-gateway" -# Optional: send spans to a different collector. +# Optional: send spans to a different collector, over OTLP/HTTP. [openshell.gateway.otlp.traces] -endpoint = "http://traces.observability.svc:4317" +endpoint = "http://traces.observability.svc:4318" +protocol = "http/protobuf" ``` -Each enabled signal needs an endpoint, either in its own table or from `endpoint`. Endpoints must be valid URIs. If one is malformed, the gateway logs the configuration error and continues with all OTLP export disabled. It does not connect at startup: an unreachable collector produces export failures, never a failure to serve. +Each enabled signal needs an endpoint, either in its own table or from `endpoint`. A table without its own `endpoint` uses the whole `[openshell.gateway.otlp]` collector, including its `protocol`. A table with its own `endpoint` takes nothing else from `[openshell.gateway.otlp]`, so settings for one collector never reach another, and setting `protocol` in a table without an `endpoint` is a configuration error. Endpoints must be valid URIs without a query string: exporters send no query parameters, so an endpoint with one is a configuration error, and a collector that needs parameters must take them as headers. If an endpoint is malformed, the gateway logs the configuration error and continues with all OTLP export disabled. It does not connect at startup: an unreachable collector produces export failures, never a failure to serve. -The transport is **OTLP over gRPC only**. HTTP/protobuf and HTTP/JSON are not supported, and `OTEL_EXPORTER_OTLP_PROTOCOL` has no effect. Point each endpoint at a collector's gRPC receiver, conventionally port `4317`, not the HTTP receiver on `4318`. A URI alone cannot distinguish the two, so an HTTP endpoint is accepted at startup and then fails on export. +`protocol` selects the transport to a collector: `grpc`, the default, or `http/protobuf`. With `http/protobuf`, the endpoint is a base URL and each signal is posted to its path under it, `/v1/traces` or `/v1/logs`, as with `OTEL_EXPORTER_OTLP_ENDPOINT`. HTTP/JSON is not supported. Point a `grpc` endpoint at a collector's gRPC receiver, conventionally port `4317`, and an `http/protobuf` endpoint at its HTTP receiver, conventionally port `4318`. A URI alone cannot distinguish the two, so a mismatched endpoint is accepted at startup and then fails on export. Sandbox supervisors and standalone compute drivers must come from the same release as the gateway to use `http/protobuf`: an older supervisor exports over gRPC, and an older driver rejects the protocol setting. -Use an `https://` endpoint for TLS. The gateway verifies the collector certificate against the system trust roots. +Use an `https://` endpoint for TLS over `grpc`. The gateway verifies the collector certificate against the system trust roots. An `https://` endpoint with `http/protobuf` is a configuration error. The OpenTelemetry SDK logs export failures after startup. Spans in a failed batch are dropped rather than retried. @@ -663,7 +664,7 @@ In-process compute drivers read their backend-specific settings from `[openshell ### OCSF Log Export -Set `enabled = true` in `[openshell.gateway.otlp.ocsf]` to export gateway OCSF events as OTLP log records. It defaults to `false`, so trace export alone never sends OCSF events. Records go to the table's `endpoint`, or to `[openshell.gateway.otlp]` `endpoint` when it is omitted, with the same resource attributes as spans, under the `openshell.ocsf` instrumentation scope. Use a separate endpoint to send OCSF events to a collector that feeds a SIEM while spans go elsewhere. Sandbox supervisors receive the same endpoint and export their own OCSF events there directly, so the endpoint must also be reachable from sandboxes. Supervisor records carry the supervisor's resource attributes, including `openshell.sandbox.id` and `openshell.gateway.name`. Supervisors also send the collector headers that the gateway uses for OCSF export, from `OTEL_EXPORTER_OTLP_LOGS_HEADERS` or else `OTEL_EXPORTER_OTLP_HEADERS`, so a collector that requires authentication accepts their records too. The gateway delivers the headers with each sandbox's launch credentials, never through the sandbox environment. Other `OTEL_*` variables do not reach supervisors. +Set `enabled = true` in `[openshell.gateway.otlp.ocsf]` to export gateway OCSF events as OTLP log records. It defaults to `false`, so trace export alone never sends OCSF events. Records go to the table's `endpoint`, or to `[openshell.gateway.otlp]` `endpoint` when it is omitted, with the same resource attributes as spans, under the `openshell.ocsf` instrumentation scope. Use a separate endpoint to send OCSF events to a collector that feeds a SIEM while spans go elsewhere. Sandbox supervisors receive the same endpoint and protocol and export their own OCSF events there directly, so the endpoint must also be reachable from sandboxes. Supervisor records carry the supervisor's resource attributes, including `openshell.sandbox.id` and `openshell.gateway.name`. Supervisors also send the collector headers that the gateway uses for OCSF export, from `OTEL_EXPORTER_OTLP_LOGS_HEADERS` or else `OTEL_EXPORTER_OTLP_HEADERS`, so a collector that requires authentication accepts their records too. The gateway delivers the headers with each sandbox's launch credentials, never through the sandbox environment. Other `OTEL_*` variables do not reach supervisors. ```toml [openshell.gateway.otlp.ocsf] @@ -692,7 +693,7 @@ Metrics include `openshell_ocsf_otlp_queued_total`, `openshell_ocsf_otlp_exporte ### Agent trace relay -Set `enabled = true` in `[openshell.gateway.otlp.agent]` to have each sandbox forward the workload's own traces. It defaults to `false`, so trace export alone never changes the workload environment. Traces go to the table's `endpoint`, or to `[openshell.gateway.otlp]` `endpoint` when it is omitted, and that endpoint must be reachable from sandboxes. +Set `enabled = true` in `[openshell.gateway.otlp.agent]` to have each sandbox forward the workload's own traces. It defaults to `false`, so trace export alone never changes the workload environment. Traces go to the table's collector, or to the `[openshell.gateway.otlp]` collector when it has no `endpoint`, and that collector must be reachable from sandboxes. Use `protocol = "http/protobuf"` for trace backends that accept only OTLP/HTTP. ```toml [openshell.gateway.otlp.agent] @@ -700,7 +701,7 @@ enabled = true endpoint = "http://agent-collector.observability.svc:4317" # optional ``` -While the relay is active, the sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` in every workload process, and sets `OTEL_EXPORTER_OTLP_COMPRESSION` and `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION` to `none`, so a standard OpenTelemetry SDK exports without configuration. These are lowest-precedence defaults. They are not set when the sandbox creation request or an environment passed to `openshell sandbox exec` names `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, or when the image `ENV` names one for the main process, since `openshell sandbox exec` sessions do not inherit the image environment, and an endpoint the workload sets itself, for example with a shell `export`, takes precedence over them. A workload that exports elsewhere does so subject to the sandbox's network policy, and its traces carry no relay attribution. Provider credentials cannot redirect traces. While the relay is off, nothing is injected. Because the generic endpoint is set, an SDK that also exports metrics or logs sends them to the relay, which answers with an empty OTLP success and discards them. To keep metrics and logs, point those exporters elsewhere with `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` and `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over OTLP/gRPC. The collector's address is never visible inside the sandbox. +While the relay is active, the sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` in every workload process, and sets `OTEL_EXPORTER_OTLP_COMPRESSION` and `OTEL_EXPORTER_OTLP_TRACES_COMPRESSION` to `none`, so a standard OpenTelemetry SDK exports without configuration. These are lowest-precedence defaults. They are not set when the sandbox creation request or an environment passed to `openshell sandbox exec` names `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, or when the image `ENV` names one for the main process, since `openshell sandbox exec` sessions do not inherit the image environment, and an endpoint the workload sets itself, for example with a shell `export`, takes precedence over them. A workload that exports elsewhere does so subject to the sandbox's network policy, and its traces carry no relay attribution. Provider credentials cannot redirect traces. While the relay is off, nothing is injected. Because the generic endpoint is set, an SDK that also exports metrics or logs sends them to the relay, which answers with an empty OTLP success and discards them. To keep metrics and logs, point those exporters elsewhere with `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` and `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over the configured `protocol`. The collector's address is never visible inside the sandbox. Every forwarded resource carries `openshell.sandbox.id`, with the same value as the supervisor's own spans, and `openshell.telemetry.source=agent`. Values the agent supplies for those two keys are replaced at the resource level; span and scope attributes pass through unchanged, so filter and join on the resource attributes. Filter on `openshell.telemetry.source` to separate agent traces from infrastructure traces, and join on `openshell.sandbox.id` to correlate them. @@ -712,7 +713,7 @@ The relayed hop does not yet support TLS or authentication to the collector; an ### Tuning -`[openshell.gateway.otlp]` decides whether and where to export. How the SDK exports is controlled by the standard OpenTelemetry environment variables, which the gateway reads through the SDK rather than mirroring as TOML keys. They apply to the gateway and its in-process compute drivers. The gateway does not pass them to sandbox supervisors, which receive only the endpoints and the [OCSF collector headers](#ocsf-log-export). +`[openshell.gateway.otlp]` decides whether and where to export. How the SDK exports is controlled by the standard OpenTelemetry environment variables, which the gateway reads through the SDK rather than mirroring as TOML keys. They apply to the gateway and its in-process compute drivers. The gateway does not pass them to sandbox supervisors, which receive only the endpoints, protocols, and the [OCSF collector headers](#ocsf-log-export). The gateway honors these variables: @@ -750,7 +751,7 @@ These variables have no effect. Only `[openshell.gateway.otlp]` decides which si |---|---| | `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | Enablement has one source, `[openshell.gateway.otlp]`, so an environment variable cannot silently turn export on or redirect it. | | `OTEL_SDK_DISABLED`, `OTEL_TRACES_EXPORTER`, `OTEL_LOGS_EXPORTER`, `OTEL_METRICS_EXPORTER` | `[openshell.gateway.otlp]` decides which signals are exported. To disable export, omit it. | -| `OTEL_EXPORTER_OTLP_PROTOCOL`, `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL` | The gateway is built with the gRPC exporter only. | +| `OTEL_EXPORTER_OTLP_PROTOCOL`, `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL` | `protocol` in each table selects the transport, and supervisors follow the same setting. | | `OTEL_EXPORTER_OTLP_INSECURE`, `OTEL_EXPORTER_OTLP_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE`, `OTEL_EXPORTER_OTLP_CLIENT_KEY`, and their per-signal forms | The endpoint scheme selects TLS, and collector certificates are verified against the system trust roots. Client certificates are not supported. | | `OTEL_PROPAGATORS` | The gateway always propagates W3C Trace Context. | | `OTEL_BLRP_SCHEDULE_DELAY`, `OTEL_BLRP_MAX_QUEUE_SIZE`, `OTEL_BLRP_MAX_EXPORT_BATCH_SIZE`, `OTEL_BLRP_EXPORT_TIMEOUT` | OCSF log export uses its own queue and batching, described in [OCSF Log Export](#ocsf-log-export). | diff --git a/docs/observability/overview.mdx b/docs/observability/overview.mdx index 73ae964707..cb763427e1 100644 --- a/docs/observability/overview.mdx +++ b/docs/observability/overview.mdx @@ -20,7 +20,7 @@ OpenShell produces three kinds of observability signals: traces, metrics, and OC The examples on this page are `gateway.toml` tables. In the Helm chart, set each table under `gatewayConfig`, as described in [Configure Chart Values](/kubernetes/setup#configure-chart-values). -All OTLP export uses OTLP over gRPC. Configure it in `gateway.toml`, and the gateway passes the settings to the sandbox supervisors it starts. Supervisors connect to the collector directly, so the collector must be reachable from sandboxes as well as from the gateway. The gateway ignores the `OTEL_*` variables that would choose which signals are exported or where they go, and honors those that tune export, such as sampling. Refer to [Tuning](/how-it-works/gateways/configuration#tuning) for the full list. +OTLP export uses gRPC by default, or HTTP/protobuf for collectors that set it. Configure it in `gateway.toml`, and the gateway passes the settings to the sandbox supervisors it starts. Supervisors connect to the collector directly, so the collector must be reachable from sandboxes as well as from the gateway. The gateway ignores the `OTEL_*` variables that would choose which signals are exported or where they go, and honors those that tune export, such as sampling. Refer to [Tuning](/how-it-works/gateways/configuration#tuning) for the full list. ## Traces diff --git a/docs/observability/traces.mdx b/docs/observability/traces.mdx index ef8bd6f5f8..87854c019c 100644 --- a/docs/observability/traces.mdx +++ b/docs/observability/traces.mdx @@ -19,7 +19,7 @@ Set an OTLP collector endpoint in `gateway.toml`: endpoint = "https://backend:4317" ``` -In the Helm chart, set the table under `gatewayConfig`. The gateway passes the endpoint to the compute drivers and sandbox supervisors it starts, and supervisors connect to the collector directly, so it must be reachable from sandboxes too. For a separate trace collector, TLS, sampling, and the other settings, refer to [OTLP Export](/how-it-works/gateways/configuration#otlp-export). +In the Helm chart, set the table under `gatewayConfig`. The gateway passes the endpoint to the compute drivers and sandbox supervisors it starts, and supervisors connect to the collector directly, so it must be reachable from sandboxes too. For a separate trace collector, OTLP/HTTP, TLS, sampling, and the other settings, refer to [OTLP Export](/how-it-works/gateways/configuration#otlp-export). ## What Is Traced diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index cfe3bb513d..22e9215626 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -2080,11 +2080,16 @@ dependencies = [ name = "openshell-otel" version = "0.0.0" dependencies = [ + "async-trait", + "bytes", "futures", "http", + "http-body-util", + "hyper-util", "metrics", "openshell-ocsf", "opentelemetry", + "opentelemetry-http", "opentelemetry-otlp", "opentelemetry_sdk", "serde_json", @@ -2101,6 +2106,11 @@ dependencies = [ name = "openshell-otel-test-support" version = "0.0.0" dependencies = [ + "bytes", + "http", + "http-body-util", + "hyper", + "hyper-util", "openshell-otel", "opentelemetry-proto", "prost", @@ -2131,14 +2141,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry-http" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302249bd8ee804789faaa7b5b2cbf79bc61cb864714079c644e3828af5b7e811" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", +] + [[package]] name = "opentelemetry-otlp" version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0756af24841e367128ef31c831999937b8b5c5197d47914590d8f4e2e88c2e1a" dependencies = [ + "flate2", "http", + "httpdate", "opentelemetry", + "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", "prost", diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 9e56f7117c..018f1dc0a8 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1538,7 +1538,7 @@ credential failures. | Symptom | Likely cause | Check | |---|---|---| | `openshell status` fails | Gateway endpoint unreachable or auth mismatch | `openshell gateway info`, gateway logs | -| Collector receives no OCSF log records, or `openshell.ocsf.gap` records | `[openshell.gateway.otlp.ocsf]` missing or without `enabled = true`, an unreachable or non-gRPC endpoint, or export failures and queue pressure | Check `[openshell.gateway.otlp.ocsf]` for `enabled = true` and the endpoint it resolves to, the gateway's `OTLP exporting enabled` startup log for `ocsf_endpoint`, export warnings, and `openshell_ocsf_otlp_*` metrics. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | +| Collector receives no OCSF log records, or `openshell.ocsf.gap` records | `[openshell.gateway.otlp.ocsf]` missing or without `enabled = true`, an unreachable endpoint or one that does not speak the table's `protocol`, or export failures and queue pressure | Check `[openshell.gateway.otlp.ocsf]` for `enabled = true` and the endpoint it resolves to, the gateway's `OTLP exporting enabled` startup log for `ocsf_endpoint` and `ocsf_protocol`, export warnings, and `openshell_ocsf_otlp_*` metrics. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | Gateway OCSF JSONL stops growing, has gaps, or is rejected by a SIEM | File errors, queue pressure, or shipper falling behind retention | Inspect gateway warnings and `openshell_ocsf_log_*` metrics; check `[openshell.gateway.ocsf_log]`, directory permissions, free space, per-replica paths, and shipper rotation checkpoints. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | | Sandbox log shows `OTLP agent trace relay inactive: not enabled by the gateway` at startup, then `agent OTLP export attempted but the agent trace relay is not enabled` when the agent exports | The gateway does not enable `[openshell.gateway.otlp.agent]`, so the supervisor refuses the workload's OTLP relay address | Set `enabled = true` in `[openshell.gateway.otlp.agent]` (Helm: `gatewayConfig."openshell.gateway.otlp.agent".enabled`) with an endpoint from that table or `[openshell.gateway.otlp]`, and recreate the sandbox; `openshell logs --source sandbox` then shows `OTLP agent trace relay enabled` | diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index fdd426f76b..7a4800aaee 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2146,11 +2146,16 @@ dependencies = [ name = "openshell-otel" version = "0.0.0" dependencies = [ + "async-trait", + "bytes", "futures", "http", + "http-body-util", + "hyper-util", "metrics", "openshell-ocsf", "opentelemetry", + "opentelemetry-http", "opentelemetry-otlp", "opentelemetry_sdk", "serde_json", @@ -2167,6 +2172,11 @@ dependencies = [ name = "openshell-otel-test-support" version = "0.0.0" dependencies = [ + "bytes", + "http", + "http-body-util", + "hyper", + "hyper-util", "openshell-otel", "opentelemetry-proto", "prost", @@ -2291,14 +2301,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry-http" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302249bd8ee804789faaa7b5b2cbf79bc61cb864714079c644e3828af5b7e811" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", +] + [[package]] name = "opentelemetry-otlp" version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0756af24841e367128ef31c831999937b8b5c5197d47914590d8f4e2e88c2e1a" dependencies = [ + "flate2", "http", + "httpdate", "opentelemetry", + "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", "prost",