From ac2fe5b0e62ac03babc1953697eff7cb5e30d9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20Hu=C3=9F?= Date: Wed, 7 Oct 2026 07:42:17 +0200 Subject: [PATCH 1/6] feat(supervisor): relay agent OTLP traces to the collector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serve the reserved endpoint 192.0.0.8:4318 inside the supervisor, attribute each batch with openshell.sandbox.id and openshell.telemetry.source=agent at the protobuf wire level, buffer it with fixed bounds, and forward it over OTLP/gRPC to the collector the supervisor already uses, flushing before the exit report and keeping the relay up for exec sessions. Signed-off-by: Roland Huß --- .agents/skills/helm-dev-environment/SKILL.md | 4 +- Cargo.lock | 4 + Cargo.toml | 1 + crates/openshell-core/src/sandbox_env.rs | 83 ++ crates/openshell-otel-test-support/Cargo.toml | 1 + crates/openshell-otel-test-support/src/lib.rs | 148 ++- crates/openshell-sandbox/src/boundary_exec.rs | 64 + crates/openshell-sandbox/src/child_env.rs | 49 + crates/openshell-sandbox/src/process.rs | 276 +++- .../openshell-supervisor-network/src/proxy.rs | 465 ++++++- .../openshell-supervisor-network/src/run.rs | 2 + crates/openshell-supervisor/Cargo.toml | 8 + crates/openshell-supervisor/README.md | 6 + .../src/backend_setup/tests.rs | 1 + .../src/legacy_startup_regression_tests.rs | 1 + crates/openshell-supervisor/src/lib.rs | 29 + crates/openshell-supervisor/src/main.rs | 1 + .../src/otlp_relay/buffer.rs | 105 ++ .../src/otlp_relay/enrichment.rs | 1026 +++++++++++++++ .../src/otlp_relay/exporter.rs | 294 +++++ .../src/otlp_relay/mod.rs | 1088 +++++++++++++++ .../src/otlp_relay/receiver.rs | 1165 +++++++++++++++++ .../openshell-supervisor/src/shared_host.rs | 3 + docs/how-it-works/gateways/configuration.mdx | 10 + e2e/rust/Cargo.lock | 432 +++++- e2e/rust/Cargo.toml | 7 + e2e/rust/e2e-podman.sh | 7 + e2e/rust/tests/otlp_relay.rs | 410 ++++++ e2e/support/podman-gateway-config.sh | 7 + e2e/with-docker-gateway.sh | 7 + skills/debug-openshell-cluster/SKILL.md | 1 + tests/suites/features/Cargo.lock | 432 +++++- 32 files changed, 6032 insertions(+), 105 deletions(-) create mode 100644 crates/openshell-supervisor/src/otlp_relay/buffer.rs create mode 100644 crates/openshell-supervisor/src/otlp_relay/enrichment.rs create mode 100644 crates/openshell-supervisor/src/otlp_relay/exporter.rs create mode 100644 crates/openshell-supervisor/src/otlp_relay/mod.rs create mode 100644 crates/openshell-supervisor/src/otlp_relay/receiver.rs create mode 100644 e2e/rust/tests/otlp_relay.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 0b5a069eeb..bd499f4030 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -170,7 +170,9 @@ trace-context annotation. The same command exposes OTLP/gRPC on `http://127.0.0.1:4317` and, when deployed, the Kubernetes gateway on `http://127.0.0.1:8090`. The local `gateway:docker`, `gateway:podman`, and `gateway:vm` tasks detect the collector listener at startup and enable trace -export only while it is reachable. +export only while it is reachable. Spans an agent exports from inside a sandbox +arrive through the supervisor's relay with `openshell.telemetry.source=agent` +and share `openshell.sandbox.id` with that sandbox's supervisor spans. The Skaffold profile for HA reverse-proxy development is available from `deploy/helm/openshell/`: diff --git a/Cargo.lock b/Cargo.lock index f36af0eb0f..5d3d19ee6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4800,6 +4800,7 @@ dependencies = [ "openshell-crypto", "openshell-otel", "opentelemetry-proto", + "prost", "rcgen", "tempfile", "tokio", @@ -5100,6 +5101,7 @@ dependencies = [ "openshell-isolation-interface", "openshell-ocsf", "openshell-otel", + "openshell-otel-test-support", "openshell-policy", "openshell-sandbox-backend", "openshell-supervisor-control", @@ -5108,6 +5110,7 @@ dependencies = [ "openshell-supervisor-network", "openshell-supervisor-process", "opentelemetry", + "opentelemetry-proto", "prost", "prost-types", "rustix 1.1.4", @@ -5124,6 +5127,7 @@ dependencies = [ "tokio-tungstenite 0.26.2", "tonic", "tonic-prost", + "tonic-types", "tracing", "tracing-appender", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index 905065d843..ee03d87f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,7 @@ tracing-appender = "0.2" # OpenTelemetry — OTLP/gRPC export. Kept in lockstep with the workspace's # tonic 0.14 / prost 0.14 via opentelemetry-proto's `grpc-tonic` feature. +opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] } opentelemetry = "0.32" opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] } opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "gzip-tonic", "tls-aws-lc", "tls-roots", "trace"] } diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index dea5a9851d..e5d212a135 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -299,11 +299,94 @@ pub const DEFAULT_SANDBOX_GID: u32 = 1000; /// OCI only for the former contract. pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER"; +/// Standard OpenTelemetry environment variable for the OTLP exporter endpoint. +/// +/// The sandbox runtime sets it in every workload process to +/// [`OTLP_RELAY_ENDPOINT`], after the environment from the sandbox creation +/// request, so agent SDKs export to the supervisor's relay and a value in the +/// creation request never wins. A per-exec environment is a developer's +/// explicit instruction and still overrides it for that exec only. +pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; + +/// Standard OpenTelemetry environment variable for the OTLP exporter protocol. +/// +/// Set to `http/protobuf` alongside [`OTEL_EXPORTER_OTLP_ENDPOINT`]. +pub const OTEL_EXPORTER_OTLP_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL"; + +/// Signal-specific endpoint variable. SDKs give it precedence over the +/// generic endpoint, so it is set too (to the full traces URL) or an image +/// that ships one would silently bypass the relay. +pub const OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"; + +/// Signal-specific protocol variable, set alongside +/// [`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`] for the same reason. +pub const OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"; + +/// Standard OTLP compression variable. The relay refuses compressed bodies, +/// so both this and the traces-specific variant are set to `none`. +pub const OTEL_EXPORTER_OTLP_COMPRESSION: &str = "OTEL_EXPORTER_OTLP_COMPRESSION"; + +/// Signal-specific compression variable, see +/// [`OTEL_EXPORTER_OTLP_COMPRESSION`]. +pub const OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: &str = "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION"; + +/// Reserved destination agent processes export OTLP to. +/// +/// This address is never routed. A workload `connect()` to any non-loopback +/// address is intercepted by the sandbox seccomp broker and staged for the +/// supervisor, which recognises this destination and serves the OTLP +/// receiver on the staged stream itself instead of dialing upstream. So the +/// value is a label the supervisor switches on, and it must stay outside +/// loopback (loopback connects complete locally without mediation) and +/// outside `198.18.0.0/15`, which the policy DNS runtime uses for synthetic +/// answers. `192.0.0.8` is the IPv4 dummy address (RFC 7600): reserved, +/// unroutable, and never assigned to a real service. +pub const OTLP_RELAY_ADDR: &str = "192.0.0.8:4318"; + +/// [`OTLP_RELAY_ADDR`] as the URL injected through +/// [`OTEL_EXPORTER_OTLP_ENDPOINT`]. +pub const OTLP_RELAY_ENDPOINT: &str = "http://192.0.0.8:4318"; + +/// The traces URL injected through [`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`], +/// which unlike the generic endpoint names the full path. +pub const OTLP_RELAY_TRACES_ENDPOINT: &str = "http://192.0.0.8:4318/v1/traces"; + // The corporate upstream-proxy configuration deliberately has no reserved // environment variables: it travels on the supervisor's argv // (`--upstream-proxy` and friends), which a sandbox image cannot forge the // way it could bake `ENV` values. +#[cfg(test)] +mod otlp_relay_address_tests { + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use super::{OTLP_RELAY_ADDR, OTLP_RELAY_ENDPOINT, OTLP_RELAY_TRACES_ENDPOINT}; + + #[test] + fn relay_address_is_an_unroutable_non_loopback_label() { + let addr: SocketAddr = OTLP_RELAY_ADDR.parse().expect("relay address parses"); + assert_eq!(addr.port(), 4318, "OTLP HTTP default port"); + let IpAddr::V4(ip) = addr.ip() else { + panic!("relay address must be IPv4"); + }; + assert!( + !ip.is_loopback(), + "loopback would bypass seccomp mediation and never reach the supervisor" + ); + assert_eq!(ip, Ipv4Addr::new(192, 0, 0, 8), "RFC 7600 dummy address"); + let [a, b, ..] = ip.octets(); + assert!( + !(a == 198 && (b == 18 || b == 19)), + "198.18.0.0/15 is the policy DNS synthetic answer pool" + ); + assert_eq!(OTLP_RELAY_ENDPOINT, format!("http://{OTLP_RELAY_ADDR}")); + assert_eq!( + OTLP_RELAY_TRACES_ENDPOINT, + format!("{OTLP_RELAY_ENDPOINT}/v1/traces") + ); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/openshell-otel-test-support/Cargo.toml b/crates/openshell-otel-test-support/Cargo.toml index 8e9cebd516..ab80c855df 100644 --- a/crates/openshell-otel-test-support/Cargo.toml +++ b/crates/openshell-otel-test-support/Cargo.toml @@ -13,6 +13,7 @@ repository.workspace = true [dependencies] openshell-otel = { path = "../openshell-otel" } opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "logs", "trace"] } +prost = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true, features = ["gzip", "tls-aws-lc"] } diff --git a/crates/openshell-otel-test-support/src/lib.rs b/crates/openshell-otel-test-support/src/lib.rs index 04fa3f1f91..4058b5a190 100644 --- a/crates/openshell-otel-test-support/src/lib.rs +++ b/crates/openshell-otel-test-support/src/lib.rs @@ -3,6 +3,64 @@ //! Shared OTLP collector fixture for `OpenShell` tracing tests. +/// Helpers for tests that exercise the supervisor's agent trace relay from +/// the agent side: the attribution keys the relay sets and a builder for the +/// protobuf export request an SDK would send. +pub mod relay { + use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; + use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value}; + use opentelemetry_proto::tonic::resource::v1::Resource; + use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span}; + use prost::Message; + + /// Resource attribute the relay sets to the sandbox id. + pub const SANDBOX_ID_KEY: &str = "openshell.sandbox.id"; + /// Resource attribute the relay sets to mark agent-originated traces. + pub const SOURCE_KEY: &str = "openshell.telemetry.source"; + /// Value of [`SOURCE_KEY`] on relayed traces. + pub const SOURCE_VALUE: &str = "agent"; + + /// A string resource or span attribute. + #[must_use] + pub fn string_attribute(key: &str, value: &str) -> KeyValue { + KeyValue { + key: key.to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(value.to_string())), + }), + key_strindex: 0, + } + } + + /// Protobuf-encoded `ExportTraceServiceRequest` with one span named + /// `span_name` and the given string resource attributes. + #[must_use] + pub fn encoded_trace_request(span_name: &str, attributes: &[(&str, &str)]) -> Vec { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: attributes + .iter() + .map(|(key, value)| string_attribute(key, value)) + .collect(), + ..Default::default() + }), + scope_spans: vec![ScopeSpans { + spans: vec![Span { + name: span_name.to_string(), + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }], + } + .encode_to_vec() + } +} + +use std::collections::{HashMap, HashSet}; +use std::net::SocketAddr; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -18,6 +76,9 @@ use opentelemetry_proto::tonic::common::v1::any_value::Value; use opentelemetry_proto::tonic::resource::v1::Resource; use opentelemetry_proto::tonic::trace::v1::Span; +const LOOPBACK_EPHEMERAL: SocketAddr = + SocketAddr::new(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), 0); + /// Serialize tracing tests and keep their callsites enabled process-wide. pub async fn tracing_test_lock() -> tokio::sync::MutexGuard<'static, ()> { static LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -82,25 +143,47 @@ pub struct ReceivedTraces { pub gateway_names: Vec, pub compute_drivers: Vec, pub encodings: Vec, + /// One map per received `ResourceSpans`, holding its string-valued + /// resource attributes. + pub resource_attributes: Vec>, + /// The resource attributes of the batch each entry of `spans` arrived + /// in, aligned index by index with `spans`. + pub span_resources: Vec>, + /// Resource attribute keys that appeared more than once within a single + /// resource. The maps above keep only the last value for such a key, so + /// tests that assert a value was replaced rather than appended must also + /// assert this list is empty. + pub duplicate_resource_keys: Vec, } impl ReceivedTraces { - fn record_resource(&mut self, resource: Option) { + /// Records the well-known resource attributes and returns all + /// string-valued ones, keeping the last value of a duplicated key. + fn record_resource(&mut self, resource: Option) -> HashMap { + let mut attributes = HashMap::new(); + let mut seen_keys = HashSet::new(); for attribute in resource .into_iter() .flat_map(|resource| resource.attributes) { + // Track duplicates for every value type; the string map keeps + // only the last value per key. + if !seen_keys.insert(attribute.key.clone()) { + self.duplicate_resource_keys.push(attribute.key.clone()); + } let Some(Value::StringValue(value)) = attribute.value.and_then(|value| value.value) else { continue; }; match attribute.key.as_str() { - "service.name" => self.service_names.push(value), - "openshell.gateway.name" => self.gateway_names.push(value), - "openshell.gateway.compute_driver" => self.compute_drivers.push(value), + "service.name" => self.service_names.push(value.clone()), + "openshell.gateway.name" => self.gateway_names.push(value.clone()), + "openshell.gateway.compute_driver" => self.compute_drivers.push(value.clone()), _ => {} } + attributes.insert(attribute.key, value); } + attributes } } @@ -129,10 +212,14 @@ impl TraceService for Collector { received.encodings.push(encoding.to_string()); } for resource_span in request.into_inner().resource_spans { - received.record_resource(resource_span.resource); + let attributes = received.record_resource(resource_span.resource); for scope_span in resource_span.scope_spans { - received.spans.extend(scope_span.spans); + for span in scope_span.spans { + received.spans.push(span); + received.span_resources.push(attributes.clone()); + } } + received.resource_attributes.push(attributes); } } self.exported.notify_one(); @@ -186,26 +273,44 @@ pub struct OtlpTestServer { } impl OtlpTestServer { + /// Starts the collector on an ephemeral loopback port. pub async fn start() -> Self { - Self::start_with(None).await + Self::start_on(LOOPBACK_EPHEMERAL).await } /// Serve over TLS at `https://localhost:` using `identity`. pub async fn start_tls(identity: tonic::transport::Identity) -> Self { - Self::start_with(Some(identity)).await + Self::try_start_with(LOOPBACK_EPHEMERAL, Some(identity)) + .await + .expect("OTLP test collector should bind a loopback listener") } - async fn start_with(identity: Option) -> Self { + /// Starts the collector on a fixed address, for tests whose peer learned + /// the collector endpoint before the collector existed. + pub async fn start_on(addr: SocketAddr) -> Self { + Self::try_start_on(addr) + .await + .expect("OTLP test collector should bind its listener") + } + + /// [`Self::start_on`] that reports a bind failure instead of panicking, + /// so a test can retry with another port when a parallel test took it. + pub async fn try_start_on(addr: SocketAddr) -> std::io::Result { + Self::try_start_with(addr, None).await + } + + async fn try_start_with( + addr: SocketAddr, + identity: Option, + ) -> std::io::Result { let received = Arc::new(Mutex::new(ReceivedTraces::default())); let exported = Arc::new(tokio::sync::Notify::new()); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("OTLP test collector should bind a loopback listener"); - let port = listener.local_addr().unwrap().port(); + let listener = tokio::net::TcpListener::bind(addr).await?; + let local_addr = listener.local_addr()?; let endpoint = if identity.is_some() { - format!("https://localhost:{port}") + format!("https://localhost:{}", local_addr.port()) } else { - format!("http://127.0.0.1:{port}") + format!("http://{local_addr}") }; let collector = Collector { received: Arc::clone(&received), @@ -234,13 +339,13 @@ impl OtlpTestServer { ) .await }); - Self { + Ok(Self { endpoint, received, exported, shutdown, task, - } + }) } #[must_use] @@ -254,6 +359,15 @@ impl OtlpTestServer { .expect("OTLP export should complete"); } + /// Snapshot of everything received so far, without stopping the server. + #[must_use] + pub fn received(&self) -> ReceivedTraces { + self.received + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + pub async fn shutdown(self) -> ReceivedTraces { self.shutdown .send(()) diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index 32d56c50f3..1da8e6adf1 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -182,6 +182,13 @@ impl LocalBoundaryExec { } } crate::process::strip_proxy_env_std(&mut command); + // Point OTel SDKs at the supervisor's trace relay, after the creation + // request's environment and the provider variables, matching the main + // process. A per-exec `spec.env` value applied below still wins for + // that exec only. + for (key, value) in crate::child_env::otlp_relay_env_vars() { + command.env(key, value); + } for (key, value) in &spec.env { if !key.starts_with(crate::process::RESERVED_ENV_PREFIX) { command.env(key, value); @@ -769,6 +776,63 @@ mod tests { assert_eq!(command.get_args().collect::>(), ["-i"]); } + fn env_of(command: &Command, key: &str) -> Option { + command + .get_envs() + .filter(|(name, _)| *name == std::ffi::OsStr::new(key)) + .last() + .and_then(|(_, value)| value.map(|value| value.to_string_lossy().into_owned())) + } + + #[test] + fn exec_children_see_the_relay_endpoint_unless_the_exec_overrides_it() { + let executor = executor(); + let spec = |env: Vec<(String, String)>| ExecSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + shell: None, + runtime_helper: None, + env, + workdir: None, + pty: false, + }; + + let command = executor.command(&spec(Vec::new())).expect("resolve exec"); + assert_eq!( + env_of( + &command, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT + ) + .as_deref(), + Some(openshell_core::sandbox_env::OTLP_RELAY_ENDPOINT) + ); + assert_eq!( + env_of( + &command, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL + ) + .as_deref(), + Some("http/protobuf") + ); + + // A per-exec environment is a developer's explicit instruction and + // is applied last (contracts/sandbox-environment.md). + let command = executor + .command(&spec(vec![( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.to_string(), + "http://127.0.0.1:4318".to_string(), + )])) + .expect("resolve exec"); + assert_eq!( + env_of( + &command, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT + ) + .as_deref(), + Some("http://127.0.0.1:4318") + ); + } + #[tokio::test] async fn non_pty_exec_preserves_stdin_stdout_and_stderr() { let mut session = executor() diff --git a/crates/openshell-sandbox/src/child_env.rs b/crates/openshell-sandbox/src/child_env.rs index 50549a7439..4302482c8a 100644 --- a/crates/openshell-sandbox/src/child_env.rs +++ b/crates/openshell-sandbox/src/child_env.rs @@ -3,6 +3,37 @@ use std::path::Path; +use openshell_core::sandbox_env::{ + OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, OTLP_RELAY_ENDPOINT, OTLP_RELAY_TRACES_ENDPOINT, +}; + +/// OpenTelemetry exporter variables every workload process receives. +/// +/// They point agent SDKs at the supervisor's OTLP relay and are applied after +/// the environment from the sandbox creation request, so an endpoint named +/// there never wins: it would either point into the fenced network and fail +/// or bypass attribution. The signal-specific traces variables are set as +/// well because SDKs give them precedence over the generic ones, and +/// compression is disabled because the relay refuses compressed bodies. The +/// values are the same whether or not a collector is configured; without one +/// the relay address refuses connections immediately. A per-exec environment +/// still overrides them for that exec only. +pub fn otlp_relay_env_vars() -> [(&'static str, &'static str); 6] { + [ + (OTEL_EXPORTER_OTLP_ENDPOINT, OTLP_RELAY_ENDPOINT), + (OTEL_EXPORTER_OTLP_PROTOCOL, "http/protobuf"), + ( + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + OTLP_RELAY_TRACES_ENDPOINT, + ), + (OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "http/protobuf"), + (OTEL_EXPORTER_OTLP_COMPRESSION, "none"), + (OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, "none"), + ] +} + pub fn tls_env_vars( ca_cert_path: &Path, combined_bundle_path: &Path, @@ -50,4 +81,22 @@ mod tests { assert!(stdout.contains("CURL_CA_BUNDLE=/etc/openshell-tls/ca-bundle.pem")); assert!(stdout.contains("GIT_SSL_CAINFO=/etc/openshell-tls/ca-bundle.pem")); } + + #[test] + fn otlp_relay_env_points_sdks_at_the_reserved_endpoint() { + assert_eq!( + otlp_relay_env_vars(), + [ + ("OTEL_EXPORTER_OTLP_ENDPOINT", "http://192.0.0.8:4318"), + ("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"), + ( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "http://192.0.0.8:4318/v1/traces" + ), + ("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "http/protobuf"), + ("OTEL_EXPORTER_OTLP_COMPRESSION", "none"), + ("OTEL_EXPORTER_OTLP_TRACES_COMPRESSION", "none"), + ] + ); + } } diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 1b0f853d23..430fd78b7e 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -269,6 +269,36 @@ pub(crate) fn configured_user_environment() -> HashMap { .unwrap_or_default() } +/// Composes the environment of a canonical workload process. Shared by the +/// PTY and pipe spawn paths so both apply the same layering: the inherited +/// sandbox environment minus the reserved `OPENSHELL_` namespace, the +/// declared user environment, supervisor-only and proxy variables stripped, +/// provider placeholders, TLS trust paths, and finally the OTLP relay +/// endpoint, applied last so a caller-supplied endpoint never wins. +fn compose_canonical_process_environment( + cmd: &mut Command, + policy: &SandboxPolicy, + workspace: &ResolvedWorkspace, + interactive: bool, + user_environment: &HashMap, + provider_env: &HashMap, + ca_paths: Option<&(PathBuf, PathBuf)>, +) { + apply_canonical_process_environment(cmd, policy, workspace, interactive, user_environment); + strip_supervisor_only_env(cmd); + inject_provider_env(cmd, provider_env); + strip_proxy_env(cmd); + // Set TLS trust store env vars so sandbox processes trust the ephemeral CA. + if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { + for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { + cmd.env(key, value); + } + } + for (key, value) in child_env::otlp_relay_env_vars() { + cmd.env(key, value); + } +} + #[cfg(unix)] pub fn harden_child_process() -> Result<()> { use rustix::process::{Resource, Rlimit, setrlimit}; @@ -511,30 +541,20 @@ impl ProcessHandle { // inherited environment. The entrypoint drops to the sandbox user // before `exec`; without this strip, sandbox code could recover // supervisor credentials from its inherited environment. - apply_canonical_process_environment( + compose_canonical_process_environment( &mut cmd, policy, workspace, interactive, &configured_user_environment(), + provider_env, + ca_paths, ); - strip_supervisor_only_env(&mut cmd); - - inject_provider_env(&mut cmd, provider_env); if let Some(dir) = workspace.root() { cmd.current_dir(dir); } - strip_proxy_env(&mut cmd); - - // Set TLS trust store env vars so sandbox processes trust the ephemeral CA - if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { - for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { - cmd.env(key, value); - } - } - // Probe Landlock availability and emit OCSF logs from the parent // process where the tracing subscriber is functional. The child's // pre_exec context cannot reliably emit structured logs. @@ -671,30 +691,20 @@ impl ProcessHandle { // Strip supervisor-only identity material from the entrypoint's // inherited environment. - apply_canonical_process_environment( + compose_canonical_process_environment( &mut cmd, policy, workspace, interactive, &configured_user_environment(), + provider_env, + ca_paths, ); - strip_supervisor_only_env(&mut cmd); - - inject_provider_env(&mut cmd, provider_env); if let Some(dir) = workspace.root() { cmd.current_dir(dir); } - strip_proxy_env(&mut cmd); - - // Set TLS trust store env vars so sandbox processes trust the ephemeral CA - if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { - for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { - cmd.env(key, value); - } - } - // Create a dedicated session for PTY children and a dedicated process // group for pipe children so attachment signals target only the // canonical workload tree. @@ -1147,34 +1157,74 @@ mod tests { assert_eq!(variables.get("TERM"), Some(&"xterm-256color")); } + /// Re-executes the test binary for `test_path` with `env` added to its + /// environment, so a test can observe an inherited environment without + /// touching the harness process. Returns `true` in the parent once the + /// child passed, and `false` in the child, which then runs the test body. + #[cfg(unix)] + fn run_in_isolated_child(marker: &str, test_path: &str, env: &[(&str, String)]) -> bool { + if std::env::var_os(marker).is_some() { + return false; + } + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", test_path, "--nocapture"]) + .env(marker, "1"); + for (key, value) in env { + command.env(key, value); + } + let output = command.output().expect("run isolated environment test"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "isolated environment test failed:\n{stdout}\n{stderr}" + ); + // A filter that matches nothing also exits 0; make sure the test ran. + assert!( + stdout.contains("1 passed"), + "the isolated test did not run; check the test path {test_path}:\n{stdout}" + ); + true + } + + /// Runs an `env` probe command on a fresh current-thread runtime and + /// parses its `KEY=value` output. + #[cfg(unix)] + fn probe_environment(mut cmd: Command) -> HashMap { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let output = runtime + .block_on(async { cmd.output().await }) + .expect("run environment probe"); + assert!(output.status.success()); + String::from_utf8(output.stdout) + .unwrap() + .lines() + .filter_map(|line| line.split_once('=')) + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect() + } + #[cfg(unix)] #[test] fn canonical_process_drops_inherited_reserved_environment() { // The sandbox's own control variables live in the reserved // OPENSHELL_ namespace and must not reach the workload, while the - // image's ordinary ENV must. Run in a fresh copy of the test binary - // so the test harness environment is untouched. - const CHILD_MARKER: &str = "OPENSHELL_TEST_RESERVED_ENV_CHILD"; - if std::env::var_os(CHILD_MARKER).is_none() { - let status = std::process::Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "process::tests::canonical_process_drops_inherited_reserved_environment", - "--nocapture", - ]) - .env(CHILD_MARKER, "1") - .env(openshell_core::sandbox_env::LOG_LEVEL, "debug") - .env(openshell_core::sandbox_env::USER_ENVIRONMENT, "{}") - .env("IMAGE_LANG", "keep") - .status() - .expect("run isolated environment test"); - assert!(status.success(), "isolated environment test failed"); + // image's ordinary ENV must. + if run_in_isolated_child( + "OPENSHELL_TEST_RESERVED_ENV_CHILD", + "process::tests::canonical_process_drops_inherited_reserved_environment", + &[ + (openshell_core::sandbox_env::LOG_LEVEL, "debug".into()), + (openshell_core::sandbox_env::USER_ENVIRONMENT, "{}".into()), + ("IMAGE_LANG", "keep".into()), + ], + ) { return; } - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); let current_user = User::from_uid(nix::unistd::geteuid()).unwrap().unwrap(); let policy = policy_with_process(ProcessPolicy { run_as_user: Some(current_user.name), @@ -1190,28 +1240,25 @@ mod tests { false, &HashMap::from([("DECLARED".into(), "yes".into())]), ); - let output = runtime - .block_on(async { cmd.output().await }) - .expect("run environment probe"); - assert!(output.status.success()); - let environment = String::from_utf8(output.stdout).unwrap(); - let variables: HashMap<_, _> = environment - .lines() - .filter_map(|line| line.split_once('=')) - .collect(); + let variables = probe_environment(cmd); assert!( !variables .keys() .any(|key| key.starts_with(RESERVED_ENV_PREFIX) - && *key != openshell_core::sandbox_env::SANDBOX), + && key.as_str() != openshell_core::sandbox_env::SANDBOX), "reserved variables reached the workload: {variables:?}" ); assert_eq!( - variables.get(openshell_core::sandbox_env::SANDBOX), - Some(&"1") + variables + .get(openshell_core::sandbox_env::SANDBOX) + .map(String::as_str), + Some("1") ); - assert_eq!(variables.get("IMAGE_LANG"), Some(&"keep")); - assert_eq!(variables.get("DECLARED"), Some(&"yes")); + assert_eq!( + variables.get("IMAGE_LANG").map(String::as_str), + Some("keep") + ); + assert_eq!(variables.get("DECLARED").map(String::as_str), Some("yes")); } #[cfg(unix)] @@ -1266,6 +1313,113 @@ mod tests { } } + /// The production composition applies the relay endpoint after the + /// declared environment and strips the collector address. Runs in a + /// re-executed copy of the test binary so the collector address can sit + /// in the inherited environment, where `OPENSHELL_OTLP_ENDPOINT` lives + /// for a real supervisor, without touching the harness environment. + #[cfg(unix)] + #[test] + fn canonical_process_sees_relay_endpoint_and_never_the_collector() { + const COLLECTOR_HOST: &str = "collector.invalid"; + if run_in_isolated_child( + "OPENSHELL_TEST_RELAY_ENV_CHILD", + "process::tests::canonical_process_sees_relay_endpoint_and_never_the_collector", + &[( + openshell_core::sandbox_env::OTLP_ENDPOINT, + format!("http://{COLLECTOR_HOST}:4317"), + )], + ) { + return; + } + let current_user = User::from_uid(nix::unistd::geteuid()).unwrap().unwrap(); + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some(current_user.name), + run_as_group: None, + }); + // Mirror production: inherit the sandbox environment, no env_clear. + let mut cmd = Command::new("/usr/bin/env"); + cmd.stdout(StdStdio::piped()); + let declared = HashMap::from([ + ( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.into(), + "http://example.invalid".into(), + ), + ( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL.into(), + "grpc".into(), + ), + // SDKs prefer the signal-specific variables; a declared one must + // not redirect traces around the relay either. + ( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.into(), + "http://example.invalid/v1/traces".into(), + ), + ( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION.into(), + "gzip".into(), + ), + ]); + // A provider could also try to name an endpoint; the relay must win + // over that layer too. + let provider_env = HashMap::from([( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.to_string(), + "http://provider.invalid".to_string(), + )]); + compose_canonical_process_environment( + &mut cmd, + &policy, + &ResolvedWorkspace::default(), + false, + &declared, + &provider_env, + None, + ); + let variables = probe_environment(cmd); + assert_eq!( + variables + .get(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT) + .map(String::as_str), + Some(openshell_core::sandbox_env::OTLP_RELAY_ENDPOINT), + "the relay endpoint wins over the declared one: {variables:?}" + ); + assert_eq!( + variables + .get(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL) + .map(String::as_str), + Some("http/protobuf") + ); + assert_eq!( + variables + .get(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) + .map(String::as_str), + Some(openshell_core::sandbox_env::OTLP_RELAY_TRACES_ENDPOINT), + "the traces-specific endpoint is forced too: {variables:?}" + ); + assert_eq!( + variables + .get(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION) + .map(String::as_str), + Some("none") + ); + assert!( + !variables.contains_key(openshell_core::sandbox_env::OTLP_ENDPOINT), + "the collector variable reached the workload: {variables:?}" + ); + assert!( + !variables + .values() + .any(|value| value.contains(COLLECTOR_HOST)), + "the collector address reached the workload: {variables:?}" + ); + assert!( + !variables + .values() + .any(|value| value.contains("provider.invalid")), + "a provider endpoint overrode the relay: {variables:?}" + ); + } + #[cfg(unix)] #[allow(unsafe_code)] fn probe_hardened_child(probe: unsafe fn() -> i64) -> i64 { diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index bd93c78c57..39a71a5aeb 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -47,10 +47,12 @@ use openshell_ocsf::{ Url as OcsfUrl, ocsf_emit, }; use openshell_sandbox_backend::NetworkMediationSource; +use std::future::Future; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, LazyLock}; use tokio::io::{ AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, @@ -81,6 +83,51 @@ enum ProxyAcceptError { Source(openshell_isolation_interface::contract::BackendError), } +/// A future that serves one reserved-destination stream to completion. +pub type ReservedStreamFuture = Pin + Send + 'static>>; + +/// Serves workload connections to a reserved, unroutable destination inside +/// the supervisor instead of dialing upstream. +/// +/// The proxy consults the handler before the sandbox commits the workload +/// socket. `None` refuses the open, which the workload observes as `EAGAIN`; +/// `Some` accepts it, after which the proxy answers `RelayReady` and drives +/// the returned future on its own task. Nothing may be read from `stream` +/// until that future is polled. +pub trait ReservedStreamHandler: Send + Sync { + fn serve(&self, stream: BoundaryDuplexStream) -> Option; +} + +/// A reserved destination the proxy serves locally. +#[derive(Clone)] +pub struct ReservedDestination { + pub addr: SocketAddr, + pub handler: Arc, +} + +/// The OTLP relay address, known to the proxy even when no handler is +/// installed so a workload connect to it never falls through to policy +/// evaluation and upstream dialing. +static OTLP_RELAY_ADDR: LazyLock = LazyLock::new(|| { + openshell_core::sandbox_env::OTLP_RELAY_ADDR + .parse() + .expect("OTLP_RELAY_ADDR is a valid socket address") +}); + +/// Set once the first relay-address open was denied for lack of a handler, +/// so the warning is logged once per supervisor lifetime and later denials +/// drop to debug. +static RELAY_DENIAL_WARNED: AtomicBool = AtomicBool::new(false); + +#[cfg(test)] +pub(crate) fn reset_relay_denial_warning() { + RELAY_DENIAL_WARNED.store(false, Ordering::SeqCst); +} + +fn is_reserved_relay_destination(destination: SocketAddr) -> bool { + destination == *OTLP_RELAY_ADDR +} + use self::destination::{ DestinationDenial, DestinationDenialKind, DestinationRequest, DestinationValidationPlan, build_pinned_validation_plan, build_validation_plan, validate_destination, @@ -258,6 +305,7 @@ impl ProxyHandle { policy_dns_store: Option>, direct_listener_identity: Option, required_proxy_authorization: Option>, + reserved_destination: Option, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -427,6 +475,7 @@ impl ProxyHandle { let trusted_gateway = *trusted_host_gateway; let dtx = denial_tx.clone(); let has_policy_local = policy_local_ctx.is_some(); + let reserved = reserved_destination.clone(); tokio::spawn(async move { if let Some(connection) = preauthorize_transparent_open( connection, @@ -437,6 +486,7 @@ impl ProxyHandle { trusted_gateway, has_policy_local, dtx.as_ref(), + reserved.as_ref(), ) .await { @@ -582,6 +632,7 @@ async fn preauthorize_transparent_open( trusted_host_gateway: Option, has_policy_local: bool, denial_tx: Option<&mpsc::UnboundedSender>, + reserved: Option<&ReservedDestination>, ) -> Option { let PendingTcpOpen { stream, @@ -596,6 +647,48 @@ async fn preauthorize_transparent_open( timing, operation: "tcp", }; + // A reserved destination is served inside the supervisor. It is decided + // before host mapping, policy, and SSRF validation because the address is + // an unroutable label, not a place anything could dial. + if let Some(reserved) = reserved.filter(|reserved| reserved.addr == destination) { + if let Some(served) = reserved.handler.serve(stream) { + debug!(%destination, "Serving staged connection on reserved destination"); + // If the sandbox side is already gone, dropping `served` closes + // the stream and releases the handler's slot. + if completion.send(TcpOpenDecision::RelayReady).is_ok() { + tokio::spawn(served); + } + } else { + // The handler refuses when its connection cap is reached or once + // it has stopped accepting; it knows which, the proxy does not. + debug!( + %destination, + "reserved destination handler refused the staged connection" + ); + let _ = completion.send(TcpOpenDecision::Denied(TcpOpenDenial::ResourceExhausted)); + } + return None; + } + if is_reserved_relay_destination(destination) { + // No relay is running. A standard OTel SDK retries every few seconds + // for the life of the agent, so this is logged once at warn and then + // at debug, and deliberately emits no OCSF denial event: the address + // is an OpenShell label, not an external endpoint, and the relay's + // inactive state was already recorded at startup. + if RELAY_DENIAL_WARNED.swap(true, Ordering::SeqCst) { + debug!( + %destination, + "agent OTLP export attempted but no collector is configured" + ); + } else { + warn!( + %destination, + "agent OTLP export attempted but no collector is configured" + ); + } + let _ = completion.send(TcpOpenDecision::Denied(TcpOpenDenial::PolicyDenied)); + return None; + } if openshell_core::google_cloud::is_metadata_destination(destination) { let identity_check = binary_identity .as_ref() @@ -7474,7 +7567,8 @@ process: None, None, false, - Some(&denial_tx) + Some(&denial_tx), + None, ) .await .is_some() @@ -7492,6 +7586,7 @@ process: None, false, Some(&denial_tx), + None, ) .await .is_none() @@ -7515,7 +7610,8 @@ process: None, None, false, - Some(&denial_tx) + Some(&denial_tx), + None, ) .await .is_none() @@ -7802,6 +7898,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } None, false, None, + None, ) .await; if valid_identity { @@ -7847,6 +7944,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } None, false, None, + None, ) .await .expect("policy-backed mapping is admitted"); @@ -7883,6 +7981,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } None, false, Some(&denial_tx), + None, ) .await .is_none() @@ -7925,6 +8024,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } None, false, Some(denial_tx), + None, ) .await .is_some(); @@ -8095,10 +8195,11 @@ process: { run_as_user: sandbox, run_as_group: sandbox } timing: MediationTiming::default(), decision, }; - let (stream, supplied_identity, socket_addrs, transparent) = - preauthorize_transparent_open(pending, None, &engine, &cache, None, None, true, None) - .await - .expect("sandbox-local open is admitted"); + let (stream, supplied_identity, socket_addrs, transparent) = preauthorize_transparent_open( + pending, None, &engine, &cache, None, None, true, None, None, + ) + .await + .expect("sandbox-local open is admitted"); assert_eq!(completion.await.unwrap(), TcpOpenDecision::RelayReady); let proposals = AgentProposals::new(true); @@ -8228,10 +8329,11 @@ network_policies: timing: MediationTiming::default(), decision, }; - let (stream, supplied_identity, socket_addrs, transparent) = - preauthorize_transparent_open(pending, None, &engine, &cache, None, None, false, None) - .await - .expect("open is admitted"); + let (stream, supplied_identity, socket_addrs, transparent) = preauthorize_transparent_open( + pending, None, &engine, &cache, None, None, false, None, None, + ) + .await + .expect("open is admitted"); assert_eq!(completion.await.unwrap(), TcpOpenDecision::RelayReady); // The workload writes before the proxy reads anything, so its request sits // right behind the synthesized CONNECT header in the proxy's first read. @@ -8381,7 +8483,8 @@ process: None, None, false, - None + None, + None, ) .await .is_none() @@ -8466,7 +8569,8 @@ process: None, None, false, - None + None, + None, ) .await .is_none() @@ -8477,6 +8581,338 @@ process: ); } + // ---- reserved destinations ---- + + /// Serialises the tests that observe the process-wide relay-denial + /// warning flag. + static RELAY_DENIAL_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + /// Policy that allows one public endpoint and nothing else. It never + /// mentions the relay address, so a served relay stream proves the + /// reserved path bypassed policy. + fn reserved_test_engine() -> OpaEngine { + OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + allowed: + name: allowed + endpoints: + - host: 203.0.113.7 + port: 443 + binaries: + - path: /usr/bin/curl +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .unwrap() + } + + /// A staged open from `/usr/bin/curl` to `destination`, plus the + /// decision receiver and the workload-side peer of the staged stream. + fn staged_open( + engine: &OpaEngine, + destination: &str, + ) -> ( + PendingTcpOpen, + tokio::sync::oneshot::Receiver, + tokio::io::DuplexStream, + ) { + let (stream, peer) = tokio::io::duplex(256); + let (decision, completion) = tokio::sync::oneshot::channel(); + let open = PendingTcpOpen { + stream: Box::new(stream), + binary_identity: Ok(ContractBinaryIdentity { + executable: ContractExecutableIdentity { + path: PathBuf::from("/usr/bin/curl"), + digest: Some("00".repeat(32).parse().unwrap()), + }, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }), + destination: destination.parse().unwrap(), + socket: Some( + openshell_isolation_interface::contract::NetworkSocketMetadata { + socket_cookie: 7, + nonblocking: false, + process_generation: 1, + }, + ), + policy_generation: engine.current_generation(), + timing: MediationTiming::default(), + decision, + }; + (open, completion, peer) + } + + /// Handler double: counts calls, optionally refuses, and hands every + /// accepted stream back to the test. + struct RecordingReservedHandler { + accept: bool, + served: mpsc::UnboundedSender, + calls: Arc, + } + + impl ReservedStreamHandler for RecordingReservedHandler { + fn serve(&self, stream: BoundaryDuplexStream) -> Option { + self.calls.fetch_add(1, Ordering::SeqCst); + if !self.accept { + return None; + } + let served = self.served.clone(); + Some(Box::pin(async move { + let _ = served.send(stream); + })) + } + } + + fn reserved_relay( + accept: bool, + ) -> ( + ReservedDestination, + mpsc::UnboundedReceiver, + Arc, + ) { + let (served, served_rx) = mpsc::unbounded_channel(); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let handler = RecordingReservedHandler { + accept, + served, + calls: Arc::clone(&calls), + }; + let reserved = ReservedDestination { + addr: *OTLP_RELAY_ADDR, + handler: Arc::new(handler), + }; + (reserved, served_rx, calls) + } + + #[tokio::test] + async fn reserved_destination_is_served_without_policy_evaluation() { + let engine = reserved_test_engine(); + let (reserved, mut served_rx, calls) = reserved_relay(true); + let (open, decision, mut peer) = + staged_open(&engine, openshell_core::sandbox_env::OTLP_RELAY_ADDR); + + assert!( + preauthorize_transparent_open( + open, + None, + &engine, + &BinaryIdentityCache::new(), + None, + None, + false, + None, + Some(&reserved) + ) + .await + .is_none(), + "a reserved open never becomes a proxy connection" + ); + assert_eq!(decision.await.unwrap(), TcpOpenDecision::RelayReady); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + // The handler holds the workload's stream: bytes written by the + // workload side arrive at the handler. + let mut stream = tokio::time::timeout(std::time::Duration::from_secs(1), served_rx.recv()) + .await + .expect("handler future ran") + .expect("handler received the stream"); + peer.write_all(b"POST /v1/traces").await.unwrap(); + let mut head = [0u8; 15]; + stream.read_exact(&mut head).await.unwrap(); + assert_eq!(&head, b"POST /v1/traces"); + } + + #[tokio::test] + async fn reserved_destination_is_refused_when_handler_is_at_capacity() { + let engine = reserved_test_engine(); + let (reserved, _served_rx, calls) = reserved_relay(false); + let (open, decision, _peer) = + staged_open(&engine, openshell_core::sandbox_env::OTLP_RELAY_ADDR); + + assert!( + preauthorize_transparent_open( + open, + None, + &engine, + &BinaryIdentityCache::new(), + None, + None, + false, + None, + Some(&reserved) + ) + .await + .is_none() + ); + assert_eq!( + decision.await.unwrap(), + TcpOpenDecision::Denied(TcpOpenDenial::ResourceExhausted), + "refusal happens before the sandbox commits the socket" + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn reserved_relay_destination_is_denied_without_a_handler() { + let _serialised = RELAY_DENIAL_TEST_LOCK.lock().await; + let engine = reserved_test_engine(); + let (open, decision, _peer) = + staged_open(&engine, openshell_core::sandbox_env::OTLP_RELAY_ADDR); + + assert!( + preauthorize_transparent_open( + open, + None, + &engine, + &BinaryIdentityCache::new(), + None, + None, + false, + None, + None + ) + .await + .is_none() + ); + assert_eq!( + decision.await.unwrap(), + TcpOpenDecision::Denied(TcpOpenDenial::PolicyDenied), + "the relay address must never reach policy evaluation or dialing" + ); + } + + /// Counts events at a given level for the duration of a test. + struct LevelCounter { + level: tracing::Level, + count: Arc, + } + + impl tracing_subscriber::Layer for LevelCounter { + fn on_event( + &self, + event: &tracing::Event<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + if *event.metadata().level() == self.level { + self.count.fetch_add(1, Ordering::SeqCst); + } + } + } + + #[tokio::test] + async fn no_handler_denial_logs_warn_once_then_debug() { + use tracing_subscriber::layer::SubscriberExt as _; + + let _serialised = RELAY_DENIAL_TEST_LOCK.lock().await; + reset_relay_denial_warning(); + let warnings = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let subscriber = tracing_subscriber::registry().with(LevelCounter { + level: tracing::Level::WARN, + count: Arc::clone(&warnings), + }); + let _subscriber = tracing::subscriber::set_default(subscriber); + + let engine = reserved_test_engine(); + for expected_warnings in [1, 1] { + let (open, decision, _peer) = + staged_open(&engine, openshell_core::sandbox_env::OTLP_RELAY_ADDR); + assert!( + preauthorize_transparent_open( + open, + None, + &engine, + &BinaryIdentityCache::new(), + None, + None, + false, + None, + None + ) + .await + .is_none() + ); + assert_eq!( + decision.await.unwrap(), + TcpOpenDecision::Denied(TcpOpenDenial::PolicyDenied) + ); + assert!(RELAY_DENIAL_WARNED.load(Ordering::SeqCst)); + assert_eq!( + warnings.load(Ordering::SeqCst), + expected_warnings, + "only the first denial warns; later ones log at debug" + ); + } + } + + #[tokio::test] + async fn reserved_handler_ignores_other_destinations() { + let engine = reserved_test_engine(); + let (reserved, _served_rx, calls) = reserved_relay(true); + + let (denied, denied_result) = { + let (open, decision, _peer) = staged_open(&engine, "203.0.113.8:443"); + (open, decision) + }; + assert!( + preauthorize_transparent_open( + denied, + None, + &engine, + &BinaryIdentityCache::new(), + None, + None, + false, + None, + Some(&reserved) + ) + .await + .is_none() + ); + assert_eq!( + denied_result.await.unwrap(), + TcpOpenDecision::Denied(TcpOpenDenial::PolicyDenied) + ); + + let (allowed, allowed_result) = { + let (open, decision, _peer) = staged_open(&engine, "203.0.113.7:443"); + (open, decision) + }; + assert!( + preauthorize_transparent_open( + allowed, + None, + &engine, + &BinaryIdentityCache::new(), + None, + None, + false, + None, + Some(&reserved) + ) + .await + .is_some(), + "policy-allowed destinations still flow to the proxy" + ); + assert_eq!(allowed_result.await.unwrap(), TcpOpenDecision::RelayReady); + + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "the handler is consulted only for its own address" + ); + } + struct FailedMediationSource; #[tokio::test] @@ -8878,6 +9314,7 @@ network_policies: {} None, None, None, + None, ) .await .expect("proxy starts before source accept"); diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 340b3984c5..9f8ce690f0 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -296,6 +296,7 @@ pub async fn run_networking( #[cfg(target_os = "linux")] transparent_runtime: Option, network_mediation_source: Option>, proxy_listener: Option, + reserved_destination: Option, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so @@ -613,6 +614,7 @@ pub async fn run_networking( proxy_listener .as_ref() .map(|proxy| proxy.authorization.clone()), + reserved_destination, ) .await?; Some(proxy_handle) diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index e6e7fcd6ba..645b813604 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -29,11 +29,17 @@ openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-mid openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } openshell-supervisor-process = { path = "../openshell-supervisor-process" } +bytes = { workspace = true } clap = { workspace = true } futures = { workspace = true } +http = { workspace = true } +http-body-util = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true, features = ["tokio", "http1", "server", "server-graceful"] } miette = { workspace = true } nix = { workspace = true } opentelemetry = { workspace = true } +opentelemetry-proto = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } rustls = { workspace = true } @@ -47,6 +53,7 @@ tokio = { workspace = true } tokio-rustls = { workspace = true } tokio-stream = { workspace = true } tonic = { workspace = true, features = ["channel", "tls-native-roots"] } +tonic-types = { workspace = true } tracing = { workspace = true } tracing-appender = { workspace = true } tracing-subscriber = { workspace = true } @@ -61,6 +68,7 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } +openshell-otel-test-support = { path = "../openshell-otel-test-support" } bytes = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 10964b8542..46e74225a9 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -88,6 +88,12 @@ Diagnostics and OCSF shorthand share a bounded, nonblocking stderr writer. With Console JSON is installed independently of optional file appenders and uses an INFO filter independent of the diagnostic filter. Its runtime enabled flag and target schema version are shared with the existing JSONL file layer. The supervisor retains the writer guards until shutdown. The gateway log push layer continues to emit shorthand only. +## OTLP agent trace relay + +The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_ENDPOINT`, the same one the supervisor uses for its own spans. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards batches over OTLP/gRPC with a lazy connection. + +Limits are fixed: 2 MiB per request body, at most 512 `ResourceSpans` entries, 256 KiB of encoded `Resource` fields, and 8192 resource attributes per request (each of these answers 413), a 10 second header-read timeout, 16 concurrent connections, 32 buffered batches, and a 5 second deadline per export attempt. Only the resources are decoded; spans are copied byte for byte, at most two requests at a time. Worst-case memory attributable to the agent is about 67 MiB of buffered batches, plus 32 MiB of bodies awaiting enrichment across the 16 connections, plus about 12 MiB for the two enrichments in progress. A full buffer answers 503 with `Retry-After`. Without a collector endpoint nothing starts and the proxy refuses the relay address. After the agent exits, `run_sandbox_with_backend` flushes buffered batches for up to 5 seconds before it reports the exit to the gateway, then logs a final summary of rejected and failed counts. + ## Safe runtime and parent liveness The supervisor drives the safe isolation-backend contract and network/policy diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index b4704bf623..a2ad0bdb68 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -704,6 +704,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { provider_workload_api_socket: None, // The binary loads the proxy trust directory from its environment. proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + otlp_endpoint: None, }; let bindings = crate::SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index 699880f1ad..6e90be6537 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -304,6 +304,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl trace_context: None, provider_workload_api_socket: None, proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), + otlp_endpoint: None, }; let bindings = SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 46eb9e8839..0c96fdcdf5 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -23,6 +23,7 @@ mod config_runtime; mod denial_aggregator; mod endpoint_status; mod mechanistic_mapper; +mod otlp_relay; mod provider_readiness; pub mod sandbox_instance_host; pub mod shared_host; @@ -634,6 +635,7 @@ pub async fn run_network_proxy( None, None, None, + None, ) .await?; @@ -791,6 +793,9 @@ pub struct SandboxInstanceConfig { /// SPIFFE Workload API socket for provider token grants; absent fails closed. pub provider_workload_api_socket: Option, pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, + /// OTLP/gRPC collector for relayed agent traces; `None` keeps the relay + /// inactive. + pub otlp_endpoint: Option, } impl SandboxInstanceConfig { @@ -939,6 +944,7 @@ pub async fn run_sandbox_instance( trace_context, provider_workload_api_socket, proxy_tls, + otlp_endpoint, } = config; // The owner binds the event context and log sink to the instance's // threads; the instance itself uses the rest. @@ -1260,6 +1266,11 @@ pub async fn run_sandbox_instance( (ready, backend_name, ca_file_paths) }; + // The agent trace relay must exist before networking starts so the + // handler is installed when the first staged connection arrives. + let (mut otlp_relay, otlp_reserved_destination) = + otlp_relay::activate(otlp_endpoint, sandbox_id.clone()); + let mut networking = Some( openshell_supervisor_network::run::run_networking( &policy, @@ -1293,6 +1304,7 @@ pub async fn run_sandbox_instance( None, remote_network_source, proxy_listener, + otlp_reserved_destination, ) .await?, ); @@ -1584,6 +1596,17 @@ pub async fn run_sandbox_instance( boundary_access .publish_main_exit(exit_code, await_main_process_attachment) .await; + // Flush relayed agent traces before the exit is reported so the last + // spans of a short run are not lost. Bounded; never fails. When the + // access plane is retained for exec sessions, the relay keeps serving + // them and shuts down at final teardown instead. + if retain_access { + if let Some(relay) = otlp_relay.as_ref() { + relay.flush(otlp_relay::FLUSH_TIMEOUT).await; + } + } else if let Some(relay) = otlp_relay.take() { + relay.shutdown(otlp_relay::FLUSH_TIMEOUT).await; + } // `shutdown_requested` has already completed when shutdown won the // lifecycle select above and must not be polled again. let mut completion_cancelled = !retain_access; @@ -1628,6 +1651,11 @@ pub async fn run_sandbox_instance( info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; } + // Exec sessions served after the main exit may have exported traces; + // flush those too before the relay goes away. + if let Some(relay) = otlp_relay.take() { + relay.shutdown(otlp_relay::FLUSH_TIMEOUT).await; + } drop(instance_readiness); drop(running); drop(boundary_access); @@ -4367,6 +4395,7 @@ mod tests { trace_context: None, provider_workload_api_socket: None, proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), + otlp_endpoint: None, }; let bindings = SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index de29e9509a..8fd0382019 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -596,6 +596,7 @@ fn main() -> Result<()> { .filter(|path| !path.is_empty()) .map(PathBuf::from), proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), + otlp_endpoint: args.otlp_endpoint.clone(), }; // Log push already shares these credentials and the logging // layers these OCSF JSON settings. diff --git a/crates/openshell-supervisor/src/otlp_relay/buffer.rs b/crates/openshell-supervisor/src/otlp_relay/buffer.rs new file mode 100644 index 0000000000..2512f69d1c --- /dev/null +++ b/crates/openshell-supervisor/src/otlp_relay/buffer.rs @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded buffer of enriched, encoded trace batches between the receiver +//! and the export task. +//! +//! Sending never blocks and never drops silently: a full buffer surfaces as +//! [`BufferFull`] so the receiver can answer 503 and the agent SDK retries. + +use bytes::Bytes; +use tokio::sync::mpsc; + +/// The buffer has no room for another batch. +#[derive(Debug, PartialEq, Eq)] +pub enum SendError { + /// Capacity reached; the batch was not buffered. + BufferFull, + /// The export task is gone; the batch was not buffered. + BufferClosed, +} + +impl std::fmt::Display for SendError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::BufferFull => f.write_str("relay buffer is full"), + Self::BufferClosed => f.write_str("relay buffer is closed"), + } + } +} + +impl std::error::Error for SendError {} + +/// Producer side, held by the receiver. +/// Not `Clone`: the receiver holds the only sender, so closing the buffer +/// at shutdown ends the export task's input. +pub struct BatchSender { + tx: mpsc::Sender, +} + +impl BatchSender { + /// Buffers `batch` without waiting. + pub fn try_send(&self, batch: Bytes) -> Result<(), SendError> { + self.tx.try_send(batch).map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => SendError::BufferFull, + mpsc::error::TrySendError::Closed(_) => SendError::BufferClosed, + }) + } +} + +/// Consumer side, held by the export task. +pub struct BatchReceiver { + rx: mpsc::Receiver, +} + +impl BatchReceiver { + /// Next buffered batch in arrival order, or `None` once every sender has + /// been dropped and the buffer is drained. + pub async fn recv(&mut self) -> Option { + self.rx.recv().await + } +} + +/// Creates a buffer holding at most `capacity` batches. +pub fn new_buffer(capacity: usize) -> (BatchSender, BatchReceiver) { + let (tx, rx) = mpsc::channel(capacity); + (BatchSender { tx }, BatchReceiver { rx }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn capacity_is_honoured_and_full_is_reported() { + let (tx, mut rx) = new_buffer(32); + for i in 0..32u8 { + tx.try_send(Bytes::from(vec![i])).unwrap(); + } + assert_eq!( + tx.try_send(Bytes::from_static(b"x")), + Err(SendError::BufferFull) + ); + assert_eq!(rx.recv().await.unwrap(), Bytes::from(vec![0u8])); + tx.try_send(Bytes::from_static(b"x")).unwrap(); + } + + #[tokio::test] + async fn recv_returns_none_after_the_sender_drops() { + let (tx, mut rx) = new_buffer(4); + tx.try_send(Bytes::from_static(b"a")).unwrap(); + drop(tx); + assert_eq!(rx.recv().await.unwrap(), Bytes::from_static(b"a")); + assert!(rx.recv().await.is_none()); + } + + #[tokio::test] + async fn send_after_receiver_drop_reports_closed() { + let (tx, rx) = new_buffer(4); + drop(rx); + assert_eq!( + tx.try_send(Bytes::from_static(b"a")), + Err(SendError::BufferClosed) + ); + } +} diff --git a/crates/openshell-supervisor/src/otlp_relay/enrichment.rs b/crates/openshell-supervisor/src/otlp_relay/enrichment.rs new file mode 100644 index 0000000000..03801e25b1 --- /dev/null +++ b/crates/openshell-supervisor/src/otlp_relay/enrichment.rs @@ -0,0 +1,1026 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Resource attribution for relayed agent traces. +//! +//! Every forwarded `ResourceSpans` entry carries exactly two supervisor-owned +//! attributes. Agent-supplied values for those keys are always discarded so +//! a workload cannot impersonate another sandbox or pose as infrastructure. +//! +//! Enrichment works at the protobuf wire level. Only each entry's `resource` +//! field is decoded and re-encoded, which drops unknown fields inside it; +//! scopes, spans, and every other field, known or unknown, are copied byte +//! for byte. The agent controls the body, so a full decode would let a 2 MiB +//! request of empty messages materialise hundreds of MiB of structs, and a +//! request of a million empty `ResourceSpans` entries would grow fifty-fold +//! once each gained the two attributes. [`MAX_RESOURCE_BYTES`] and +//! [`MAX_RESOURCE_ATTRIBUTES`] bound the decoded working set per request and +//! [`MAX_RESOURCE_SPANS`] bounds the growth. +//! +//! Memory: the output is the only allocation proportional to the request, +//! sized once for the worst-case growth and shrunk to its length before it +//! is buffered. Each entry is walked twice, once to merge its resource and +//! size the output, once to copy the remaining fields straight into it. + +use bytes::{Buf, Bytes}; +use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value}; +use opentelemetry_proto::tonic::resource::v1::Resource; +use prost::encoding::{ + WireType, decode_key, decode_varint, encode_key, encode_varint, encoded_len_varint, key_len, +}; +use prost::{DecodeError, Message}; + +/// Resource attribute carrying the sandbox identifier. Matches the value the +/// supervisor attaches to its own spans. +pub const SANDBOX_ID_KEY: &str = "openshell.sandbox.id"; +/// Resource attribute marking the batch as agent-originated. +pub const SOURCE_KEY: &str = "openshell.telemetry.source"; +/// Value of [`SOURCE_KEY`] on every relayed batch. +pub const SOURCE_VALUE: &str = "agent"; + +/// Maximum `ResourceSpans` entries per request. +/// +/// An SDK exports one entry per resource; a collector sidecar or a +/// multi-process agent can produce one per distinct resource. The cap bounds +/// how much the two added attributes can grow a request. For batches with +/// many resources [`MAX_RESOURCE_ATTRIBUTES`] binds first: at this many +/// entries each resource may carry about eight attributes. +pub const MAX_RESOURCE_SPANS: usize = 512; +/// Upper bound on the bytes attribution adds to one entry. +/// +/// The two string attributes encode to 67 bytes plus the sandbox id, an +/// entry without a resource gains a `resource` field header, and the length +/// prefixes may widen by a byte each: 160 covers sandbox ids of up to 80 +/// bytes. Ids are UUID strings today. `growth_bound_holds_at_the_maximum_id` +/// pins it. +pub const ATTRIBUTION_BYTES_PER_ENTRY: usize = 160; +/// Longest sandbox id [`ATTRIBUTION_BYTES_PER_ENTRY`] accounts for. +pub const MAX_SANDBOX_ID_BYTES: usize = 80; +/// Maximum total encoded size of the `resource` fields in one request. +/// +/// These are the only bytes that are decoded. Real resources are a few KiB +/// of attributes, so this leaves room for hundreds of distinct resources. +pub const MAX_RESOURCE_BYTES: usize = 256 * 1024; +/// Maximum decoded resource elements in one request, across all entries. +/// +/// Counts every `KeyValue`, `AnyValue`, and `EntityRef` the resource decode +/// would materialise, including values nested in `kvlist_value` and +/// `array_value` lists. An empty element is two wire bytes but 32 to 96 +/// bytes decoded, so the byte budget alone would admit 130 thousand of them +/// (8 to 12 MiB). Counting them on the wire before decoding keeps the +/// decoded set under about 1 MiB, including the doubling when the two +/// attribution attributes are pushed. +pub const MAX_RESOURCE_ATTRIBUTES: usize = 8192; +/// Nesting depth of attribute values beyond which a resource is rejected as +/// malformed. prost's own limit is 100; real attributes nest once or twice. +pub const MAX_VALUE_DEPTH: usize = 32; + +/// `ExportTraceServiceRequest.resource_spans`. +const RESOURCE_SPANS_TAG: u32 = 1; +/// `ResourceSpans.resource`. +const RESOURCE_TAG: u32 = 1; +/// `Resource.attributes`. +const ATTRIBUTES_TAG: u32 = 1; +/// `Resource.entity_refs`. +const ENTITY_REFS_TAG: u32 = 3; +/// `KeyValue.value`. +const KEY_VALUE_VALUE_TAG: u32 = 2; +/// `AnyValue.array_value`. +const ANY_VALUE_ARRAY_TAG: u32 = 5; +/// `AnyValue.kvlist_value`. +const ANY_VALUE_KVLIST_TAG: u32 = 6; +/// `ArrayValue.values` and `KeyValueList.values`. +const LIST_VALUES_TAG: u32 = 1; + +const BUFFER_UNDERFLOW: &str = "buffer underflow"; + +/// The request could not be enriched. +#[derive(Debug)] +pub enum EnrichError { + /// The bytes are not a well-formed protobuf message for this schema. + Malformed(String), + /// More than [`MAX_RESOURCE_SPANS`] entries. + TooManyResourceSpans, + /// More than [`MAX_RESOURCE_ATTRIBUTES`] resource attributes in total. + TooManyResourceAttributes, + /// The `resource` fields of the request exceed [`MAX_RESOURCE_BYTES`]. + ResourceTooLarge, +} + +impl std::fmt::Display for EnrichError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Malformed(error) => write!(f, "malformed ExportTraceServiceRequest: {error}"), + Self::TooManyResourceSpans => { + write!(f, "more than {MAX_RESOURCE_SPANS} resource_spans entries") + } + Self::TooManyResourceAttributes => { + write!(f, "more than {MAX_RESOURCE_ATTRIBUTES} resource attributes") + } + Self::ResourceTooLarge => { + write!( + f, + "resource fields exceed {MAX_RESOURCE_BYTES} bytes in total" + ) + } + } + } +} + +impl std::error::Error for EnrichError {} + +impl From for EnrichError { + fn from(error: DecodeError) -> Self { + Self::Malformed(error.to_string()) + } +} + +/// Attributes every resource of a protobuf `ExportTraceServiceRequest`. +/// +/// Replaces the two attribution attributes on each `ResourceSpans.resource` +/// and returns the re-encoded request. Everything other than the resources +/// is copied unchanged. +pub fn enrich(raw: &[u8], sandbox_id: &str) -> Result { + enrich_sized(raw, sandbox_id).map(Bytes::from) +} + +/// [`enrich`] as a `Vec` whose capacity equals its length, so the buffer +/// never holds growth slack. +fn enrich_sized(raw: &[u8], sandbox_id: &str) -> Result, EnrichError> { + let mut out = enrich_to_vec(raw, sandbox_id)?; + out.shrink_to_fit(); + Ok(out) +} + +/// [`enrich`] before the final shrink, sized once for the worst-case growth +/// so no entry triggers a doubling reallocation. +fn enrich_to_vec(raw: &[u8], sandbox_id: &str) -> Result, EnrichError> { + let mut out = Vec::with_capacity(raw.len() + MAX_RESOURCE_SPANS * ATTRIBUTION_BYTES_PER_ENTRY); + let mut entries = 0usize; + let mut budget = DecodeBudget { + resource_bytes: MAX_RESOURCE_BYTES, + elements: MAX_RESOURCE_ATTRIBUTES, + }; + for_each_field(raw, |tag, wire_type, value, field| { + if tag == RESOURCE_SPANS_TAG { + if wire_type != WireType::LengthDelimited { + return Err(EnrichError::Malformed( + "resource_spans is not length-delimited".into(), + )); + } + entries += 1; + if entries > MAX_RESOURCE_SPANS { + return Err(EnrichError::TooManyResourceSpans); + } + enrich_resource_spans(value, sandbox_id, &mut budget, &mut out) + } else { + out.extend_from_slice(field); + Ok(()) + } + })?; + Ok(out) +} + +/// What one request may still decode: `resource` bytes and the elements +/// inside them, both counted on the wire before any decoding happens. +struct DecodeBudget { + resource_bytes: usize, + elements: usize, +} + +impl DecodeBudget { + fn charge(&mut self, resource: &[u8]) -> Result<(), EnrichError> { + self.resource_bytes = self + .resource_bytes + .checked_sub(resource.len()) + .ok_or(EnrichError::ResourceTooLarge)?; + let mut elements = 0usize; + for_each_field(resource, |tag, wire_type, value, _| { + if wire_type != WireType::LengthDelimited { + return Ok(()); + } + match tag { + ATTRIBUTES_TAG => elements += 1 + key_value_elements(value, 0)?, + // One `EntityRef` plus each of its repeated id and + // description keys. + ENTITY_REFS_TAG => elements += 1 + length_delimited_fields(value)?, + _ => {} + } + Ok(()) + })?; + self.elements = self + .elements + .checked_sub(elements) + .ok_or(EnrichError::TooManyResourceAttributes)?; + Ok(()) + } +} + +/// Decoded elements nested inside one `KeyValue`: its value and whatever +/// that value lists. +fn key_value_elements(key_value: &[u8], depth: usize) -> Result { + let mut elements = 0usize; + for_each_field(key_value, |tag, wire_type, value, _| { + if tag == KEY_VALUE_VALUE_TAG && wire_type == WireType::LengthDelimited { + elements += any_value_elements(value, depth + 1)?; + } + Ok(()) + })?; + Ok(elements) +} + +/// Decoded elements for one `AnyValue`: itself plus any list it holds. +fn any_value_elements(any_value: &[u8], depth: usize) -> Result { + if depth > MAX_VALUE_DEPTH { + return Err(EnrichError::Malformed( + "attribute value nesting too deep".into(), + )); + } + let mut elements = 1usize; + for_each_field(any_value, |tag, wire_type, list, _| { + if wire_type != WireType::LengthDelimited { + return Ok(()); + } + match tag { + ANY_VALUE_ARRAY_TAG => for_each_field(list, |tag, wire_type, value, _| { + if tag == LIST_VALUES_TAG && wire_type == WireType::LengthDelimited { + elements += any_value_elements(value, depth + 1)?; + } + Ok(()) + })?, + ANY_VALUE_KVLIST_TAG => for_each_field(list, |tag, wire_type, value, _| { + if tag == LIST_VALUES_TAG && wire_type == WireType::LengthDelimited { + elements += 1 + key_value_elements(value, depth + 1)?; + } + Ok(()) + })?, + _ => {} + } + Ok(()) + })?; + Ok(elements) +} + +/// Number of length-delimited fields directly inside `message`. +fn length_delimited_fields(message: &[u8]) -> Result { + let mut fields = 0usize; + for_each_field(message, |_, wire_type, _, _| { + if wire_type == WireType::LengthDelimited { + fields += 1; + } + Ok(()) + })?; + Ok(fields) +} + +/// Appends one `ResourceSpans` entry to `out` with its resource attributed. +fn enrich_resource_spans( + entry: &[u8], + sandbox_id: &str, + budget: &mut DecodeBudget, + out: &mut Vec, +) -> Result<(), EnrichError> { + // Pass 1: merge the resource and measure everything else. + let mut resource = Resource::default(); + let mut rest_len = 0usize; + for_each_field(entry, |tag, wire_type, value, field| { + if tag == RESOURCE_TAG { + if wire_type != WireType::LengthDelimited { + return Err(EnrichError::Malformed( + "resource is not length-delimited".into(), + )); + } + budget.charge(value)?; + // Repeated occurrences of an embedded message merge, as in any + // protobuf parser. + resource.merge(value)?; + } else { + rest_len += field.len(); + } + Ok(()) + })?; + + resource + .attributes + .retain(|attribute| attribute.key != SANDBOX_ID_KEY && attribute.key != SOURCE_KEY); + resource + .attributes + .push(string_attribute(SANDBOX_ID_KEY, sandbox_id)); + resource + .attributes + .push(string_attribute(SOURCE_KEY, SOURCE_VALUE)); + + // Pass 2: write the entry header, the resource, then the rest verbatim. + let resource_len = resource.encoded_len(); + let entry_len = + key_len(RESOURCE_TAG) + encoded_len_varint(resource_len as u64) + resource_len + rest_len; + encode_key(RESOURCE_SPANS_TAG, WireType::LengthDelimited, out); + encode_varint(entry_len as u64, out); + encode_key(RESOURCE_TAG, WireType::LengthDelimited, out); + resource + .encode_length_delimited(out) + .expect("Vec grows on demand"); + for_each_field(entry, |tag, _, _, field| { + if tag != RESOURCE_TAG { + out.extend_from_slice(field); + } + Ok(()) + }) +} + +/// Walks the top-level fields of `message`, calling `visit(tag, wire_type, +/// value, field)` for each. `value` is the payload of a length-delimited +/// field and empty otherwise; `field` is the complete key-plus-value +/// encoding, for verbatim copying. +fn for_each_field<'a>( + message: &'a [u8], + mut visit: impl FnMut(u32, WireType, &'a [u8], &'a [u8]) -> Result<(), EnrichError>, +) -> Result<(), EnrichError> { + let mut cursor = message; + while cursor.has_remaining() { + let start = message.len() - cursor.remaining(); + let (tag, wire_type) = decode_key(&mut cursor)?; + let value: &'a [u8] = match wire_type { + WireType::Varint => { + decode_varint(&mut cursor)?; + &[] + } + WireType::SixtyFourBit => { + skip(&mut cursor, 8)?; + &[] + } + WireType::ThirtyTwoBit => { + skip(&mut cursor, 4)?; + &[] + } + WireType::LengthDelimited => { + let len = usize::try_from(decode_varint(&mut cursor)?) + .map_err(|_| EnrichError::Malformed(BUFFER_UNDERFLOW.into()))?; + if len > cursor.remaining() { + return Err(EnrichError::Malformed(BUFFER_UNDERFLOW.into())); + } + let value = &cursor[..len]; + cursor.advance(len); + value + } + WireType::StartGroup | WireType::EndGroup => { + return Err(EnrichError::Malformed("groups are not supported".into())); + } + }; + let end = message.len() - cursor.remaining(); + visit(tag, wire_type, value, &message[start..end])?; + } + Ok(()) +} + +fn skip(cursor: &mut &[u8], len: usize) -> Result<(), EnrichError> { + if cursor.remaining() < len { + return Err(EnrichError::Malformed(BUFFER_UNDERFLOW.into())); + } + cursor.advance(len); + Ok(()) +} + +fn string_attribute(key: &str, value: &str) -> KeyValue { + KeyValue { + key: key.to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(value.to_string())), + }), + key_strindex: 0, + } +} + +#[cfg(test)] +pub mod test_util { + use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; + use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value}; + use opentelemetry_proto::tonic::resource::v1::Resource; + use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span}; + use prost::Message; + + /// Builds a protobuf-encoded request with one span named `span_name` and + /// the given string resource attributes. + pub fn encoded_request(span_name: &str, attributes: &[(&str, &str)]) -> Vec { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: attributes + .iter() + .map(|(key, value)| KeyValue { + key: (*key).to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue((*value).to_string())), + }), + key_strindex: 0, + }) + .collect(), + ..Default::default() + }), + scope_spans: vec![ScopeSpans { + spans: vec![Span { + name: span_name.to_string(), + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }], + } + .encode_to_vec() + } + + /// Builds a protobuf-encoded request with one span carrying a span + /// attribute of `payload_bytes` bytes, so the body is large while the + /// resource stays small. + pub fn encoded_request_with_large_span(span_name: &str, payload_bytes: usize) -> Vec { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource::default()), + scope_spans: vec![ScopeSpans { + spans: vec![Span { + name: span_name.to_string(), + attributes: vec![KeyValue { + key: "payload".to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + "x".repeat(payload_bytes), + )), + }), + key_strindex: 0, + }], + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }], + } + .encode_to_vec() + } + + /// String-valued resource attributes of the first `ResourceSpans` entry. + pub fn resource_attributes(encoded: &[u8]) -> Vec<(String, String)> { + let decoded = ExportTraceServiceRequest::decode(encoded).expect("decodes"); + let resource = decoded.resource_spans[0] + .resource + .as_ref() + .expect("resource present"); + resource + .attributes + .iter() + .filter_map(|attribute| { + let value = attribute.value.as_ref()?.value.as_ref()?; + let any_value::Value::StringValue(value) = value else { + return None; + }; + Some((attribute.key.clone(), value.clone())) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; + use opentelemetry_proto::tonic::common::v1::InstrumentationScope; + use opentelemetry_proto::tonic::trace::v1::span::Event; + use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span, Status}; + use prost::Message; + + use super::test_util::{encoded_request, resource_attributes}; + use super::*; + + /// Wraps `payload` as one length-delimited field with `tag`. + fn field(tag: u32, payload: &[u8]) -> Vec { + let mut out = Vec::new(); + encode_key(tag, WireType::LengthDelimited, &mut out); + encode_varint(payload.len() as u64, &mut out); + out.extend_from_slice(payload); + out + } + + /// A `ResourceSpans` entry whose `resource` is `attributes` worth of + /// empty `KeyValue`s (`0a 00`), the densest decode per wire byte. + fn entry_with_empty_attributes(attributes: usize) -> Vec { + field( + RESOURCE_SPANS_TAG, + &field(RESOURCE_TAG, &[0x0a, 0x00].repeat(attributes)), + ) + } + + #[test] + fn adds_both_attributes_to_an_empty_resource() { + let raw = ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: None, + scope_spans: vec![ScopeSpans { + spans: vec![Span { + name: "s".into(), + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }], + } + .encode_to_vec(); + + let enriched = enrich(&raw, "sb-1").unwrap(); + let attributes = resource_attributes(&enriched); + assert_eq!( + attributes, + vec![ + (SANDBOX_ID_KEY.to_string(), "sb-1".to_string()), + (SOURCE_KEY.to_string(), SOURCE_VALUE.to_string()), + ] + ); + } + + #[test] + fn replaces_spoofed_values() { + let raw = encoded_request( + "s", + &[(SANDBOX_ID_KEY, "evil"), (SOURCE_KEY, "infrastructure")], + ); + let enriched = enrich(&raw, "sb-real").unwrap(); + let attributes = resource_attributes(&enriched); + assert_eq!( + attributes + .iter() + .filter(|(key, _)| key == SANDBOX_ID_KEY) + .count(), + 1 + ); + assert_eq!( + attributes + .iter() + .filter(|(key, _)| key == SOURCE_KEY) + .count(), + 1 + ); + assert!(attributes.contains(&(SANDBOX_ID_KEY.to_string(), "sb-real".to_string()))); + assert!(attributes.contains(&(SOURCE_KEY.to_string(), SOURCE_VALUE.to_string()))); + assert!(!attributes.iter().any(|(_, value)| value == "evil")); + assert!( + !attributes + .iter() + .any(|(_, value)| value == "infrastructure") + ); + } + + #[test] + fn replaces_spoofed_keys_regardless_of_value_type() { + let spoofed = Resource { + attributes: vec![KeyValue { + key: SANDBOX_ID_KEY.to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::IntValue(7)), + }), + key_strindex: 0, + }], + ..Default::default() + }; + let raw = field( + RESOURCE_SPANS_TAG, + &field(RESOURCE_TAG, &spoofed.encode_to_vec()), + ); + let enriched = enrich(&raw, "sb-real").unwrap(); + let decoded = ExportTraceServiceRequest::decode(enriched.as_ref()).unwrap(); + let keys: Vec<&str> = decoded.resource_spans[0] + .resource + .as_ref() + .unwrap() + .attributes + .iter() + .map(|attribute| attribute.key.as_str()) + .collect(); + assert_eq!(keys, vec![SANDBOX_ID_KEY, SOURCE_KEY]); + } + + #[test] + fn preserves_other_attributes_and_all_spans() { + let raw = encoded_request("keep-me", &[("service.name", "unit-agent")]); + let enriched = enrich(&raw, "sb-1").unwrap(); + let attributes = resource_attributes(&enriched); + assert!(attributes.contains(&("service.name".to_string(), "unit-agent".to_string()))); + assert_eq!(attributes.len(), 3); + + let decoded = ExportTraceServiceRequest::decode(enriched.as_ref()).unwrap(); + let spans: Vec<&Span> = decoded + .resource_spans + .iter() + .flat_map(|rs| rs.scope_spans.iter()) + .flat_map(|ss| ss.spans.iter()) + .collect(); + assert_eq!(spans.len(), 1); + assert_eq!(spans[0].name, "keep-me"); + } + + #[test] + fn copies_scopes_spans_and_schema_url_verbatim() { + let span = Span { + trace_id: vec![1; 16], + span_id: vec![2; 8], + name: "detailed".into(), + start_time_unix_nano: 10, + end_time_unix_nano: 20, + attributes: vec![string_attribute("http.method", "GET")], + events: vec![Event { + name: "retry".into(), + time_unix_nano: 15, + ..Default::default() + }], + status: Some(Status { + code: 2, + message: "boom".into(), + }), + ..Default::default() + }; + let original = ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![string_attribute("service.name", "a")], + dropped_attributes_count: 3, + ..Default::default() + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { + name: "lib".into(), + version: "1.2".into(), + ..Default::default() + }), + spans: vec![span], + schema_url: "https://schema/scope".into(), + }], + schema_url: "https://schema/resource".into(), + }], + }; + + let enriched = enrich(&original.encode_to_vec(), "sb-1").unwrap(); + let decoded = ExportTraceServiceRequest::decode(enriched.as_ref()).unwrap(); + let entry = &decoded.resource_spans[0]; + assert_eq!(entry.scope_spans, original.resource_spans[0].scope_spans); + assert_eq!(entry.schema_url, "https://schema/resource"); + let resource = entry.resource.as_ref().unwrap(); + assert_eq!(resource.dropped_attributes_count, 3); + assert_eq!(resource.attributes.len(), 3); + assert_eq!(resource.attributes[0].key, "service.name"); + } + + #[test] + fn copies_unknown_fields_of_every_wire_type_verbatim() { + // Unknown fields inside an entry, one per wire type, plus an unknown + // varint field at the request level. + let mut varint_field = Vec::new(); + encode_key(7, WireType::Varint, &mut varint_field); + encode_varint(300, &mut varint_field); + let mut fixed64_field = Vec::new(); + encode_key(8, WireType::SixtyFourBit, &mut fixed64_field); + fixed64_field.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); + let mut fixed32_field = Vec::new(); + encode_key(9, WireType::ThirtyTwoBit, &mut fixed32_field); + fixed32_field.extend_from_slice(&[9, 10, 11, 12]); + let delimited_field = field(10, b"opaque"); + let mut entry = Vec::new(); + entry.extend_from_slice(&varint_field); + entry.extend_from_slice(&field(RESOURCE_TAG, &Resource::default().encode_to_vec())); + entry.extend_from_slice(&fixed64_field); + entry.extend_from_slice(&fixed32_field); + entry.extend_from_slice(&delimited_field); + let mut request_varint = Vec::new(); + encode_key(2, WireType::Varint, &mut request_varint); + encode_varint(1, &mut request_varint); + let mut raw = field(RESOURCE_SPANS_TAG, &entry); + raw.extend_from_slice(&request_varint); + + let enriched = enrich(&raw, "sb-1").unwrap(); + assert!(ExportTraceServiceRequest::decode(enriched.as_ref()).is_ok()); + let contains = |needle: &[u8]| enriched.windows(needle.len()).any(|w| w == needle); + for unknown in [ + &varint_field, + &fixed64_field, + &fixed32_field, + &delimited_field, + &request_varint, + ] { + assert!(contains(unknown), "missing {unknown:?}"); + } + // The non-resource fields keep their relative order after the + // resource, which is written first. + let tail: Vec = [ + varint_field.as_slice(), + fixed64_field.as_slice(), + fixed32_field.as_slice(), + delimited_field.as_slice(), + request_varint.as_slice(), + ] + .concat(); + assert!(enriched.ends_with(&tail)); + } + + #[test] + fn merges_repeated_resource_fields_like_a_protobuf_parser() { + // Two `resource` occurrences in one entry: attributes concatenate. + let first = Resource { + attributes: vec![string_attribute("a", "1")], + ..Default::default() + }; + let second = Resource { + attributes: vec![string_attribute("b", "2")], + ..Default::default() + }; + let mut entry = field(RESOURCE_TAG, &first.encode_to_vec()); + entry.extend_from_slice(&field(RESOURCE_TAG, &second.encode_to_vec())); + let raw = field(RESOURCE_SPANS_TAG, &entry); + + let enriched = enrich(&raw, "sb-1").unwrap(); + let attributes = resource_attributes(&enriched); + assert_eq!( + attributes, + vec![ + ("a".to_string(), "1".to_string()), + ("b".to_string(), "2".to_string()), + (SANDBOX_ID_KEY.to_string(), "sb-1".to_string()), + (SOURCE_KEY.to_string(), SOURCE_VALUE.to_string()), + ] + ); + } + + #[test] + fn rejects_invalid_bytes() { + let malformed = |raw: &[u8]| matches!(enrich(raw, "sb-1"), Err(EnrichError::Malformed(_))); + assert!(malformed(&[0xff, 0xfe, 0xfd, 0xfc]), "invalid key"); + assert!(malformed(&[0x0a, 0x10, 0x00]), "length past the end"); + assert!(malformed(&[0x0b]), "group"); + assert!(malformed(&[0x41, 0x01]), "truncated fixed64"); + assert!(malformed(&[0x4d, 0x01]), "truncated fixed32"); + assert!( + malformed(&[0x08, 0x01]), + "resource_spans with a varint wire type" + ); + assert!( + malformed(&field(RESOURCE_SPANS_TAG, &[0x08, 0x01])), + "resource with a varint wire type" + ); + assert!( + malformed(&field( + RESOURCE_SPANS_TAG, + &field(RESOURCE_TAG, &[0xff, 0xff]) + )), + "garbage inside the resource" + ); + } + + #[test] + fn rejects_more_resource_spans_than_the_cap_before_decoding_them() { + // Entry MAX+1 carries a malformed resource: the cap trips first, so + // the error is the cap, not a decode failure. + let mut raw = [0x0a, 0x00].repeat(MAX_RESOURCE_SPANS); + raw.extend_from_slice(&field( + RESOURCE_SPANS_TAG, + &field(RESOURCE_TAG, &[0xff, 0xff]), + )); + assert!(matches!( + enrich(&raw, "sb-1"), + Err(EnrichError::TooManyResourceSpans) + )); + + let at_cap: Vec = [0x0a, 0x00].repeat(MAX_RESOURCE_SPANS); + let enriched = enrich(&at_cap, "sb-1").unwrap(); + let decoded = ExportTraceServiceRequest::decode(enriched.as_ref()).unwrap(); + assert_eq!(decoded.resource_spans.len(), MAX_RESOURCE_SPANS); + } + + #[test] + fn growth_bound_holds_at_the_maximum_id() { + // Resource-less entries gain the most: two attributes plus a new + // `resource` header plus prefix widening. + let at_cap: Vec = [0x0a, 0x00].repeat(MAX_RESOURCE_SPANS); + let id = "x".repeat(MAX_SANDBOX_ID_BYTES); + let enriched = enrich(&at_cap, &id).unwrap(); + assert!( + enriched.len() <= at_cap.len() + MAX_RESOURCE_SPANS * ATTRIBUTION_BYTES_PER_ENTRY, + "grew by {} per entry, bound is {ATTRIBUTION_BYTES_PER_ENTRY}", + (enriched.len() - at_cap.len()) / MAX_RESOURCE_SPANS + ); + let single = enrich(&[0x0a, 0x00], &id).unwrap(); + assert!(single.len() - 2 <= ATTRIBUTION_BYTES_PER_ENTRY); + } + + #[test] + fn output_carries_no_spare_capacity_into_the_buffer() { + // 500 entries (not a power of two, so doubling could not land + // exactly on the bound) each gaining the attributes: well past the + // slack an unsized Vec would start with. + let entries = 500; + let raw: Vec = field( + RESOURCE_SPANS_TAG, + &field(2, &[0x12, 0x00].repeat(2000)), // a 4 KiB scope_spans + ) + .repeat(entries); + let out = enrich_to_vec(&raw, "sb-1").unwrap(); + // Any reallocation would have at least doubled the capacity, so the + // initial size must still be exact. + assert_eq!( + out.capacity(), + raw.len() + MAX_RESOURCE_SPANS * ATTRIBUTION_BYTES_PER_ENTRY, + "the output was reallocated" + ); + let sized = enrich_sized(&raw, "sb-1").unwrap(); + assert_eq!(sized.capacity(), sized.len()); + assert_eq!(sized.len(), out.len()); + } + + #[test] + fn rejects_nested_elements_beyond_the_cap_before_decoding_them() { + // One top-level attribute whose value is a list of empty entries: + // the top-level count is 1, but every entry decodes to a struct. The + // last entry is a KeyValue prost would reject (its key encoded as a + // varint), so the result proves the cap tripped before any decoding. + let mut entries = [0x0a, 0x00].repeat(MAX_RESOURCE_ATTRIBUTES); + entries.extend_from_slice(&field(LIST_VALUES_TAG, &[0x08, 0x01])); + let kvlist = field(ANY_VALUE_KVLIST_TAG, &entries); + let attribute = field(ATTRIBUTES_TAG, &field(KEY_VALUE_VALUE_TAG, &kvlist)); + let raw = field(RESOURCE_SPANS_TAG, &field(RESOURCE_TAG, &attribute)); + assert!(raw.len() < MAX_RESOURCE_BYTES); + assert!(matches!( + enrich(&raw, "sb-1"), + Err(EnrichError::TooManyResourceAttributes) + )); + + // Exact boundary: the attribute, its value, and the entries sum to + // the cap (accepted) or one over it (refused), so the nested count + // is pinned from both sides. + let nested_budget = |entries: usize| { + let kvlist = field(ANY_VALUE_KVLIST_TAG, &[0x0a, 0x00].repeat(entries)); + let attribute = field(ATTRIBUTES_TAG, &field(KEY_VALUE_VALUE_TAG, &kvlist)); + enrich( + &field(RESOURCE_SPANS_TAG, &field(RESOURCE_TAG, &attribute)), + "sb-1", + ) + }; + assert!(nested_budget(MAX_RESOURCE_ATTRIBUTES - 2).is_ok()); + assert!(matches!( + nested_budget(MAX_RESOURCE_ATTRIBUTES - 1), + Err(EnrichError::TooManyResourceAttributes) + )); + + // The same through an array of empty values. + let array = field( + ANY_VALUE_ARRAY_TAG, + &[0x0a, 0x00].repeat(MAX_RESOURCE_ATTRIBUTES + 1), + ); + let attribute = field(ATTRIBUTES_TAG, &field(KEY_VALUE_VALUE_TAG, &array)); + let raw = field(RESOURCE_SPANS_TAG, &field(RESOURCE_TAG, &attribute)); + assert!(matches!( + enrich(&raw, "sb-1"), + Err(EnrichError::TooManyResourceAttributes) + )); + + // Entity references are decoded too and count the same way. + let entity_refs: Vec = [0x1a, 0x00].repeat(MAX_RESOURCE_ATTRIBUTES + 1); + let raw = field(RESOURCE_SPANS_TAG, &field(RESOURCE_TAG, &entity_refs)); + assert!(matches!( + enrich(&raw, "sb-1"), + Err(EnrichError::TooManyResourceAttributes) + )); + + // A realistic nested attribute survives enrichment intact. + let nested = Resource { + attributes: vec![KeyValue { + key: "process.command_args".into(), + value: Some(AnyValue { + value: Some(any_value::Value::ArrayValue( + opentelemetry_proto::tonic::common::v1::ArrayValue { + values: ["python3", "-c", "main()"] + .into_iter() + .map(|arg| AnyValue { + value: Some(any_value::Value::StringValue(arg.into())), + }) + .collect(), + }, + )), + }), + key_strindex: 0, + }], + ..Default::default() + }; + let raw = field( + RESOURCE_SPANS_TAG, + &field(RESOURCE_TAG, &nested.encode_to_vec()), + ); + let enriched = enrich(&raw, "sb-1").unwrap(); + let decoded = ExportTraceServiceRequest::decode(enriched.as_ref()).unwrap(); + let resource = decoded.resource_spans[0].resource.as_ref().unwrap(); + assert_eq!(resource.attributes.len(), 3); + assert_eq!(resource.attributes[0], nested.attributes[0]); + } + + /// `levels` nested arrays, each holding one `AnyValue`: the innermost + /// value sits at depth `levels + 1` in the element walk. Arrays cost two + /// prost nesting levels per array level, so prost's own limit of 100 is + /// never the one that trips here. + fn array_nested(levels: usize) -> Vec { + let mut value = Vec::new(); // an empty AnyValue + for _ in 0..levels { + value = field(ANY_VALUE_ARRAY_TAG, &field(LIST_VALUES_TAG, &value)); + } + let attribute = field(ATTRIBUTES_TAG, &field(KEY_VALUE_VALUE_TAG, &value)); + field(RESOURCE_SPANS_TAG, &field(RESOURCE_TAG, &attribute)) + } + + #[test] + fn attribute_value_depth_is_bounded_exactly() { + assert!(enrich(&array_nested(MAX_VALUE_DEPTH - 1), "sb-1").is_ok()); + assert!(matches!( + enrich(&array_nested(MAX_VALUE_DEPTH), "sb-1"), + Err(EnrichError::Malformed(message)) if message.contains("nesting too deep") + )); + } + + #[test] + fn rejects_more_resource_attributes_than_the_cap_before_decoding_them() { + // Within the byte budget but far over the attribute count: each + // empty attribute would decode to a 64-byte struct. + let raw = entry_with_empty_attributes(MAX_RESOURCE_ATTRIBUTES + 1); + assert!(raw.len() < MAX_RESOURCE_BYTES); + assert!(matches!( + enrich(&raw, "sb-1"), + Err(EnrichError::TooManyResourceAttributes) + )); + // The count is cumulative across entries. + let half = entry_with_empty_attributes(MAX_RESOURCE_ATTRIBUTES / 2 + 1); + let two: Vec = half.repeat(2); + assert!(matches!( + enrich(&two, "sb-1"), + Err(EnrichError::TooManyResourceAttributes) + )); + let at_cap = entry_with_empty_attributes(MAX_RESOURCE_ATTRIBUTES); + assert!(enrich(&at_cap, "sb-1").is_ok()); + } + + /// A `ResourceSpans` entry whose `resource` is one string attribute with + /// a `payload`-byte value: byte-heavy, attribute-light. + fn entry_with_large_resource(payload: usize) -> Vec { + field(RESOURCE_SPANS_TAG, &large_resource(payload)) + } + + /// A `resource` field holding one string attribute of `payload` bytes. + fn large_resource(payload: usize) -> Vec { + let resource = Resource { + attributes: vec![string_attribute("blob", &"x".repeat(payload))], + ..Default::default() + }; + field(RESOURCE_TAG, &resource.encode_to_vec()) + } + + #[test] + fn resource_budget_is_cumulative_across_occurrences_and_entries() { + // One entry, one resource over the budget. + let over = entry_with_large_resource(MAX_RESOURCE_BYTES); + assert!(matches!( + enrich(&over, "sb-1"), + Err(EnrichError::ResourceTooLarge) + )); + + // One entry, two resource occurrences that only together exceed it. + let half = large_resource(MAX_RESOURCE_BYTES / 2); + let mut entry = half.clone(); + entry.extend_from_slice(&half); + assert!(matches!( + enrich(&field(RESOURCE_SPANS_TAG, &entry), "sb-1"), + Err(EnrichError::ResourceTooLarge) + )); + + // Many entries whose resources together exceed it. + let each = entry_with_large_resource(MAX_RESOURCE_BYTES / 4); + let raw: Vec = each.repeat(5); + assert!(matches!( + enrich(&raw, "sb-1"), + Err(EnrichError::ResourceTooLarge) + )); + + // Just under the budget it is accepted. + let under = entry_with_large_resource(MAX_RESOURCE_BYTES - 64); + assert!(enrich(&under, "sb-1").is_ok()); + } + + #[test] + fn a_body_of_a_million_empty_spans_is_copied_verbatim() { + // One entry holding ~1M empty spans plus one unknown field inside the + // scope. A decode-and-re-encode implementation would drop the unknown + // field; the wire-level copy keeps every byte. + let mut scope: Vec = [0x12, 0x00].repeat(1024 * 1024); // ScopeSpans.spans, empty + scope.extend_from_slice(&field(15, b"opaque")); + let entry = field(2, &scope); // ResourceSpans.scope_spans + let raw = field(RESOURCE_SPANS_TAG, &entry); + + let enriched = enrich(&raw, "sb-1").unwrap(); + assert!( + enriched.len() <= raw.len() + ATTRIBUTION_BYTES_PER_ENTRY, + "growth is two attributes" + ); + assert!(enriched.ends_with(&entry)); + } +} diff --git a/crates/openshell-supervisor/src/otlp_relay/exporter.rs b/crates/openshell-supervisor/src/otlp_relay/exporter.rs new file mode 100644 index 0000000000..41cfcb9e03 --- /dev/null +++ b/crates/openshell-supervisor/src/otlp_relay/exporter.rs @@ -0,0 +1,294 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OTLP/gRPC forwarder for enriched agent batches. +//! +//! The channel connects lazily so an unreachable collector never fails +//! startup, and tonic re-establishes it on later attempts. Batches are sent +//! as the already-encoded bytes the receiver produced: the agent-controlled +//! payload is never decoded into a span tree on the export path. + +use std::time::Duration; + +use bytes::{BufMut as _, Bytes}; +use http::uri::PathAndQuery; +use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceResponse; +use prost::Message; +use tonic::client::Grpc; +use tonic::codec::{Codec, DecodeBuf, Decoder, EncodeBuf, Encoder}; +use tonic::transport::{Channel, Endpoint}; +use tonic::{GrpcMethod, Status}; + +use super::EXPORT_TIMEOUT; + +/// TCP connect deadline, applied per resolved address by hyper, kept well +/// under [`EXPORT_TIMEOUT`] so the batch that triggers a failed connect +/// usually observes the connect error rather than a request timeout. DNS +/// resolution is bounded only by [`EXPORT_TIMEOUT`]. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(2); +/// HTTP/2 PING cadence while an export is in flight. Pings are never sent on +/// an idle connection: stock collectors (grpc-go) close connections that +/// ping without an open stream. +const HTTP2_PING_INTERVAL: Duration = Duration::from_secs(2); +/// Unanswered-PING deadline; together with the interval it closes a dead +/// connection inside [`EXPORT_TIMEOUT`] so the next export redials. +const HTTP2_PING_TIMEOUT: Duration = Duration::from_secs(2); +/// TCP keepalive idle time, probe interval, and probe count for idle +/// connections, where HTTP/2 pings are not allowed. Detects a vanished peer +/// in about 30 seconds on Linux. +const TCP_KEEPALIVE_IDLE: Duration = Duration::from_secs(15); +const TCP_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5); +const TCP_KEEPALIVE_RETRIES: u32 = 3; + +const SERVICE: &str = "opentelemetry.proto.collector.trace.v1.TraceService"; +const METHOD: &str = "Export"; +const PATH: &str = "/opentelemetry.proto.collector.trace.v1.TraceService/Export"; + +/// One export attempt failed. The batch is discarded by the caller. +#[derive(Debug)] +pub enum ExportError { + /// The collector answered with an error or the transport failed. + Status(Status), + /// No answer within [`EXPORT_TIMEOUT`]. + Timeout, +} + +impl std::fmt::Display for ExportError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Status(status) => write!(f, "{}: {}", status.code(), status.message()), + Self::Timeout => write!(f, "no response within {EXPORT_TIMEOUT:?}"), + } + } +} + +impl std::error::Error for ExportError {} + +/// Forwards encoded `ExportTraceServiceRequest` batches to one collector. +pub struct Exporter { + client: Grpc, + endpoint: String, +} + +impl Exporter { + /// Prepares a lazy channel to `endpoint`. Fails only when the endpoint is + /// not a valid URI; no I/O happens here. + pub fn new(endpoint: &str) -> Result { + // Liveness detection for a collector that accepted the TCP connection + // and then went silent or vanished without a FIN, so the lazy channel + // redials instead of timing out every batch on a dead connection for + // the sandbox lifetime. HTTP/2 pings run only while an export is in + // flight; TCP keepalive covers idle connections. Detection of a dead + // connection during an export is covered by the silent-collector + // test; idle detection runs on the TCP keepalive timers and has no + // unit test, because those intervals are real-time constants. + // + // Deliberately not built on `openshell_extension_core::transport`: + // that recipe connects eagerly, pings idle connections, and carries a + // first-party TLS trust policy, none of which fit a third-party + // collector that must not gate supervisor startup. + let channel = Endpoint::from_shared(endpoint.to_string())? + .connect_timeout(CONNECT_TIMEOUT) + .tcp_keepalive(Some(TCP_KEEPALIVE_IDLE)) + .tcp_keepalive_interval(Some(TCP_KEEPALIVE_INTERVAL)) + .tcp_keepalive_retries(Some(TCP_KEEPALIVE_RETRIES)) + .http2_keep_alive_interval(HTTP2_PING_INTERVAL) + .keep_alive_timeout(HTTP2_PING_TIMEOUT) + .connect_lazy(); + Ok(Self { + client: Grpc::new(channel), + endpoint: endpoint.to_string(), + }) + } + + /// The collector URI this exporter targets. + #[must_use] + pub fn endpoint(&self) -> &str { + &self.endpoint + } + + /// Sends one encoded batch, bounded by [`EXPORT_TIMEOUT`]. + pub async fn export(&self, batch: Bytes) -> Result<(), ExportError> { + let mut client = self.client.clone(); + let send = async move { + client.ready().await.map_err(|error| { + Status::unavailable(format!("collector channel not ready: {error}")) + })?; + let mut request = tonic::Request::new(batch); + request + .extensions_mut() + .insert(GrpcMethod::new(SERVICE, METHOD)); + client + .unary(request, PathAndQuery::from_static(PATH), PassthroughCodec) + .await + }; + match tokio::time::timeout(EXPORT_TIMEOUT, send).await { + Ok(Ok(_)) => Ok(()), + Ok(Err(status)) => Err(ExportError::Status(status)), + Err(_) => Err(ExportError::Timeout), + } + } +} + +/// gRPC codec that writes a pre-encoded request body as-is and decodes the +/// collector's `ExportTraceServiceResponse`. +struct PassthroughCodec; + +impl Codec for PassthroughCodec { + type Encode = Bytes; + type Decode = ExportTraceServiceResponse; + type Encoder = PassthroughEncoder; + type Decoder = ResponseDecoder; + + fn encoder(&mut self) -> Self::Encoder { + PassthroughEncoder + } + + fn decoder(&mut self) -> Self::Decoder { + ResponseDecoder + } +} + +struct PassthroughEncoder; + +impl Encoder for PassthroughEncoder { + type Item = Bytes; + type Error = Status; + + fn encode(&mut self, item: Bytes, dst: &mut EncodeBuf<'_>) -> Result<(), Status> { + dst.put(item); + Ok(()) + } +} + +struct ResponseDecoder; + +impl Decoder for ResponseDecoder { + type Item = ExportTraceServiceResponse; + type Error = Status; + + fn decode(&mut self, src: &mut DecodeBuf<'_>) -> Result, Status> { + ExportTraceServiceResponse::decode(src) + .map(Some) + .map_err(|error| Status::internal(format!("malformed export response: {error}"))) + } +} + +/// Collector doubles shared with the lifecycle tests in the parent module. +#[cfg(test)] +pub mod test_util { + /// A loopback listener that accepts TCP connections and never answers, + /// so an export's HTTP/2 PING goes unanswered. Deterministic on every + /// platform, unlike an unroutable address. + pub struct SilentCollector { + pub endpoint: String, + task: tokio::task::JoinHandle<()>, + } + + impl SilentCollector { + pub async fn start() -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + // Accepted streams are kept open, never read, so the peer's + // HTTP/2 handshake waits forever. + #[allow(clippy::collection_is_never_read)] + let mut held = Vec::new(); + loop { + if let Ok((stream, _)) = listener.accept().await { + held.push(stream); + } + } + }); + Self { endpoint, task } + } + } + + impl Drop for SilentCollector { + fn drop(&mut self) { + self.task.abort(); + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Instant; + + use openshell_otel_test_support::OtlpTestServer; + + use super::super::enrichment::test_util::encoded_request; + use super::test_util::SilentCollector; + use super::*; + + #[test] + fn new_rejects_an_invalid_uri() { + assert!(Exporter::new("not a uri").is_err()); + } + + #[tokio::test] + async fn export_reaches_the_collector_with_attributes_intact() { + let collector = OtlpTestServer::start().await; + let exporter = Exporter::new(collector.endpoint()).unwrap(); + let batch = Bytes::from(encoded_request( + "exported-span", + &[("service.name", "unit-agent"), ("custom", "kept")], + )); + + exporter.export(batch).await.expect("export succeeds"); + collector.wait_for_export().await; + let received = collector.shutdown().await; + + assert!( + received + .spans + .iter() + .any(|span| span.name == "exported-span") + ); + assert_eq!(received.resource_attributes.len(), 1); + let attributes = &received.resource_attributes[0]; + assert_eq!( + attributes.get("service.name").map(String::as_str), + Some("unit-agent") + ); + assert_eq!(attributes.get("custom").map(String::as_str), Some("kept")); + } + + #[tokio::test] + async fn export_to_a_closed_port_returns_status_promptly() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + let exporter = Exporter::new(&endpoint).unwrap(); + + let started = Instant::now(); + let error = exporter + .export(Bytes::from(encoded_request("s", &[]))) + .await + .expect_err("closed port fails"); + assert!(matches!(error, ExportError::Status(_)), "{error}"); + assert!(started.elapsed() < EXPORT_TIMEOUT); + } + + #[tokio::test] + async fn export_to_a_silent_collector_fails_when_its_ping_goes_unanswered() { + // The collector accepts TCP and never speaks. With an export in + // flight, hyper pings after HTTP2_PING_INTERVAL and closes the + // connection after HTTP2_PING_TIMEOUT, so the failure is a transport + // error inside the export deadline rather than a 5 s timeout, and + // the lazy channel is left closed for the next export to redial. + let collector = SilentCollector::start().await; + let exporter = Exporter::new(&collector.endpoint).unwrap(); + let started = Instant::now(); + let error = exporter + .export(Bytes::from(encoded_request("s", &[]))) + .await + .expect_err("silent collector fails"); + assert!(matches!(error, ExportError::Status(_)), "{error}"); + let elapsed = started.elapsed(); + assert!( + elapsed >= HTTP2_PING_INTERVAL && elapsed < EXPORT_TIMEOUT, + "elapsed {elapsed:?}" + ); + } +} diff --git a/crates/openshell-supervisor/src/otlp_relay/mod.rs b/crates/openshell-supervisor/src/otlp_relay/mod.rs new file mode 100644 index 0000000000..95134d52f1 --- /dev/null +++ b/crates/openshell-supervisor/src/otlp_relay/mod.rs @@ -0,0 +1,1088 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor-direct OTLP trace relay. +//! +//! A sandboxed agent exports OTLP/HTTP protobuf traces to the reserved +//! address [`openshell_core::sandbox_env::OTLP_RELAY_ADDR`]. The sandbox +//! broker stages that connection to the supervisor, the network proxy hands +//! the stream to [`receiver::OtlpConnectionServer`], and the accepted batch +//! is enriched with the sandbox id, buffered, and forwarded over OTLP/gRPC to +//! the collector the supervisor already uses for its own spans. + +pub mod buffer; +pub mod enrichment; +pub mod exporter; +pub mod receiver; + +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use openshell_isolation_interface::contract::BoundaryDuplexStream; +use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; +use openshell_supervisor_network::proxy::{ + ReservedDestination, ReservedStreamFuture, ReservedStreamHandler, +}; +use tokio::sync::watch; +use tokio::task::JoinHandle; +use tracing::{info, warn}; + +use self::buffer::new_buffer; +use self::exporter::Exporter; +use self::receiver::{OtlpConnectionServer, ReceiverHandle}; + +/// Maximum accepted request body, in wire bytes. +pub const MAX_BODY_BYTES: usize = 2 * 1024 * 1024; +/// Maximum size of a batch after attribution, derived from the per-entry +/// growth bound and the entry cap. +/// +/// Relay memory attributable to the agent is bounded by three parts: the +/// export buffer, [`BUFFER_CAPACITY`] batches of this size (about 67 MiB); +/// bodies read but not yet enriched, [`MAX_CONCURRENT_CONNECTIONS`] times +/// [`MAX_BODY_BYTES`] (32 MiB); and the working set of the +/// [`ENRICHMENT_CONCURRENCY`] enrichments in progress, each about twice the +/// body plus at most about 1 MiB of decoded resource elements (about +/// 12 MiB). Every allocation is sized to its content before it is held. +pub const MAX_ENRICHED_BYTES: usize = + MAX_BODY_BYTES + enrichment::MAX_RESOURCE_SPANS * enrichment::ATTRIBUTION_BYTES_PER_ENTRY; +/// How long a connection may idle before a complete request header arrives. +pub const HEADER_READ_TIMEOUT: Duration = Duration::from_secs(10); +/// How long a request may take to deliver its body once the headers are in, +/// so a stalled sender cannot hold a connection slot indefinitely. +pub const BODY_READ_TIMEOUT: Duration = Duration::from_secs(30); +/// Maximum number of receiver connections served at once. +pub const MAX_CONCURRENT_CONNECTIONS: usize = 16; +/// Requests enriched at the same time. +/// +/// Enrichment is a short CPU-bound copy whose working set is about twice the +/// body plus the decoded resources, so two slots bound that transient +/// regardless of how many connections hold a body. +pub const ENRICHMENT_CONCURRENCY: usize = 2; +/// Number of enriched batches held between receipt and export. +pub const BUFFER_CAPACITY: usize = 32; +/// Deadline for one export attempt to the collector. +pub const EXPORT_TIMEOUT: Duration = Duration::from_secs(5); +/// Bound on the graceful shutdown of in-flight receiver connections. +pub const RECEIVER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2); +/// Overall bound on flushing buffered batches before the exit report. +pub const FLUSH_TIMEOUT: Duration = Duration::from_secs(5); +/// Interval of the cumulative-count summary log line. +pub const SUMMARY_INTERVAL: Duration = Duration::from_secs(30); +/// `Retry-After` value sent with a 503 back-pressure response. +pub const RETRY_AFTER_SECS: u64 = 1; + +/// Point-in-time copy of the relay counters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CounterSnapshot { + /// Batches delivered to the collector. + pub exported: u64, + /// Export requests refused with 503 because the buffer was full. + pub rejected: u64, + /// Export attempts that failed, timed out, or were discarded at exit. + pub export_failures: u64, +} + +/// Cumulative relay counters shared by the receiver, exporter, and summary +/// task. Counts are monotonic for the supervisor's lifetime. +#[derive(Debug, Default)] +pub struct RelayCounters { + accepted: AtomicU64, + exported: AtomicU64, + rejected: AtomicU64, + export_failures: AtomicU64, + first_rejection_logged: AtomicBool, + first_failure_logged: AtomicBool, + final_summary_logged: AtomicBool, +} + +impl RelayCounters { + /// Records one batch accepted into the buffer. + pub fn record_accepted(&self) { + self.accepted.fetch_add(1, Ordering::Relaxed); + } + + /// Records one export request refused with 503 because the buffer was + /// full. Warns on the first occurrence only. + pub fn record_rejection(&self) { + self.rejected.fetch_add(1, Ordering::Relaxed); + if !self.first_rejection_logged.swap(true, Ordering::Relaxed) { + warn!( + buffer_capacity = BUFFER_CAPACITY, + "OTLP relay rejected an agent export: buffer full" + ); + } + } + + /// Records one export attempt that failed or timed out. Warns on the + /// first occurrence only. + pub fn record_export_failure(&self, endpoint: &str, error: &dyn std::fmt::Display) { + self.export_failures.fetch_add(1, Ordering::Relaxed); + if !self.first_failure_logged.swap(true, Ordering::Relaxed) { + warn!(endpoint, %error, "OTLP relay export to collector failed"); + } + } + + /// Records one batch delivered to the collector. + pub fn record_export_success(&self) { + self.exported.fetch_add(1, Ordering::Relaxed); + } + + /// Current counts. + pub fn snapshot(&self) -> CounterSnapshot { + CounterSnapshot { + exported: self.exported.load(Ordering::Relaxed), + rejected: self.rejected.load(Ordering::Relaxed), + export_failures: self.export_failures.load(Ordering::Relaxed), + } + } + + /// Accepted batches not yet exported or failed: buffered plus in flight. + fn pending(&self) -> u64 { + let CounterSnapshot { + exported, + export_failures, + .. + } = self.snapshot(); + self.accepted + .load(Ordering::Relaxed) + .saturating_sub(exported) + .saturating_sub(export_failures) + } + + fn log_summary(&self) { + let CounterSnapshot { + exported, + rejected, + export_failures, + } = self.snapshot(); + info!(exported, rejected, export_failures, "OTLP relay summary"); + } + + /// Logs the final summary once, and only when something was rejected or + /// failed (FR-009). Called from `shutdown` and, for lifecycle paths that + /// drop the relay without shutting it down, from `Drop`. + fn log_final_summary(&self) { + let CounterSnapshot { + rejected, + export_failures, + .. + } = self.snapshot(); + if (rejected > 0 || export_failures > 0) + && !self.final_summary_logged.swap(true, Ordering::Relaxed) + { + self.log_summary(); + } + } +} + +/// Input to [`start`]. +#[derive(Debug, Clone)] +pub struct OtlpRelayConfig { + /// OTLP/gRPC collector URI (`OPENSHELL_OTLP_ENDPOINT`). + pub endpoint: String, + /// Sandbox identifier attached to every forwarded resource. + pub sandbox_id: String, +} + +/// The relay could not be started; it stays inactive. +#[derive(Debug)] +pub enum StartError { + /// The collector endpoint is not a URI tonic accepts. + InvalidEndpoint(tonic::transport::Error), + /// The collector endpoint uses a scheme the relay does not speak. TLS to + /// the collector is a tracked follow-up; the supervisor's own exporter + /// rejects `https` the same way. + UnsupportedScheme(String), +} + +impl std::fmt::Display for StartError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidEndpoint(error) => write!(f, "invalid collector endpoint: {error}"), + Self::UnsupportedScheme(scheme) => write!( + f, + "unsupported collector endpoint scheme {scheme:?}: the relay speaks plaintext OTLP/gRPC (http) only" + ), + } + } +} + +impl std::error::Error for StartError {} + +/// A running relay. Owned by the supervisor lifecycle; dropping it aborts +/// the background tasks and logs the final summary, [`OtlpRelay::shutdown`] +/// flushes first. +pub struct OtlpRelay { + server: Arc, + receiver: Option, + export_task: JoinHandle<()>, + summary_task: JoinHandle<()>, + counters: Arc, +} + +impl Drop for OtlpRelay { + fn drop(&mut self) { + self.export_task.abort(); + self.summary_task.abort(); + self.counters.log_final_summary(); + } +} + +/// Serves staged workload connections to the relay address with the OTLP +/// connection server. +struct RelayHandler(Arc); + +impl ReservedStreamHandler for RelayHandler { + fn serve(&self, stream: BoundaryDuplexStream) -> Option { + // Reserve before the proxy answers `RelayReady`, so an exhausted or + // stopped server refuses the open instead of resetting it later. + let permit = self.0.try_reserve()?; + let server = Arc::clone(&self.0); + Some(Box::pin(async move { server.serve(permit, stream).await })) + } +} + +fn relay_addr() -> SocketAddr { + openshell_core::sandbox_env::OTLP_RELAY_ADDR + .parse() + .expect("OTLP_RELAY_ADDR is a valid socket address") +} + +/// Starts the relay for `config`. Must be called inside a tokio runtime. +/// +/// Returns the relay handle and the reserved destination to install in the +/// network proxy. Fails only when the endpoint is not a valid plaintext URI; +/// an unreachable collector is handled lazily by the exporter. +pub fn start(config: OtlpRelayConfig) -> Result<(OtlpRelay, ReservedDestination), StartError> { + if let Ok(uri) = config.endpoint.parse::() + && let Some(scheme) = uri.scheme_str() + && !scheme.eq_ignore_ascii_case("http") + { + return Err(StartError::UnsupportedScheme(scheme.to_string())); + } + let exporter = Exporter::new(&config.endpoint).map_err(StartError::InvalidEndpoint)?; + if config.sandbox_id.len() > enrichment::MAX_SANDBOX_ID_BYTES { + // The attribution growth bound assumes a shorter id; batches near + // the wire limit with many entries will be refused with 413. + warn!( + sandbox_id_bytes = config.sandbox_id.len(), + max = enrichment::MAX_SANDBOX_ID_BYTES, + "OTLP relay: sandbox id exceeds the attribution budget; large multi-resource batches will be refused" + ); + } + let (relay, reserved) = assemble(exporter, config.sandbox_id, None); + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enabled") + .unmapped("endpoint", config.endpoint) + .message("OTLP agent trace relay enabled") + .build() + ); + Ok((relay, reserved)) +} + +/// Decides whether the relay runs for this sandbox and records the outcome +/// as a configuration state change. +/// +/// `None` endpoint: nothing is started and the proxy refuses the relay +/// address. `None` sandbox id: the same, because attribution is the point of +/// the relay and the supervisor's own spans carry no sandbox id either. An +/// endpoint that does not parse or uses an unsupported scheme leaves the +/// relay inactive the same way. +pub fn activate( + endpoint: Option, + sandbox_id: Option, +) -> (Option, Option) { + let Some(endpoint) = endpoint else { + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Disabled, "disabled") + .message("OTLP agent trace relay inactive: no collector endpoint configured") + .build() + ); + return (None, None); + }; + let Some(sandbox_id) = sandbox_id else { + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Low) + .status(StatusId::Success) + .state(StateId::Disabled, "disabled") + .unmapped("endpoint", endpoint) + .message("OTLP agent trace relay inactive: no sandbox id to attribute traces to") + .build() + ); + return (None, None); + }; + match start(OtlpRelayConfig { + endpoint: endpoint.clone(), + sandbox_id, + }) { + Ok((relay, reserved)) => (Some(relay), Some(reserved)), + Err(error) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Disabled, "disabled") + .unmapped("endpoint", endpoint) + .unmapped("error", error.to_string()) + .message("OTLP agent trace relay inactive: collector endpoint rejected") + .build() + ); + (None, None) + } + } +} + +/// Builds the relay around a prepared exporter. `export_gate`, when given, +/// holds the export task until it reads `true`; tests use it to keep batches +/// buffered. +fn assemble( + exporter: Exporter, + sandbox_id: String, + export_gate: Option>, +) -> (OtlpRelay, ReservedDestination) { + let counters = Arc::new(RelayCounters::default()); + let (buffer_tx, mut buffer_rx) = new_buffer(BUFFER_CAPACITY); + let (server, receiver) = + OtlpConnectionServer::new(buffer_tx, sandbox_id, Arc::clone(&counters)); + + let export_task = tokio::spawn({ + let counters = Arc::clone(&counters); + async move { + if let Some(mut gate) = export_gate { + let _ = gate.wait_for(|open| *open).await; + } + while let Some(batch) = buffer_rx.recv().await { + match exporter.export(batch).await { + Ok(()) => counters.record_export_success(), + Err(error) => counters.record_export_failure(exporter.endpoint(), &error), + } + } + } + }); + let summary_task = tokio::spawn(summary_loop(Arc::clone(&counters))); + + let reserved = ReservedDestination { + addr: relay_addr(), + handler: Arc::new(RelayHandler(Arc::clone(&server))), + }; + let relay = OtlpRelay { + server, + receiver: Some(receiver), + export_task, + summary_task, + counters, + }; + (relay, reserved) +} + +/// Logs the cumulative counts every [`SUMMARY_INTERVAL`], but only while the +/// rejected or failed counts keep changing. +async fn summary_loop(counters: Arc) { + let mut interval = tokio::time::interval_at( + tokio::time::Instant::now() + SUMMARY_INTERVAL, + SUMMARY_INTERVAL, + ); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut last_reported = (0, 0); + loop { + interval.tick().await; + let CounterSnapshot { + rejected, + export_failures, + .. + } = counters.snapshot(); + if (rejected, export_failures) != last_reported { + last_reported = (rejected, export_failures); + counters.log_summary(); + } + } +} + +fn elapsed_ms(since: Instant) -> u64 { + u64::try_from(since.elapsed().as_millis()).unwrap_or(u64::MAX) +} + +impl OtlpRelay { + /// Waits up to `bound` for every buffered batch to reach the collector + /// while the relay keeps serving. Used before the exit report when the + /// supervisor stays up for exec sessions afterwards, so their traces + /// are still relayed; [`Self::shutdown`] runs at final teardown. + pub async fn flush(&self, bound: Duration) { + let started = Instant::now(); + let exported_before = self.counters.snapshot().exported; + while self.counters.pending() > 0 && started.elapsed() < bound { + tokio::time::sleep(Duration::from_millis(20)).await; + } + let pending = self.counters.pending(); + if pending == 0 { + info!( + flushed = self.counters.snapshot().exported - exported_before, + elapsed_ms = elapsed_ms(started), + "OTLP relay flushed before exit" + ); + } else { + warn!( + pending, + "OTLP relay flush deadline exceeded; batches stay buffered while the access plane is retained" + ); + } + } + + /// Stops accepting agent connections, flushes buffered batches to the + /// collector for at most `flush_bound`, and logs the final summary. + /// Never fails; batches still pending at the deadline are discarded and + /// counted as export failures. + pub async fn shutdown(mut self, flush_bound: Duration) { + let started = Instant::now(); + let exported_before = self.counters.snapshot().exported; + + self.server.stop_accepting(); + if let Some(receiver) = self.receiver.take() { + receiver.shutdown().await; + } + self.server.close_buffer(); + + let remaining = flush_bound.saturating_sub(started.elapsed()); + match tokio::time::timeout(remaining, &mut self.export_task).await { + Ok(Ok(())) => { + info!( + flushed = self.counters.snapshot().exported - exported_before, + elapsed_ms = elapsed_ms(started), + "OTLP relay flushed before exit" + ); + } + Ok(Err(join_error)) => { + let discarded = self.counters.pending(); + self.counters + .export_failures + .fetch_add(discarded, Ordering::Relaxed); + warn!( + %join_error, + discarded, + "OTLP relay export task ended abnormally; buffered batches were not flushed" + ); + } + Err(_) => { + // Let the cancellation land before reading the counters, so + // an export that completed in the meantime is not counted as + // discarded. + self.export_task.abort(); + let _ = (&mut self.export_task).await; + let discarded = self.counters.pending(); + self.counters + .export_failures + .fetch_add(discarded, Ordering::Relaxed); + warn!( + discarded, + "OTLP relay flush deadline exceeded; discarding buffered batches" + ); + } + } + self.summary_task.abort(); + self.counters.log_final_summary(); + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Mutex, MutexGuard}; + use std::time::Duration; + + use openshell_otel_test_support::OtlpTestServer; + use tokio::io::AsyncWriteExt as _; + use tracing_subscriber::layer::SubscriberExt as _; + + use super::enrichment::test_util::{encoded_request, encoded_request_with_large_span}; + use super::enrichment::{SANDBOX_ID_KEY, SOURCE_KEY, SOURCE_VALUE}; + use super::exporter::test_util::SilentCollector; + use super::receiver::test_util::{connect, read_head, request, send}; + use super::*; + + const PROTOBUF: &str = "application/x-protobuf"; + + /// Serialises the tests that install a log capture, so concurrent + /// dispatcher registration on other test threads cannot race the + /// callsite interest of the events under test. + static LOG_CAPTURE_LOCK: Mutex<()> = Mutex::new(()); + + /// One captured event: level, message, and its integer fields. + #[derive(Clone, Debug)] + struct CapturedEvent { + level: tracing::Level, + message: String, + fields: Vec<(String, u64)>, + } + + /// Captures events for the duration of a test. Works for events emitted + /// on the test thread, which on a current-thread runtime includes every + /// spawned task. + #[derive(Clone, Default)] + struct LogCapture(Arc>>); + + /// Keeps the capture installed and the serialisation lock held. + struct CaptureGuard { + _lock: MutexGuard<'static, ()>, + _default: tracing::subscriber::DefaultGuard, + } + + impl LogCapture { + fn install(&self) -> CaptureGuard { + let lock = LOG_CAPTURE_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let default = + tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone())); + CaptureGuard { + _lock: lock, + _default: default, + } + } + + fn count(&self, level: tracing::Level, message: &str) -> usize { + self.events(level, message).len() + } + + fn events(&self, level: tracing::Level, message: &str) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .filter(|event| event.level == level && event.message == message) + .cloned() + .collect() + } + + /// Events at INFO or above, regardless of message. + fn info_and_above(&self) -> usize { + self.0 + .lock() + .unwrap() + .iter() + .filter(|event| event.level <= tracing::Level::INFO) + .count() + } + } + + impl CapturedEvent { + fn field(&self, name: &str) -> Option { + self.fields + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| *value) + } + } + + #[derive(Default)] + struct EventVisitor { + message: Option, + fields: Vec<(String, u64)>, + } + + impl tracing::field::Visit for EventVisitor { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + if field.name() == "message" { + self.message = Some(format!("{value:?}")); + } + } + + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + self.fields.push((field.name().to_string(), value)); + } + + fn record_i64(&mut self, field: &tracing::field::Field, value: i64) { + if let Ok(value) = u64::try_from(value) { + self.fields.push((field.name().to_string(), value)); + } + } + } + + impl tracing_subscriber::Layer for LogCapture { + fn on_event( + &self, + event: &tracing::Event<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut visitor = EventVisitor::default(); + event.record(&mut visitor); + if let Some(message) = visitor.message { + self.0.lock().unwrap().push(CapturedEvent { + level: *event.metadata().level(), + message, + fields: visitor.fields, + }); + } + } + } + + fn gated(endpoint: &str) -> (OtlpRelay, ReservedDestination, watch::Sender) { + let (gate_tx, gate_rx) = watch::channel(false); + let exporter = Exporter::new(endpoint).unwrap(); + let (relay, reserved) = assemble(exporter, "sb-test".into(), Some(gate_rx)); + (relay, reserved, gate_tx) + } + + async fn post(relay: &OtlpRelay, span: &str) -> u16 { + let mut client = connect(&relay.server); + let body = encoded_request(span, &[("service.name", "unit-agent")]); + send(&mut client, &request("POST", "/v1/traces", PROTOBUF, &body)) + .await + .status + } + + async fn wait_until(deadline: Duration, mut condition: impl FnMut() -> bool) { + let started = Instant::now(); + while !condition() { + assert!(started.elapsed() < deadline, "condition not met in time"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + #[tokio::test] + async fn http_post_over_duplex_reaches_collector() { + let collector = OtlpTestServer::start().await; + let (relay, reserved) = start(OtlpRelayConfig { + endpoint: collector.endpoint().to_string(), + sandbox_id: "sb-test".into(), + }) + .unwrap(); + assert_eq!( + reserved.addr.to_string(), + openshell_core::sandbox_env::OTLP_RELAY_ADDR + ); + + let (mut client, boundary) = tokio::io::duplex(64 * 1024); + let served = reserved + .handler + .serve(Box::new(boundary)) + .expect("a fresh relay accepts a stream"); + tokio::spawn(served); + + let body = encoded_request( + "unit-span", + &[ + ("service.name", "unit-agent"), + (SANDBOX_ID_KEY, "evil"), + (SOURCE_KEY, "infrastructure"), + ], + ); + client + .write_all(&request("POST", "/v1/traces", PROTOBUF, &body)) + .await + .unwrap(); + assert_eq!(read_head(&mut client).await.status, 200); + + collector.wait_for_export().await; + let received = collector.shutdown().await; + assert!(received.spans.iter().any(|span| span.name == "unit-span")); + // The collector keeps the last value per key, so a retained spoofed + // attribute would only show up as a duplicate: assert both. + assert!( + received.duplicate_resource_keys.is_empty(), + "spoofed attribution keys were appended to, not replaced: {:?}", + received.duplicate_resource_keys + ); + let attributes = &received.resource_attributes[0]; + assert_eq!(attributes.len(), 3); + assert_eq!( + attributes.get("service.name").map(String::as_str), + Some("unit-agent") + ); + assert_eq!( + attributes.get(SOURCE_KEY).map(String::as_str), + Some(SOURCE_VALUE) + ); + assert_eq!( + attributes.get(SANDBOX_ID_KEY).map(String::as_str), + Some("sb-test") + ); + assert!( + !attributes + .values() + .any(|value| value == "evil" || value == "infrastructure") + ); + drop(relay); + } + + #[tokio::test] + async fn invalid_endpoint_leaves_relay_inactive() { + let before = tokio::runtime::Handle::current() + .metrics() + .num_alive_tasks(); + let error = start(OtlpRelayConfig { + endpoint: "not a uri".into(), + sandbox_id: "sb-test".into(), + }) + .err() + .expect("unparsable endpoint fails"); + assert!(matches!(error, StartError::InvalidEndpoint(_)), "{error}"); + let after = tokio::runtime::Handle::current() + .metrics() + .num_alive_tasks(); + assert_eq!(before, after, "nothing is spawned for an inactive relay"); + } + + #[tokio::test] + async fn activate_without_endpoint_starts_nothing() { + let before = tokio::runtime::Handle::current() + .metrics() + .num_alive_tasks(); + let (relay, reserved) = activate(None, Some("sb-test".into())); + assert!(relay.is_none()); + assert!(reserved.is_none()); + let after = tokio::runtime::Handle::current() + .metrics() + .num_alive_tasks(); + assert_eq!(before, after); + } + + #[tokio::test] + async fn activate_without_sandbox_id_starts_nothing() { + // Attribution is the point of the relay; without an id the supervisor's + // own spans carry none either, so the relay stays inactive instead of + // stamping every batch with an empty id. + let collector = OtlpTestServer::start().await; + let before = tokio::runtime::Handle::current() + .metrics() + .num_alive_tasks(); + let (relay, reserved) = activate(Some(collector.endpoint().to_string()), None); + assert!(relay.is_none()); + assert!(reserved.is_none()); + let after = tokio::runtime::Handle::current() + .metrics() + .num_alive_tasks(); + assert_eq!(before, after); + drop(collector.shutdown().await); + } + + #[test] + fn https_endpoint_is_refused_at_start() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let _enter = runtime.enter(); + let error = start(OtlpRelayConfig { + endpoint: "https://collector.example:4317".into(), + sandbox_id: "sb-test".into(), + }) + .err() + .expect("https is not supported on the relay hop"); + assert!( + matches!(error, StartError::UnsupportedScheme(ref scheme) if scheme == "https"), + "{error}" + ); + // Lower-case comparison, and plain http still starts. + assert!( + start(OtlpRelayConfig { + endpoint: "HTTP://collector.example:4317".into(), + sandbox_id: "sb-test".into(), + }) + .is_ok() + ); + } + + #[tokio::test] + async fn flush_waits_for_pending_batches_and_keeps_the_relay_serving() { + let collector = OtlpTestServer::start().await; + let (relay, _reserved, gate) = gated(collector.endpoint()); + for span in ["one", "two"] { + assert_eq!(post(&relay, span).await, 200); + } + assert_eq!(relay.counters.pending(), 2); + + gate.send_replace(true); + let started = Instant::now(); + relay.flush(FLUSH_TIMEOUT).await; + assert!(started.elapsed() < FLUSH_TIMEOUT); + assert_eq!(relay.counters.pending(), 0); + + // Unlike shutdown, the relay still serves exec sessions afterwards. + assert_eq!(post(&relay, "after-flush").await, 200); + collector.wait_for_export().await; + relay.shutdown(FLUSH_TIMEOUT).await; + let received = collector.shutdown().await; + assert!(received.spans.iter().any(|span| span.name == "after-flush")); + } + + #[tokio::test] + async fn dropping_the_relay_without_shutdown_logs_the_final_summary_once() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let collector = OtlpTestServer::start().await; + let (relay, _reserved, _gate) = gated(collector.endpoint()); + relay.counters.record_rejection(); + // An early-exit lifecycle path drops the relay without `shutdown`. + drop(relay); + assert_eq!(logs.count(tracing::Level::INFO, "OTLP relay summary"), 1); + drop(collector.shutdown().await); + } + + #[tokio::test] + async fn first_rejection_warns_once() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let counters = RelayCounters::default(); + for _ in 0..10 { + counters.record_rejection(); + } + assert_eq!( + logs.count( + tracing::Level::WARN, + "OTLP relay rejected an agent export: buffer full" + ), + 1 + ); + assert_eq!(counters.snapshot().rejected, 10); + } + + #[tokio::test] + async fn first_export_failure_warns_once() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let counters = RelayCounters::default(); + for _ in 0..3 { + counters.record_export_failure("http://collector:4317", &"boom"); + } + assert_eq!( + logs.count( + tracing::Level::WARN, + "OTLP relay export to collector failed" + ), + 1 + ); + assert_eq!(counters.snapshot().export_failures, 3); + } + + #[tokio::test(start_paused = true)] + async fn summary_is_logged_only_while_counts_change() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let counters = Arc::new(RelayCounters::default()); + let summary = tokio::spawn(summary_loop(Arc::clone(&counters))); + + let summaries = || logs.count(tracing::Level::INFO, "OTLP relay summary"); + tokio::time::advance(SUMMARY_INTERVAL + Duration::from_millis(1)).await; + tokio::task::yield_now().await; + assert_eq!(summaries(), 0, "no summary while nothing was rejected"); + + counters.record_rejection(); + tokio::time::advance(SUMMARY_INTERVAL).await; + tokio::task::yield_now().await; + assert_eq!(summaries(), 1); + let first = &logs.events(tracing::Level::INFO, "OTLP relay summary")[0]; + assert_eq!(first.field("rejected"), Some(1)); + assert_eq!(first.field("export_failures"), Some(0)); + + tokio::time::advance(SUMMARY_INTERVAL).await; + tokio::task::yield_now().await; + assert_eq!(summaries(), 1, "unchanged counts emit no summary"); + + counters.record_export_failure("http://collector:4317", &"boom"); + tokio::time::advance(SUMMARY_INTERVAL).await; + tokio::task::yield_now().await; + assert_eq!(summaries(), 2); + // Counts are cumulative, not per interval. + let second = &logs.events(tracing::Level::INFO, "OTLP relay summary")[1]; + assert_eq!(second.field("rejected"), Some(1)); + assert_eq!(second.field("export_failures"), Some(1)); + summary.abort(); + } + + #[tokio::test] + async fn failed_export_does_not_stop_the_relay() { + // A freed ephemeral port can be taken by a parallel test before the + // collector binds it, so try a few candidates. + for attempt in 0..5 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + let (relay, reserved, gate) = gated(&format!("http://{addr}")); + gate.send_replace(true); + + assert_eq!(post(&relay, "lost").await, 200); + wait_until(Duration::from_secs(10), || { + relay.counters.snapshot().export_failures == 1 + }) + .await; + + let Ok(collector) = OtlpTestServer::try_start_on(addr).await else { + eprintln!("port {addr} was taken meanwhile, retrying ({attempt})"); + drop(reserved); + continue; + }; + assert_eq!(post(&relay, "delivered").await, 200); + collector.wait_for_export().await; + let received = collector.shutdown().await; + assert!(received.spans.iter().any(|span| span.name == "delivered")); + assert_eq!(relay.counters.snapshot().exported, 1); + drop(reserved); + return; + } + panic!("could not rebind a freed port in five attempts"); + } + + #[tokio::test] + async fn buffer_never_exceeds_capacity_under_flood() { + // The gate stays closed: nothing drains the buffer, which is the + // worst case of a collector that never answers. + let logs = LogCapture::default(); + let _guard = logs.install(); + let collector = SilentCollector::start().await; + let (relay, _reserved, _gate) = gated(&collector.endpoint); + + // A 1 MiB span attribute: large on the wire, small resource. + let body = encoded_request_with_large_span("flood", 1024 * 1024); + let req = Arc::new(request("POST", "/v1/traces", PROTOBUF, &body)); + let mut clients = Vec::new(); + for _ in 0..MAX_CONCURRENT_CONNECTIONS { + clients.push(connect(&relay.server)); + } + assert!(relay.server.try_reserve().is_none(), "cap reached"); + + let mut accepted = 0; + let mut rejected = 0; + for i in 0..200 { + let client = &mut clients[i % MAX_CONCURRENT_CONNECTIONS]; + let head = send(client, &req).await; + match head.status { + 200 => accepted += 1, + 503 => rejected += 1, + other => panic!("unexpected status {other}"), + } + } + assert_eq!( + accepted, BUFFER_CAPACITY, + "accepted {accepted}: exactly the buffer capacity" + ); + assert_eq!(accepted + rejected, 200); + assert_eq!(relay.counters.snapshot().rejected, rejected as u64); + // FR-009: one warning for the first rejection, nothing per request. + assert_eq!( + logs.count( + tracing::Level::WARN, + "OTLP relay rejected an agent export: buffer full" + ), + 1 + ); + assert!( + logs.info_and_above() < 5, + "relay log lines must not scale with rejections: {}", + logs.info_and_above() + ); + drop(relay); + } + + #[tokio::test] + async fn shutdown_flushes_buffered_batches() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let collector = OtlpTestServer::start().await; + let (relay, _reserved, gate) = gated(collector.endpoint()); + + for span in ["one", "two", "three"] { + assert_eq!(post(&relay, span).await, 200); + } + assert_eq!(relay.counters.pending(), 3); + + gate.send_replace(true); + let started = Instant::now(); + relay.shutdown(FLUSH_TIMEOUT).await; + assert!(started.elapsed() < FLUSH_TIMEOUT); + + let received = collector.shutdown().await; + let names: Vec<&str> = received + .spans + .iter() + .map(|span| span.name.as_str()) + .collect(); + for span in ["one", "two", "three"] { + assert!(names.contains(&span), "missing {span} in {names:?}"); + } + assert_eq!( + logs.count(tracing::Level::INFO, "OTLP relay flushed before exit"), + 1 + ); + assert_eq!( + logs.count(tracing::Level::INFO, "OTLP relay summary"), + 0, + "no final summary when nothing was rejected or failed" + ); + } + + #[tokio::test] + async fn shutdown_with_unreachable_collector_returns_within_bound() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let collector = SilentCollector::start().await; + let (relay, _reserved, gate) = gated(&collector.endpoint); + for span in ["one", "two"] { + assert_eq!(post(&relay, span).await, 200); + } + let counters = Arc::clone(&relay.counters); + + gate.send_replace(true); + let started = Instant::now(); + relay.shutdown(FLUSH_TIMEOUT).await; + assert!( + started.elapsed() < FLUSH_TIMEOUT + Duration::from_secs(1), + "shutdown took {:?}", + started.elapsed() + ); + assert_eq!( + logs.count( + tracing::Level::WARN, + "OTLP relay flush deadline exceeded; discarding buffered batches" + ), + 1 + ); + assert_eq!( + counters.snapshot().export_failures, + 2, + "both batches count as failures" + ); + let summaries = logs.events(tracing::Level::INFO, "OTLP relay summary"); + assert_eq!( + summaries.len(), + 1, + "final summary when failures are non-zero" + ); + assert_eq!(summaries[0].field("export_failures"), Some(2)); + assert_eq!(summaries[0].field("rejected"), Some(0)); + } + + #[tokio::test] + async fn shutdown_logs_final_summary_when_counters_nonzero() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let collector = OtlpTestServer::start().await; + let (relay, _reserved, gate) = gated(collector.endpoint()); + relay.counters.record_rejection(); + gate.send_replace(true); + relay.shutdown(FLUSH_TIMEOUT).await; + assert_eq!(logs.count(tracing::Level::INFO, "OTLP relay summary"), 1); + drop(collector.shutdown().await); + } + + #[tokio::test] + async fn handler_refuses_streams_once_shutdown_started() { + let collector = OtlpTestServer::start().await; + let (relay, reserved, gate) = gated(collector.endpoint()); + gate.send_replace(true); + relay.shutdown(FLUSH_TIMEOUT).await; + let (_client, boundary) = tokio::io::duplex(1024); + assert!( + reserved.handler.serve(Box::new(boundary)).is_none(), + "the proxy must refuse the open rather than reset it after RelayReady" + ); + drop(collector.shutdown().await); + } +} diff --git a/crates/openshell-supervisor/src/otlp_relay/receiver.rs b/crates/openshell-supervisor/src/otlp_relay/receiver.rs new file mode 100644 index 0000000000..f579c3160c --- /dev/null +++ b/crates/openshell-supervisor/src/otlp_relay/receiver.rs @@ -0,0 +1,1165 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OTLP/HTTP receiver served on supervisor-mediated streams. +//! +//! The agent exports to the reserved relay address. The sandbox's seccomp +//! broker stages that connect for the supervisor, whose proxy hands the +//! resulting duplex stream to [`OtlpConnectionServer::serve`] instead of +//! dialing upstream. The server speaks HTTP/1.1 on that stream and accepts +//! `POST /v1/traces` with an uncompressed protobuf body. No socket is ever +//! bound. + +use std::convert::Infallible; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use bytes::Bytes; +use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE, RETRY_AFTER}; +use http_body_util::{BodyExt as _, Full}; +use hyper::body::Incoming; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, StatusCode}; +use hyper_util::rt::{TokioIo, TokioTimer}; +use hyper_util::server::graceful::{GracefulShutdown, Watcher}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch}; +use tracing::{debug, warn}; + +use super::buffer::{BatchSender, SendError}; +use prost::Message as _; +use tonic_types::Status as RpcStatus; + +use super::{ + BODY_READ_TIMEOUT, ENRICHMENT_CONCURRENCY, HEADER_READ_TIMEOUT, MAX_BODY_BYTES, + MAX_CONCURRENT_CONNECTIONS, MAX_ENRICHED_BYTES, RECEIVER_SHUTDOWN_TIMEOUT, RETRY_AFTER_SECS, + RelayCounters, enrichment, +}; + +/// Reservation for one relay connection. +/// +/// Taken before the sandbox commits the workload socket, so an exhausted +/// server can refuse the open instead of resetting it after `RelayReady`. +pub struct ConnectionPermit { + _permit: OwnedSemaphorePermit, +} + +/// Serves OTLP/HTTP on streams handed over by the mediation path. +/// +/// One instance lives for the sandbox lifetime. Each stream is served on the +/// caller's task through [`OtlpConnectionServer::serve`]; the paired +/// [`ReceiverHandle`] owns shutdown. +pub struct OtlpConnectionServer { + /// `None` once the buffer was closed at shutdown. + buffer_tx: Mutex>, + sandbox_id: String, + counters: Arc, + connections: Arc, + /// Bounds how many bodies are being enriched at once. + enrichment_slots: Semaphore, + accepting: AtomicBool, + /// `None` once shutdown has started. Watchers are minted under this lock, + /// so none can be created after the shutdown signal is sent. + graceful: Mutex>, + abort_rx: watch::Receiver, +} + +/// Owns receiver shutdown. +pub struct ReceiverHandle { + server: Arc, + abort_tx: watch::Sender, +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl OtlpConnectionServer { + /// Creates a server and its shutdown handle. + pub fn new( + buffer_tx: BatchSender, + sandbox_id: String, + counters: Arc, + ) -> (Arc, ReceiverHandle) { + let (abort_tx, abort_rx) = watch::channel(false); + let server = Arc::new(Self { + buffer_tx: Mutex::new(Some(buffer_tx)), + sandbox_id, + counters, + connections: Arc::new(Semaphore::new(MAX_CONCURRENT_CONNECTIONS)), + enrichment_slots: Semaphore::new(ENRICHMENT_CONCURRENCY), + accepting: AtomicBool::new(true), + graceful: Mutex::new(Some(GracefulShutdown::new())), + abort_rx, + }); + let handle = ReceiverHandle { + server: Arc::clone(&server), + abort_tx, + }; + (server, handle) + } + + /// Reserves a connection slot. `None` once [`Self::stop_accepting`] was + /// called or while [`MAX_CONCURRENT_CONNECTIONS`] streams are served. + pub fn try_reserve(&self) -> Option { + if !self.accepting.load(Ordering::SeqCst) { + return None; + } + let permit = Arc::clone(&self.connections).try_acquire_owned().ok()?; + Some(ConnectionPermit { _permit: permit }) + } + + /// Makes every later [`Self::try_reserve`] return `None`. + pub fn stop_accepting(&self) { + self.accepting.store(false, Ordering::SeqCst); + } + + /// Drops the buffer sender so the export task observes end of input once + /// the buffer drains. Requests still in flight answer 503. + pub fn close_buffer(&self) { + lock(&self.buffer_tx).take(); + } + + /// Number of connection slots currently in use. + fn open_connections(&self) -> usize { + MAX_CONCURRENT_CONNECTIONS - self.connections.available_permits() + } + + /// Number of enrichment slots currently free. + #[cfg(test)] + pub(crate) fn available_enrichment_slots(&self) -> usize { + self.enrichment_slots.available_permits() + } + + fn watcher(&self) -> Option { + lock(&self.graceful).as_ref().map(GracefulShutdown::watcher) + } + + /// Serves one HTTP/1.1 connection on `stream` until the peer closes it, + /// the header-read timeout fires, graceful shutdown drains it, or the + /// abort deadline cuts it. Runs on the caller's task and releases + /// `permit` when it returns. + pub async fn serve(self: &Arc, permit: ConnectionPermit, stream: S) + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let Some(watcher) = self.watcher() else { + debug!("OTLP relay: stream arrived during shutdown, dropping"); + return; + }; + + let server = Arc::clone(self); + let service = service_fn(move |request| { + let server = Arc::clone(&server); + async move { server.handle_request(request).await } + }); + + let mut builder = http1::Builder::new(); + // A timer is mandatory once any timeout is configured; hyper panics in + // `serve_connection` otherwise. + builder + .timer(TokioTimer::new()) + .header_read_timeout(HEADER_READ_TIMEOUT); + let connection = watcher.watch(builder.serve_connection(TokioIo::new(stream), service)); + + let mut abort_rx = self.abort_rx.clone(); + tokio::select! { + result = connection => { + if let Err(error) = result { + debug!(%error, "OTLP relay connection ended with an error"); + } + } + _ = abort_rx.wait_for(|aborted| *aborted) => { + debug!("OTLP relay: aborting connection after graceful timeout"); + } + } + drop(permit); + } + + async fn handle_request( + &self, + request: Request, + ) -> Result>, Infallible> { + if request.method() != Method::POST || request.uri().path() != "/v1/traces" { + return Ok(rejected( + StatusCode::NOT_FOUND, + "not found", + "not found; the relay serves POST /v1/traces only", + )); + } + if request.headers().contains_key(CONTENT_ENCODING) { + return Ok(rejected( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "compressed body", + "compression is not supported; send an uncompressed application/x-protobuf body", + )); + } + // Media types are case-insensitive and may carry parameters. + let protobuf = request + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .is_some_and(|media_type| { + media_type + .trim() + .eq_ignore_ascii_case("application/x-protobuf") + }); + if !protobuf { + return Ok(rejected( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "unsupported content type", + "unsupported content type; send application/x-protobuf", + )); + } + let declared_length = request + .headers() + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + + let body = match tokio::time::timeout( + BODY_READ_TIMEOUT, + read_body(request.into_body(), declared_length), + ) + .await + { + Ok(BodyRead::Complete(body)) => body, + Ok(BodyRead::TooLarge) => { + return Ok(rejected( + StatusCode::PAYLOAD_TOO_LARGE, + "body exceeds limit", + "request body exceeds the 2 MiB limit", + )); + } + Ok(BodyRead::Failed(error)) => { + debug!(%error, "OTLP relay failed to read a request body"); + return Ok(rejected( + StatusCode::BAD_REQUEST, + "unreadable body", + "failed to read request body", + )); + } + // Returning drops the half-read body; hyper closes the connection + // because the request was not consumed, which frees the slot. + Err(_) => { + return Ok(rejected( + StatusCode::REQUEST_TIMEOUT, + "body read timed out", + "request body was not received in time", + )); + } + }; + + let enriched = { + let _slot = self + .enrichment_slots + .acquire() + .await + .expect("the enrichment semaphore is never closed"); + enrichment::enrich(&body, &self.sandbox_id) + }; + // The raw body is no longer needed once the enriched copy exists. + drop(body); + let enriched = match enriched { + Ok(enriched) => enriched, + Err(error @ enrichment::EnrichError::Malformed(_)) => { + return Ok(rejected( + StatusCode::BAD_REQUEST, + "protobuf decode failed", + &error.to_string(), + )); + } + Err( + error @ (enrichment::EnrichError::TooManyResourceSpans + | enrichment::EnrichError::TooManyResourceAttributes + | enrichment::EnrichError::ResourceTooLarge), + ) => { + return Ok(rejected( + StatusCode::PAYLOAD_TOO_LARGE, + "request exceeds structural limits", + &error.to_string(), + )); + } + }; + // The buffer bound is sized for MAX_ENRICHED_BYTES per batch. The + // structural limits above keep every request under it for the sandbox + // id lengths the attribution budget assumes; this guards that + // assumption. + if enriched.len() > MAX_ENRICHED_BYTES { + return Ok(rejected( + StatusCode::PAYLOAD_TOO_LARGE, + "enriched body exceeds limit", + "request body exceeds the 2 MiB limit after attribution", + )); + } + + let sent = lock(&self.buffer_tx) + .as_ref() + .map_or(Err(SendError::BufferClosed), |tx| tx.try_send(enriched)); + match sent { + Ok(()) => { + self.counters.record_accepted(); + Ok(Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, "application/x-protobuf") + .body(Full::new(Bytes::new())) + .expect("static response builds")) + } + Err(SendError::BufferFull) => { + self.counters.record_rejection(); + Ok(unavailable()) + } + Err(SendError::BufferClosed) => { + debug!("OTLP relay refused an export: relay is shutting down"); + Ok(unavailable()) + } + } + } +} + +/// Bytes of an oversize body the receiver reads and discards so the 413 +/// reaches a client that is still sending. Beyond this the connection is +/// closed and the client sees a reset instead. +const OVERSIZE_DRAIN_CAP: usize = 8 * MAX_BODY_BYTES; + +enum BodyRead { + Complete(Bytes), + TooLarge, + Failed(hyper::Error), +} + +/// Reads the body up to [`MAX_BODY_BYTES`]. Once the limit is crossed, or +/// when the declared length already exceeds it, the rest is discarded up +/// to [`OVERSIZE_DRAIN_CAP`] so the response can be delivered on a +/// connection the client has finished writing to. +async fn read_body(mut body: Incoming, declared_length: Option) -> BodyRead { + let mut too_large = declared_length.is_some_and(|length| length > MAX_BODY_BYTES); + // Size the buffer once from the declared length when the first bytes + // arrive, so growth never leaves a body-sized slack behind and a client + // that declares a length but sends nothing reserves nothing; chunked + // bodies are shrunk at the end instead. + let reserve = if too_large { + 0 + } else { + declared_length.unwrap_or(0) + }; + let mut buffered = Vec::new(); + let mut total = 0usize; + while let Some(frame) = body.frame().await { + let frame = match frame { + Ok(frame) => frame, + Err(_) if too_large => break, + Err(error) => return BodyRead::Failed(error), + }; + let Some(data) = frame.data_ref() else { + continue; + }; + total += data.len(); + if too_large { + if total > OVERSIZE_DRAIN_CAP { + break; + } + continue; + } + if total > MAX_BODY_BYTES { + too_large = true; + buffered = Vec::new(); + continue; + } + // Grow in doubling steps capped at the body limit, so a chunked body + // that stalls mid-way never holds more than MAX_BODY_BYTES of + // capacity; declared lengths are reserved exactly on the first frame. + let needed = buffered.len() + data.len(); + if needed > buffered.capacity() { + let target = if buffered.capacity() == 0 { + reserve.max(data.len()) + } else { + (buffered.capacity() * 2).min(MAX_BODY_BYTES).max(needed) + }; + buffered.reserve_exact(target - buffered.len()); + } + buffered.extend_from_slice(data); + } + if too_large { + BodyRead::TooLarge + } else { + buffered.shrink_to_fit(); + BodyRead::Complete(Bytes::from(buffered)) + } +} + +/// Builds an error response the way OTLP/HTTP specifies: a protobuf-encoded +/// `google.rpc.Status` in the request's content type, so collectors and SDKs +/// that decode it surface `message` instead of a bare status code. +fn rejected(status: StatusCode, reason: &str, message: &str) -> Response> { + debug!( + status = status.as_u16(), + reason, "OTLP relay rejected request" + ); + let code = match status { + StatusCode::NOT_FOUND => tonic::Code::NotFound, + StatusCode::PAYLOAD_TOO_LARGE => tonic::Code::ResourceExhausted, + StatusCode::REQUEST_TIMEOUT => tonic::Code::DeadlineExceeded, + _ => tonic::Code::InvalidArgument, + }; + status_response(status, code, message) +} + +fn unavailable() -> Response> { + let mut response = status_response( + StatusCode::SERVICE_UNAVAILABLE, + tonic::Code::Unavailable, + "relay buffer is full; retry after the Retry-After interval", + ); + response.headers_mut().insert( + RETRY_AFTER, + RETRY_AFTER_SECS.to_string().parse().expect("ascii"), + ); + response +} + +fn status_response(status: StatusCode, code: tonic::Code, message: &str) -> Response> { + let body = RpcStatus { + code: code as i32, + message: message.to_string(), + details: Vec::new(), + } + .encode_to_vec(); + Response::builder() + .status(status) + .header(CONTENT_TYPE, "application/x-protobuf") + .body(Full::new(Bytes::from(body))) + .expect("static response builds") +} + +impl ReceiverHandle { + /// Stops accepting new streams, disables keep-alive on every open + /// connection, waits up to [`RECEIVER_SHUTDOWN_TIMEOUT`] for in-flight + /// requests, then cuts stragglers. Always returns within roughly the + /// timeout. Consumes the handle, so it runs exactly once. + pub async fn shutdown(self) { + self.server.stop_accepting(); + let graceful = lock(&self.server.graceful) + .take() + .expect("graceful shutdown is taken exactly once, by this method"); + if tokio::time::timeout(RECEIVER_SHUTDOWN_TIMEOUT, graceful.shutdown()) + .await + .is_err() + { + warn!( + open_connections = self.server.open_connections(), + "OTLP relay: aborting connections still open after graceful timeout" + ); + self.abort_tx.send_replace(true); + } + } +} + +/// In-memory HTTP helpers shared by the receiver and lifecycle tests. They +/// stand in for the boundary duplex stream the proxy hands over in +/// production. +#[cfg(test)] +pub mod test_util { + use std::fmt::Write as _; + use std::sync::Arc; + use std::time::Duration; + + use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream}; + + use super::OtlpConnectionServer; + + /// A complete HTTP/1.1 request with a `Content-Length` body. + pub fn request(method: &str, path: &str, content_type: &str, body: &[u8]) -> Vec { + request_with_headers(method, path, &[("Content-Type", content_type)], body) + } + + pub fn request_with_headers( + method: &str, + path: &str, + headers: &[(&str, &str)], + body: &[u8], + ) -> Vec { + let mut req = format!("{method} {path} HTTP/1.1\r\nHost: relay\r\n"); + for (name, value) in headers { + let _ = write!(req, "{name}: {value}\r\n"); + } + let _ = write!(req, "Content-Length: {}\r\n\r\n", body.len()); + let mut req = req.into_bytes(); + req.extend_from_slice(body); + req + } + + /// Opens a client stream whose server half is served on a spawned task, + /// the way the proxy hook serves a staged boundary stream. + pub fn connect(server: &Arc) -> DuplexStream { + let (client, server_half) = tokio::io::duplex(64 * 1024); + let permit = server.try_reserve().expect("connection slot"); + let server = Arc::clone(server); + tokio::spawn(async move { server.serve(permit, server_half).await }); + client + } + + /// Parsed response head: status code and headers. + pub struct Head { + pub status: u16, + pub headers: Vec<(String, String)>, + /// The response body, which every relay response declares with a + /// `Content-Length`. + pub body: Vec, + } + + impl Head { + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + /// Decodes the `google.rpc.Status` body every error response carries. + pub fn rpc_status(&self) -> tonic_types::Status { + assert_eq!( + self.header("content-type"), + Some("application/x-protobuf"), + "error bodies use the request's content type" + ); + prost::Message::decode(self.body.as_slice()).expect("google.rpc.Status body") + } + } + + /// Writes `req` and reads until the response head is complete. Returns + /// the parsed head; the stream stays open. + pub async fn send(stream: &mut S, req: &[u8]) -> Head { + stream.write_all(req).await.expect("write request"); + read_head(stream).await + } + + pub async fn read_head(stream: &mut S) -> Head { + let mut buf = Vec::new(); + let mut chunk = [0u8; 1024]; + let head_end = loop { + let n = tokio::time::timeout(Duration::from_secs(5), stream.read(&mut chunk)) + .await + .expect("response within 5s") + .expect("read response"); + assert!(n > 0, "connection closed before response head"); + buf.extend_from_slice(&chunk[..n]); + if let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + break end; + } + }; + let head = String::from_utf8_lossy(&buf[..head_end]).to_string(); + let mut lines = head.lines(); + let status_line = lines.next().unwrap_or_default(); + let status = status_line + .split_whitespace() + .nth(1) + .and_then(|code| code.parse().ok()) + .unwrap_or_else(|| panic!("status line: {status_line}")); + let headers: Vec<(String, String)> = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.trim().to_string(), value.trim().to_string())) + .collect(); + // Read the body so a keep-alive connection is ready for the next + // request. Every relay response is either empty or has a declared + // Content-Length. + let body_len: usize = headers + .iter() + .find(|(name, _): &&(String, String)| name.eq_ignore_ascii_case("content-length")) + .and_then(|(_, value)| value.parse().ok()) + .unwrap_or(0); + let mut body = buf[head_end + 4..].to_vec(); + while body.len() < body_len { + let n = stream.read(&mut chunk).await.expect("read body"); + assert!(n > 0, "connection closed inside the body"); + body.extend_from_slice(&chunk[..n]); + } + body.truncate(body_len); + Head { + status, + headers, + body, + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + use super::super::buffer::{BatchReceiver, new_buffer}; + use super::super::enrichment::test_util::{ + encoded_request, encoded_request_with_large_span, resource_attributes, + }; + use super::super::{BUFFER_CAPACITY, enrichment}; + use super::test_util::{connect, read_head, request, request_with_headers, send}; + use super::*; + + const PROTOBUF: &str = "application/x-protobuf"; + + fn start(capacity: usize) -> (Arc, ReceiverHandle, BatchReceiver) { + let (tx, rx) = new_buffer(capacity); + let counters = Arc::new(RelayCounters::default()); + let (server, handle) = OtlpConnectionServer::new(tx, "sb-test".into(), counters); + (server, handle, rx) + } + + fn body() -> Vec { + encoded_request("test-span", &[("service.name", "unit-agent")]) + } + + #[tokio::test] + async fn valid_request_is_accepted_and_enriched() { + let (server, _handle, mut rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(head.status, 200); + assert_eq!(head.header("content-type"), Some(PROTOBUF)); + + let buffered = rx.recv().await.expect("one batch buffered"); + let attributes = resource_attributes(&buffered); + assert!(attributes.contains(&("service.name".to_string(), "unit-agent".to_string()))); + assert!(attributes.contains(&( + enrichment::SANDBOX_ID_KEY.to_string(), + "sb-test".to_string() + ))); + assert!(attributes.contains(&( + enrichment::SOURCE_KEY.to_string(), + enrichment::SOURCE_VALUE.to_string() + ))); + } + + #[tokio::test] + async fn other_paths_and_methods_are_not_found() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let head = send(&mut client, &request("POST", "/v1/metrics", PROTOBUF, b"")).await; + assert_eq!(head.status, 404); + let head = send(&mut client, &request("GET", "/v1/traces", PROTOBUF, b"")).await; + assert_eq!(head.status, 404); + } + + #[tokio::test] + async fn json_body_is_unsupported() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let head = send( + &mut client, + &request("POST", "/v1/traces", "application/json", b"{}"), + ) + .await; + assert_eq!(head.status, 415); + // The spec requires the body to state the supported content type, as + // a google.rpc.Status in the request's content type. + let status = head.rpc_status(); + assert_eq!(status.code, tonic::Code::InvalidArgument as i32); + assert!( + status.message.contains("application/x-protobuf"), + "415 body must name the supported content type: {}", + status.message + ); + } + + #[tokio::test] + async fn mixed_case_protobuf_content_type_with_parameters_is_accepted() { + let (server, _handle, mut rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let head = send( + &mut client, + &request( + "POST", + "/v1/traces", + "Application/X-Protobuf; charset=binary", + &body(), + ), + ) + .await; + assert_eq!(head.status, 200); + assert!(rx.recv().await.is_some()); + } + + #[tokio::test] + async fn compressed_body_is_unsupported() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let head = send( + &mut client, + &request_with_headers( + "POST", + "/v1/traces", + &[("Content-Type", PROTOBUF), ("Content-Encoding", "gzip")], + &body(), + ), + ) + .await; + assert_eq!(head.status, 415); + // The spec requires the body to state that compression is unsupported. + let status = head.rpc_status(); + assert!( + status.message.contains("compression is not supported"), + "415 body must say compression is unsupported: {}", + status.message + ); + } + + #[tokio::test(start_paused = true)] + async fn stalled_body_is_timed_out_and_frees_its_slot() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + // Complete headers, then only a fraction of the declared body. + let req = format!( + "POST /v1/traces HTTP/1.1\r\nHost: relay\r\nContent-Type: {PROTOBUF}\r\nContent-Length: 4096\r\n\r\n" + ); + client.write_all(req.as_bytes()).await.unwrap(); + client.write_all(&[0u8; 16]).await.unwrap(); + // Let the server parse the headers and start its body timer. + for _ in 0..8 { + tokio::task::yield_now().await; + } + assert_eq!(server.open_connections(), 1); + + // Jump past the body deadline (the read helper's own 5 s timer would + // otherwise fire first under paused time); the server answers 408 and + // closes the connection. + tokio::time::advance(BODY_READ_TIMEOUT + Duration::from_secs(1)).await; + let head = read_head(&mut client).await; + assert_eq!(head.status, 408); + assert_eq!(head.rpc_status().code, tonic::Code::DeadlineExceeded as i32); + let mut buf = [0u8; 8]; + let n = tokio::time::timeout(BODY_READ_TIMEOUT, client.read(&mut buf)) + .await + .expect("server closes the stalled connection") + .unwrap_or(0); + assert_eq!(n, 0, "expected EOF after the body timeout"); + tokio::task::yield_now().await; + assert_eq!(server.open_connections(), 0); + } + + #[tokio::test] + async fn structurally_abusive_bodies_are_too_large_and_never_buffered() { + let (server, _handle, mut rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + + // One more empty `ResourceSpans` entry than the cap: far under the + // wire limit, so the 413 can only come from the structural check. + let many_entries: Vec = [0x0a, 0x00].repeat(enrichment::MAX_RESOURCE_SPANS + 1); + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &many_entries), + ) + .await; + assert_eq!(head.status, 413); + + // One resource larger than the decode bound. + let huge_resource = encoded_request( + "r", + &[("blob", &"x".repeat(enrichment::MAX_RESOURCE_BYTES))], + ); + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &huge_resource), + ) + .await; + assert_eq!(head.status, 413); + + // The connection still serves, and the first buffered batch is the + // valid one: neither refused body was buffered ahead of it. + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(head.status, 200); + let buffered = rx.recv().await.expect("the valid batch is buffered"); + let attributes = resource_attributes(&buffered); + assert!(attributes.contains(&("service.name".to_string(), "unit-agent".to_string()))); + let rejections = server.counters.snapshot().rejected; + assert_eq!(rejections, 0, "limit refusals are not back-pressure"); + } + + #[tokio::test] + async fn sixteen_simultaneous_requests_are_all_served() { + let (server, _handle, mut rx) = start(BUFFER_CAPACITY); + let valid = Arc::new(request( + "POST", + "/v1/traces", + PROTOBUF, + &encoded_request_with_large_span("burst", 256 * 1024), + )); + let garbage = Arc::new(request("POST", "/v1/traces", PROTOBUF, &[0xff, 0xfe, 0xfd])); + // Every connection writes its whole request before any response is + // read. Enrichment is synchronous, so on the test runtime the two + // enrichment slots never actually queue a request; this pins that + // sixteen concurrent requests are all answered, that malformed + // bodies release their slot on the error path, and that none of them + // is buffered. + let mut tasks = Vec::new(); + for i in 0..MAX_CONCURRENT_CONNECTIONS { + let mut client = connect(&server); + let req = if i % 5 == 0 { + Arc::clone(&garbage) + } else { + Arc::clone(&valid) + }; + tasks.push(tokio::spawn( + async move { send(&mut client, &req).await.status }, + )); + } + let statuses = + tokio::time::timeout(Duration::from_secs(10), futures::future::join_all(tasks)) + .await + .expect("all sixteen requests answered"); + let mut accepted = 0; + let mut malformed = 0; + for status in statuses { + match status.expect("request task completes") { + 200 => accepted += 1, + 400 => malformed += 1, + other => panic!("unexpected status {other}"), + } + } + assert_eq!(malformed, 4); + assert_eq!(accepted, MAX_CONCURRENT_CONNECTIONS - 4); + for _ in 0..accepted { + let batch = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("accepted batch is buffered within 5s"); + assert!(batch.is_some()); + } + assert!( + tokio::time::timeout(Duration::from_millis(100), rx.recv()) + .await + .is_err(), + "malformed bodies were buffered" + ); + assert_eq!( + server.available_enrichment_slots(), + ENRICHMENT_CONCURRENCY, + "every enrichment permit is released" + ); + } + + #[tokio::test] + async fn enriched_body_over_the_bound_is_too_large_and_never_buffered() { + // A sandbox id far longer than ATTRIBUTION_BYTES_PER_ENTRY accounts + // for: 512 resource-less entries near the wire limit grow past + // MAX_ENRICHED_BYTES, and the receiver must refuse rather than buffer + // an oversize batch. + let (tx, mut rx) = new_buffer(BUFFER_CAPACITY); + let counters = Arc::new(RelayCounters::default()); + let (server, _handle) = OtlpConnectionServer::new(tx, "x".repeat(200), counters); + let mut client = connect(&server); + let entry = { + let scope = [0x12u8, 0x00].repeat(2000); // ~4 KiB of empty spans + let mut scope_field = vec![0x12u8]; // ResourceSpans.scope_spans + scope_field.extend_from_slice(&encode_len(scope.len())); + scope_field.extend_from_slice(&scope); + let mut entry = vec![0x0au8]; // resource_spans + entry.extend_from_slice(&encode_len(scope_field.len())); + entry.extend_from_slice(&scope_field); + entry + }; + let body: Vec = entry.repeat(enrichment::MAX_RESOURCE_SPANS); + assert!(body.len() <= MAX_BODY_BYTES); + // Positive control: the same body is accepted with an ordinary id, + // so the 413 below can only come from the attribution bound. + let (control, _control_handle, mut control_rx) = start(BUFFER_CAPACITY); + let mut control_client = connect(&control); + let head = send( + &mut control_client, + &request("POST", "/v1/traces", PROTOBUF, &body), + ) + .await; + assert_eq!(head.status, 200); + assert!(control_rx.recv().await.is_some()); + + let head = send(&mut client, &request("POST", "/v1/traces", PROTOBUF, &body)).await; + assert_eq!(head.status, 413); + assert!( + tokio::time::timeout(Duration::from_millis(100), rx.recv()) + .await + .is_err(), + "an oversize enriched batch was buffered" + ); + let rejections = server.counters.snapshot().rejected; + assert_eq!(rejections, 0, "limit refusals are not back-pressure"); + } + + /// Protobuf varint length prefix. + fn encode_len(len: usize) -> Vec { + let mut out = Vec::new(); + prost::encoding::encode_varint(len as u64, &mut out); + out + } + + #[tokio::test] + async fn garbage_body_is_a_bad_request() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &[0xff, 0xfe, 0xfd]), + ) + .await; + assert_eq!(head.status, 400); + } + + #[tokio::test] + async fn keep_alive_serves_a_second_request_on_the_same_connection() { + let (server, _handle, mut rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + for _ in 0..2 { + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(head.status, 200); + } + assert!(rx.recv().await.is_some()); + assert!(rx.recv().await.is_some()); + assert_eq!(server.open_connections(), 1); + } + + #[tokio::test] + async fn declared_oversize_body_is_too_large() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let req = format!( + "POST /v1/traces HTTP/1.1\r\nHost: relay\r\nContent-Type: {PROTOBUF}\r\nContent-Length: {}\r\n\r\n", + MAX_BODY_BYTES + 1 + ); + client.write_all(req.as_bytes()).await.unwrap(); + // The server drains the oversize body so the client, which keeps + // sending, still receives the 413 on an intact connection. + client + .write_all(&vec![b'x'; MAX_BODY_BYTES + 1]) + .await + .unwrap(); + let head = read_head(&mut client).await; + assert_eq!(head.status, 413); + // The connection stays usable afterwards. + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(head.status, 200); + } + + #[tokio::test] + async fn chunked_oversize_body_is_too_large() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let req = format!( + "POST /v1/traces HTTP/1.1\r\nHost: relay\r\nContent-Type: {PROTOBUF}\r\nTransfer-Encoding: chunked\r\n\r\n" + ); + client.write_all(req.as_bytes()).await.unwrap(); + let chunk = vec![b'x'; 64 * 1024]; + let chunk_head = format!("{:x}\r\n", chunk.len()); + let mut sent = 0; + // Write chunks past the limit, then terminate the body. The server + // discards the excess and answers once the body ends. + while sent <= MAX_BODY_BYTES + chunk.len() { + client.write_all(chunk_head.as_bytes()).await.unwrap(); + client.write_all(&chunk).await.unwrap(); + client.write_all(b"\r\n").await.unwrap(); + sent += chunk.len(); + } + client.write_all(b"0\r\n\r\n").await.unwrap(); + assert_eq!(read_head(&mut client).await.status, 413); + } + + #[tokio::test] + async fn oversize_body_beyond_the_drain_cap_closes_the_connection() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let req = format!( + "POST /v1/traces HTTP/1.1\r\nHost: relay\r\nContent-Type: {PROTOBUF}\r\nContent-Length: {}\r\n\r\n", + OVERSIZE_DRAIN_CAP * 2 + ); + client.write_all(req.as_bytes()).await.unwrap(); + let chunk = vec![b'x'; 256 * 1024]; + let mut sent = 0; + // Writes fail once the server stops draining and closes; the receiver + // never buffers more than the cap. + while sent <= OVERSIZE_DRAIN_CAP * 2 { + if client.write_all(&chunk).await.is_err() { + break; + } + sent += chunk.len(); + } + assert!(sent <= OVERSIZE_DRAIN_CAP * 2, "sent {sent} bytes"); + assert!(server.try_reserve().is_some()); + } + + #[tokio::test(start_paused = true)] + async fn idle_connection_is_closed_after_the_header_timeout_and_frees_its_slot() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + tokio::task::yield_now().await; + assert_eq!(server.open_connections(), 1); + + // Nothing is written. The header-read timeout closes the connection. + let mut buf = [0u8; 8]; + let n = tokio::time::timeout(HEADER_READ_TIMEOUT * 2, client.read(&mut buf)) + .await + .expect("server closes the idle connection") + .unwrap_or(0); + assert_eq!(n, 0, "expected EOF after the header-read timeout"); + tokio::task::yield_now().await; + assert_eq!(server.open_connections(), 0); + assert!(server.try_reserve().is_some()); + } + + #[tokio::test] + async fn connection_cap_refuses_the_seventeenth_and_recovers() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + let mut permits: Vec = (0..MAX_CONCURRENT_CONNECTIONS) + .map(|i| server.try_reserve().unwrap_or_else(|| panic!("slot {i}"))) + .collect(); + assert!(server.try_reserve().is_none()); + drop(permits.pop()); + assert!(server.try_reserve().is_some()); + } + + #[tokio::test] + async fn try_reserve_refuses_after_stop_accepting() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + assert!(server.try_reserve().is_some()); + server.stop_accepting(); + assert!(server.try_reserve().is_none()); + } + + #[tokio::test] + async fn two_connections_are_served_concurrently() { + let (server, _handle, mut rx) = start(BUFFER_CAPACITY); + let mut first = connect(&server); + let mut second = connect(&server); + let first_req = request("POST", "/v1/traces", PROTOBUF, &encoded_request("one", &[])); + let second_req = request("POST", "/v1/traces", PROTOBUF, &encoded_request("two", &[])); + // Interleave: write the first request's head only, then the whole + // second request, then finish the first. + let split = first_req.len() / 2; + first.write_all(&first_req[..split]).await.unwrap(); + let second_head = send(&mut second, &second_req).await; + assert_eq!(second_head.status, 200); + first.write_all(&first_req[split..]).await.unwrap(); + let first_head = read_head(&mut first).await; + assert_eq!(first_head.status, 200); + + let a = resource_attributes(&rx.recv().await.unwrap()); + let b = resource_attributes(&rx.recv().await.unwrap()); + let id = |attrs: &Vec<(String, String)>| { + attrs + .iter() + .find(|(key, _)| key == enrichment::SANDBOX_ID_KEY) + .map(|(_, value)| value.clone()) + }; + assert_eq!(id(&a), Some("sb-test".to_string())); + assert_eq!(id(&a), id(&b)); + } + + #[tokio::test] + async fn full_buffer_answers_503_with_retry_after_and_counts_the_rejection() { + let (server, _handle, mut rx) = start(1); + let mut client = connect(&server); + let accepted = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(accepted.status, 200); + + let rejected = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(rejected.status, 503); + assert_eq!( + rejected.header("retry-after"), + Some(RETRY_AFTER_SECS.to_string().as_str()) + ); + assert_eq!( + rejected.rpc_status().code, + tonic::Code::Unavailable as i32, + "503 carries a google.rpc.Status so SDKs can surface the reason" + ); + let rejections = server.counters.snapshot().rejected; + assert_eq!(rejections, 1); + + // Draining the buffer makes the same connection usable again. + assert!(rx.recv().await.is_some()); + let again = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(again.status, 200); + } + + #[tokio::test] + async fn shutdown_completes_with_idle_keepalive_connection() { + let (server, handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(head.status, 200); + + let deadline = RECEIVER_SHUTDOWN_TIMEOUT + Duration::from_secs(1); + tokio::time::timeout(deadline, handle.shutdown()) + .await + .expect("shutdown completes despite an open keep-alive connection"); + + let mut buf = [0u8; 16]; + let n = tokio::time::timeout(Duration::from_secs(2), client.read(&mut buf)) + .await + .expect("server close within 2s") + .unwrap_or(0); + assert_eq!(n, 0, "expected EOF from server after shutdown"); + assert!(server.try_reserve().is_none()); + } + + #[tokio::test] + async fn shutdown_completes_with_half_sent_headers() { + let (server, handle, _rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + client + .write_all(b"POST /v1/traces HTTP/1.1\r\n") + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + let deadline = RECEIVER_SHUTDOWN_TIMEOUT + Duration::from_secs(1); + tokio::time::timeout(deadline, handle.shutdown()) + .await + .expect("shutdown completes with an in-progress request"); + } + + #[tokio::test] + async fn closed_buffer_answers_503() { + let (server, _handle, _rx) = start(BUFFER_CAPACITY); + server.close_buffer(); + let mut client = connect(&server); + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(head.status, 503); + let rejections = server.counters.snapshot().rejected; + assert_eq!(rejections, 0, "shutdown refusals are not back-pressure"); + } +} diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index b78ce7469c..f68195f40a 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -388,6 +388,9 @@ impl SharedHost { ca_private_key: Some(ca_private_key), directory: Some(trust_dir), }, + // The shared host has no per-instance collector endpoint yet, so + // hosted instances do not relay agent traces. + otlp_endpoint: None, }; crate::sandbox_instance_host::sandbox_instance_launch( Box::new(crate::backend_setup::RoutedBackendSetup::new( diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 7f16929759..f58651e129 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -487,6 +487,16 @@ A collector can accept a batch while rejecting some of its records, an OTLP part Metrics include `openshell_ocsf_otlp_queued_total`, `openshell_ocsf_otlp_exported_total`, `openshell_ocsf_otlp_dropped_total` (with a `reason` label of `queue_capacity`, `queue_bytes`, `oversized`, `export_failed`, `serialization`, or `closed`), `openshell_ocsf_otlp_export_errors_total`, and `openshell_ocsf_otlp_retries_total`. +### Agent trace relay + +With `[openshell.gateway.otlp].endpoint` set (Helm: `server.otlp.endpoint`), each sandbox also forwards the workload's own traces to the same collector. The sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, the traces-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` and `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, and `OTEL_EXPORTER_OTLP_COMPRESSION=none` (plus its traces-specific variant) in every workload process, after the environment from the sandbox creation request, so a standard OpenTelemetry SDK exports without configuration and a value in the creation request or the image cannot redirect traces around the relay. An environment passed to `openshell sandbox exec` is applied last and may override them for that command. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over OTLP/gRPC. The collector's address is never visible inside the sandbox. + +Every forwarded resource carries `openshell.sandbox.id`, with the same value as the supervisor's own spans, and `openshell.telemetry.source=agent`. Values the agent supplies for those two keys are replaced at the resource level; span and scope attributes pass through unchanged, so filter and join on the resource attributes. Filter on `openshell.telemetry.source` to separate agent traces from infrastructure traces, and join on `openshell.sandbox.id` to correlate them. + +Agents must export traces as OTLP/HTTP protobuf with compression disabled. The relay answers 415 to JSON bodies and to any `Content-Encoding`, 404 to the metrics and logs paths, 413 to bodies larger than 2 MiB, to requests with more than 512 `ResourceSpans` entries, and to requests whose resource attributes exceed 8192 in number or 256 KiB in total, 408 to requests whose body does not arrive within 30 seconds, and 503 with `Retry-After` while its buffer of 32 batches is full, which standard SDKs retry with backoff. Error responses carry a protobuf `google.rpc.Status` whose message states the reason. Without a configured collector, connections to the relay address are refused immediately. The relay keeps serving `openshell sandbox exec` sessions after the main process exits and flushes once more when the sandbox is torn down. + +The relayed hop does not yet support TLS or authentication to the collector; an `https` endpoint leaves the relay inactive with a logged reason. Protect the hop at the network layer, for example with a collector-side ingress policy. + ### Tuning `[openshell.gateway.otlp]` decides whether and where to export. How the SDK exports is controlled by the standard OpenTelemetry environment variables, which the gateway reads through the SDK rather than mirroring as TOML keys. They apply to the gateway and its in-process compute drivers. The gateway does not pass them to sandbox supervisors, which receive only the endpoints, the OCSF schema version, and the [OCSF collector headers](#ocsf-log-export). diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index d9a6e99d03..6a7754a4c5 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -30,6 +36,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -369,8 +384,12 @@ version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ + "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", + "windows-link", ] [[package]] @@ -437,11 +456,20 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-queue" -version = "0.3.14" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -690,6 +718,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "flume" version = "0.12.0" @@ -742,6 +781,7 @@ checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -1066,9 +1106,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.15" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -1193,6 +1233,30 @@ dependencies = [ "tower-service", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -1546,9 +1610,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libsqlite3-sys" -version = "0.37.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", @@ -1582,6 +1646,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -1604,6 +1677,16 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + [[package]] name = "mime" version = "0.3.17" @@ -1616,6 +1699,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -1664,6 +1757,15 @@ dependencies = [ "smallvec", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -1746,6 +1848,8 @@ dependencies = [ "nix", "noyalib", "openshell-crypto", + "openshell-otel-test-support", + "opentelemetry-proto", "prost", "rand 0.9.5", "rustls", @@ -1765,12 +1869,122 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-ocsf" +version = "0.0.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "serde_repr", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "openshell-otel" +version = "0.0.0" +dependencies = [ + "futures", + "http", + "metrics", + "openshell-ocsf", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", + "serde_json", + "thiserror", + "tokio", + "tonic", + "tower-http", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", +] + +[[package]] +name = "openshell-otel-test-support" +version = "0.0.0" +dependencies = [ + "openshell-otel", + "opentelemetry-proto", + "prost", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-subscriber", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +dependencies = [ + "http", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "thiserror", + "tokio", + "tonic", + "tonic-types", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -1909,6 +2123,12 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "potential_utf" version = "0.1.5" @@ -1965,6 +2185,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.47" @@ -2041,6 +2270,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + [[package]] name = "rcgen" version = "0.14.10" @@ -2444,6 +2682,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2469,6 +2716,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -2788,6 +3041,15 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "time" version = "0.3.54" @@ -2908,6 +3170,7 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", + "flate2", "h2", "http", "http-body", @@ -2917,6 +3180,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", + "rustls-native-certs", "socket2", "sync_wrapper", "tokio", @@ -2939,6 +3203,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -2974,6 +3249,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "uuid", ] [[package]] @@ -3018,6 +3294,64 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +dependencies = [ + "js-sys", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -3101,6 +3435,23 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3182,6 +3533,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "1.0.10" @@ -3219,12 +3580,65 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -3464,6 +3878,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 1dc6f0b1cf..02153aef44 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -248,8 +248,15 @@ name = "gpu" path = "tests/gpu.rs" required-features = ["e2e-gpu"] +[[test]] +name = "otlp_relay" +path = "tests/otlp_relay.rs" +required-features = ["e2e-local-container-driver"] + [dependencies] openshell-crypto = { path = "../../crates/openshell-crypto" } +openshell-otel-test-support = { path = "../../crates/openshell-otel-test-support" } +opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] } base64 = "0.22" bollard = "0.20" bytes = "1" diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index 6502fb7de2..c5c89a7773 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -18,6 +18,12 @@ RUN_WITH_GATEWAY_COMMAND="__openshell_run_podman_e2e" # shellcheck source=e2e/support/conformance.sh source "${ROOT}/e2e/support/conformance.sh" +# Enable OTLP export in the generated gateway config so the otlp_relay test +# can bind a collector stub on this port. Containers reach the host through +# host.containers.internal. Set the variable to an empty string to disable +# export and run the relay's inert case instead. +export OPENSHELL_E2E_OTLP_ENDPOINT="${OPENSHELL_E2E_OTLP_ENDPOINT-http://host.containers.internal:4317}" + # Keep required branch CI on the Podman targets that are known to pass. The # unfiltered e2e:podman task remains available while the omitted targets are # stabilized and can be added here. @@ -37,6 +43,7 @@ PODMAN_CI_TESTS=( live_policy_update local_driver_token_restart no_proxy + otlp_relay podman_corporate_proxy podman_gateway_start podman_host_gateway diff --git a/e2e/rust/tests/otlp_relay.rs b/e2e/rust/tests/otlp_relay.rs new file mode 100644 index 0000000000..e8625aa7f0 --- /dev/null +++ b/e2e/rust/tests/otlp_relay.rs @@ -0,0 +1,410 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor-direct OTLP trace relay. +//! +//! The gateway wrapper writes `[openshell.gateway.otlp].endpoint` from +//! `OPENSHELL_E2E_OTLP_ENDPOINT`. These tests bind a collector stub on that +//! endpoint's port and prove that a span exported from inside a sandbox +//! arrives with the supervisor's attribution. With the variable unset or +//! empty, the inert case runs instead. + +#![cfg(feature = "e2e-local-container-driver")] + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::time::Duration; + +use base64::Engine as _; +use openshell_e2e::harness::cli; +use openshell_e2e::harness::sandbox::SandboxGuard; +use openshell_otel_test_support::OtlpTestServer; +use openshell_otel_test_support::relay::{ + SANDBOX_ID_KEY, SOURCE_KEY, SOURCE_VALUE, encoded_trace_request, +}; +use serial_test::serial; + +const ENV_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; +const ENV_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL"; +const ENV_COLLECTOR: &str = "OPENSHELL_OTLP_ENDPOINT"; + +/// Posts the base64 body from `OTLP_BODY_B64` to the relay and prints the +/// status plus the complete environment, one `ENV key=value` line each, so +/// the test can check values and prove the collector address is nowhere. +const EXPORT_SCRIPT: &str = r#" +import base64, os, requests +body = base64.b64decode(os.environ["OTLP_BODY_B64"]) +url = os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] + "/v1/traces" +response = requests.post(url, data=body, headers={"Content-Type": "application/x-protobuf"}, timeout=10) +print("STATUS", response.status_code) +for key, value in sorted(os.environ.items()): + if key != "OTLP_BODY_B64": + print("ENV " + key + "=" + value) +"#; + +/// Main-process variant: the payload is embedded so no `--env` is needed, +/// and the process exits the moment the POST returns. +fn exit_script(body_b64: &str) -> String { + format!( + r#" +import base64, os, requests +body = base64.b64decode("{body_b64}") +url = os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] + "/v1/traces" +response = requests.post(url, data=body, headers={{"Content-Type": "application/x-protobuf"}}, timeout=10) +raise SystemExit(0 if response.status_code == 200 else 1) +"# + ) +} + +/// Same export, but reports the failure class instead of raising, for the +/// inert case. +const INERT_SCRIPT: &str = r#" +import os, requests +url = os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] + "/v1/traces" +try: + response = requests.post(url, data=b"", headers={"Content-Type": "application/x-protobuf"}, timeout=3) + print("STATUS", response.status_code) +except Exception as error: + print("ERROR", type(error).__name__) +"#; + +fn collector_endpoint() -> Option { + std::env::var("OPENSHELL_E2E_OTLP_ENDPOINT") + .ok() + .filter(|value| !value.is_empty()) +} + +fn collector_bind_addr(endpoint: &str) -> SocketAddr { + let url = url::Url::parse(endpoint).expect("OPENSHELL_E2E_OTLP_ENDPOINT is a URL"); + let port = url + .port_or_known_default() + .expect("OPENSHELL_E2E_OTLP_ENDPOINT names a port"); + // Containers reach the host through its bridge address, not loopback. + SocketAddr::from(([0, 0, 0, 0], port)) +} + +/// `host:port` of the collector as the gateway configures it, the string +/// that must not appear anywhere in the sandbox environment. +fn collector_host_port(endpoint: &str) -> String { + let url = url::Url::parse(endpoint).expect("OPENSHELL_E2E_OTLP_ENDPOINT is a URL"); + format!( + "{}:{}", + url.host_str().expect("collector host"), + url.port_or_known_default().expect("collector port") + ) +} + +/// One span with the agent's own service name and spoofed values for both +/// attribution keys, base64-encoded for the workload's environment. +fn encoded_request(span_name: &str) -> String { + let request = encoded_trace_request( + span_name, + &[ + ("service.name", "e2e-agent"), + (SANDBOX_ID_KEY, "spoofed"), + (SOURCE_KEY, "infrastructure"), + ], + ); + base64::engine::general_purpose::STANDARD.encode(request) +} + +/// The harness CLI runner with a `Result` for the failure path. +async fn run_cli(args: &[&str]) -> Result { + let (output, code) = cli::run_cli(args).await; + if code != 0 { + return Err(format!( + "openshell {} failed (exit {code}):\n{output}", + args.join(" ") + )); + } + Ok(output) +} + +async fn sandbox_id(name: &str) -> Result { + let details = run_cli(&["sandbox", "get", name, "--output", "json"]).await?; + let details: serde_json::Value = + serde_json::from_str(&details).map_err(|e| format!("parse sandbox JSON: {e}"))?; + details["id"] + .as_str() + .map(str::to_string) + .ok_or_else(|| format!("sandbox has no id: {details}")) +} + +/// Polls `sandbox get` until the gateway no longer knows the sandbox, so +/// the next create does not overlap the previous teardown. +async fn wait_for_sandbox_gone(name: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while run_cli(&["sandbox", "get", name]).await.is_ok() { + if tokio::time::Instant::now() >= deadline { + eprintln!("sandbox {name} still listed after delete; continuing"); + return; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// Waits for the sandbox to reach `phase` through the harness, then returns +/// its details for further assertions. +async fn wait_for_sandbox_phase(name: &str, phase: &str) -> Result { + cli::wait_for_sandbox_phase(name, phase, Duration::from_secs(60)).await?; + run_cli(&["sandbox", "get", name]).await +} + +async fn wait_for_sandbox_logs( + sandbox_name: &str, + expected: impl Fn(&str) -> bool, +) -> Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + loop { + let logs = run_cli(&[ + "logs", + sandbox_name, + "-n", + "500", + "--since", + "5m", + "--source", + "sandbox", + ]) + .await?; + if expected(&logs) { + return Ok(logs); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "timed out waiting for expected sandbox logs:\n{logs}" + )); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// Waits up to `within` until the collector holds every span in `names`, +/// returning the resource attributes keyed by span name. +async fn wait_for_spans( + collector: &OtlpTestServer, + names: &[&str], + within: Duration, +) -> Result>, String> { + let deadline = tokio::time::Instant::now() + within; + loop { + let received = collector.received(); + let received_names: Vec<&str> = received + .spans + .iter() + .map(|span| span.name.as_str()) + .collect(); + if names.iter().all(|name| received_names.contains(name)) { + // Supervisor batches share the collector; `span_resources` pairs + // every span with the resource of the batch it arrived in. + let by_name = received + .spans + .iter() + .zip(received.span_resources.iter()) + .map(|(span, attributes)| (span.name.clone(), attributes.clone())) + .collect(); + return Ok(by_name); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "collector received {received_names:?}, expected all of {names:?}" + )); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// The collector keeps the last value per key, so a spoofed attribute that +/// was appended to rather than replaced is only visible as a duplicate. +fn assert_no_duplicate_resource_keys(collector: &OtlpTestServer) { + let duplicates = collector.received().duplicate_resource_keys; + assert!( + duplicates.is_empty(), + "attribution keys were appended, not replaced: {duplicates:?}" + ); +} + +fn assert_attribution(attributes: &HashMap, sandbox_id: &str) { + assert!( + !attributes + .values() + .any(|value| value == "spoofed" || value == "infrastructure"), + "agent-supplied attribution values must be discarded: {attributes:?}" + ); + assert_eq!( + attributes.get("service.name").map(String::as_str), + Some("e2e-agent"), + "agent service name is preserved: {attributes:?}" + ); + assert_eq!( + attributes.get(SOURCE_KEY).map(String::as_str), + Some(SOURCE_VALUE), + "{attributes:?}" + ); + assert_eq!( + attributes.get(SANDBOX_ID_KEY).map(String::as_str), + Some(sandbox_id), + "spoofed sandbox id is replaced: {attributes:?}" + ); +} + +#[tokio::test] +#[serial] +async fn agent_span_reaches_collector_with_attribution() { + let Some(endpoint) = collector_endpoint() else { + eprintln!("OPENSHELL_E2E_OTLP_ENDPOINT is unset; skipping the collector case"); + return; + }; + let collector = OtlpTestServer::start_on(collector_bind_addr(&endpoint)).await; + + let body = format!("OTLP_BODY_B64={}", encoded_request("e2e-span")); + let mut sandbox = SandboxGuard::create(&[ + "--env", + &body, + "--no-auto-providers", + "--", + "python3", + "-c", + EXPORT_SCRIPT, + ]) + .await + .expect("sandbox exports one span to the relay"); + let output = sandbox.create_output.clone(); + assert!( + output.contains("STATUS 200"), + "relay must accept the export:\n{output}" + ); + // US1-S3: the exact values a standard SDK needs. + assert!( + output.contains(&format!("ENV {ENV_ENDPOINT}=http://192.0.0.8:4318")), + "the relay endpoint is set for the workload:\n{output}" + ); + assert!( + output.contains(&format!("ENV {ENV_PROTOCOL}=http/protobuf")), + "the exporter protocol is set for the workload:\n{output}" + ); + // US1-S4: neither the variable nor the address itself is present anywhere. + assert!( + !output.contains(ENV_COLLECTOR), + "the collector variable must not be visible inside the sandbox:\n{output}" + ); + let host_port = collector_host_port(&endpoint); + assert!( + !output.contains(&host_port), + "the collector address {host_port} must not appear in the sandbox environment:\n{output}" + ); + + let id = sandbox_id(&sandbox.name).await.expect("sandbox id"); + // US1-S1: within 5 seconds. + let spans = wait_for_spans(&collector, &["e2e-span"], Duration::from_secs(5)) + .await + .expect("span reaches the collector stub within 5 s"); + assert_no_duplicate_resource_keys(&collector); + assert_attribution(&spans["e2e-span"], &id); + + sandbox.cleanup().await; + drop(collector.shutdown().await); +} + +#[tokio::test] +#[serial] +async fn relay_is_inert_without_collector() { + if collector_endpoint().is_some() { + eprintln!("OPENSHELL_E2E_OTLP_ENDPOINT is set; skipping the inert case"); + return; + } + + let mut sandbox = + SandboxGuard::create(&["--no-auto-providers", "--", "python3", "-c", INERT_SCRIPT]) + .await + .expect("sandbox attempts an export without a collector"); + let output = sandbox.create_output.clone(); + assert!( + output.contains("ERROR ConnectionError"), + "the export must fail with a connection error, not a timeout:\n{output}" + ); + + let logs = wait_for_sandbox_logs(&sandbox.name, |logs| { + logs.contains("OTLP agent trace relay inactive") + }) + .await + .expect("supervisor reports the relay as inactive"); + assert!(!logs.contains("OTLP agent trace relay enabled"), "{logs}"); + + sandbox.cleanup().await; +} + +#[tokio::test] +#[serial] +async fn span_emitted_before_exit_is_delivered() { + let Some(endpoint) = collector_endpoint() else { + eprintln!("OPENSHELL_E2E_OTLP_ENDPOINT is unset; skipping the flush case"); + return; + }; + let collector = OtlpTestServer::start_on(collector_bind_addr(&endpoint)).await; + + // The main process posts and exits at once; the supervisor must flush + // the batch before it reports the exit. Five cycles stand in for SC-005. + let names: Vec = (1..=5).map(|i| format!("exit-span-{i}")).collect(); + let mut ids = HashMap::new(); + for name in &names { + let script = exit_script(&encoded_request(name)); + let mut sandbox = SandboxGuard::create_detached_main(&["python3", "-c", &script]) + .await + .expect("sandbox whose main process exports one span and exits"); + let details = match wait_for_sandbox_phase(&sandbox.name, "Completed").await { + Ok(details) => details, + Err(error) => { + let logs = run_cli(&[ + "logs", + &sandbox.name, + "-n", + "500", + "--since", + "5m", + "--source", + "sandbox", + ]) + .await + .unwrap_or_else(|e| format!("(logs unavailable: {e})")); + sandbox.cleanup().await; + panic!( + "main process did not exit after the export: {error}\nsupervisor logs:\n{logs}" + ); + } + }; + assert!( + !details.contains("Exit Code: 1"), + "relay must accept the export:\n{details}" + ); + // US4-S1: the gateway marks the sandbox Completed only after the + // supervisor's exit report, and the supervisor flushes before that + // report, so the span must already be at the collector now. + let snapshot = collector.received(); + assert!( + snapshot.spans.iter().any(|span| &span.name == name), + "{name} must reach the collector before the exit is reported; collector holds {:?}", + snapshot + .spans + .iter() + .map(|span| span.name.as_str()) + .collect::>() + ); + ids.insert( + name.clone(), + sandbox_id(&sandbox.name).await.expect("sandbox id"), + ); + sandbox.cleanup().await; + wait_for_sandbox_gone(&sandbox.name).await; + } + + let name_refs: Vec<&str> = names.iter().map(String::as_str).collect(); + let spans = wait_for_spans(&collector, &name_refs, Duration::from_secs(15)) + .await + .expect("every span emitted right before exit reaches the collector"); + assert_no_duplicate_resource_keys(&collector); + for name in &names { + assert_attribution(&spans[name], &ids[name]); + } + drop(collector.shutdown().await); +} diff --git a/e2e/support/podman-gateway-config.sh b/e2e/support/podman-gateway-config.sh index ba425f91ec..a645aaf391 100755 --- a/e2e/support/podman-gateway-config.sh +++ b/e2e/support/podman-gateway-config.sh @@ -83,5 +83,12 @@ e2e_write_podman_gateway_config() { e2e_write_gateway_oidc_config "${oidc_issuer}" fi fi + # Opt-in OTLP export so the otlp_relay e2e can bind a collector stub on + # the referenced port. Supervisors connect lazily, so the gateway may + # start before the stub exists. + if [ -n "${OPENSHELL_E2E_OTLP_ENDPOINT:-}" ]; then + printf '\n[openshell.gateway.otlp]\n' + printf 'endpoint = %s\n' "$(e2e_podman_toml_string "${OPENSHELL_E2E_OTLP_ENDPOINT}")" + fi } >>"${output}" } diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 47bed1182d..2c9cb0978c 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -613,6 +613,13 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}" fi fi + # Opt-in OTLP export so the otlp_relay e2e can bind a collector stub on + # the referenced port. Supervisors connect lazily, so the gateway may + # start before the stub exists. + if [ -n "${OPENSHELL_E2E_OTLP_ENDPOINT:-}" ]; then + printf '[openshell.gateway.otlp]\n' + printf 'endpoint = %s\n\n' "$(toml_string "${OPENSHELL_E2E_OTLP_ENDPOINT}")" + fi printf '[openshell.drivers.docker]\n' if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 40351c998e..9ec64d915f 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1266,6 +1266,7 @@ credential failures. | Collector receives no OCSF log records, or `openshell.ocsf.gap` records | `[openshell.gateway.otlp.ocsf]` missing or without `enabled = true`, an unreachable or non-gRPC endpoint, or export failures and queue pressure | Check `[openshell.gateway.otlp.ocsf]` for `enabled = true` and the endpoint it resolves to, the gateway's `OTLP exporting enabled` startup log for `ocsf_endpoint`, export warnings, and `openshell_ocsf_otlp_*` metrics. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | Gateway OCSF JSONL stops growing, has gaps, or is rejected by a SIEM | File errors, queue pressure, shipper falling behind retention, or a schema mismatch | Inspect gateway warnings and `openshell_ocsf_log_*` metrics; check `[openshell.gateway.ocsf_log]`, `schema_version`, directory permissions, free space, per-replica paths, and shipper rotation checkpoints. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | +| Sandbox log shows `OTLP agent trace relay inactive: no collector endpoint configured` at startup, then `agent OTLP export attempted but no collector is configured` when the agent exports | The gateway has no `[openshell.gateway.otlp].endpoint`, so the supervisor refuses the workload's OTLP relay address | Set `endpoint` (Helm: `server.otlp.endpoint`) and recreate the sandbox; `openshell logs --source sandbox` then shows `OTLP agent trace relay enabled` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | | Docker or Podman sandbox never registers | Wrong gateway endpoint, unavailable host networking, or supervisor startup failure | Gateway logs and supervisor container logs | | CPU-only VM startup reports that read-only `/proc` cannot be removed | Older supervisors can infer GPU requirements from host devices | Check gateway, VM driver, and bundled supervisor versions; use matching updated artifacts. CPU-only VMs do not require read-write `/proc`. | diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 3ed9b09a0f..be2c54cd85 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -30,6 +36,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -369,8 +384,12 @@ version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ + "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", + "windows-link", ] [[package]] @@ -437,11 +456,20 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-queue" -version = "0.3.14" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -690,6 +718,17 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "flume" version = "0.12.0" @@ -742,6 +781,7 @@ checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -1066,9 +1106,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.15" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] @@ -1193,6 +1233,30 @@ dependencies = [ "tower-service", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.3.0" @@ -1547,9 +1611,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libsqlite3-sys" -version = "0.37.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", @@ -1583,6 +1647,15 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -1605,6 +1678,16 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + [[package]] name = "mime" version = "0.3.17" @@ -1617,6 +1700,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.3" @@ -1665,6 +1758,15 @@ dependencies = [ "smallvec", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -1758,6 +1860,8 @@ dependencies = [ "nix", "noyalib", "openshell-crypto", + "openshell-otel-test-support", + "opentelemetry-proto", "prost", "rand 0.9.5", "rustls", @@ -1776,6 +1880,54 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-ocsf" +version = "0.0.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "serde_repr", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "openshell-otel" +version = "0.0.0" +dependencies = [ + "futures", + "http", + "metrics", + "openshell-ocsf", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", + "serde_json", + "thiserror", + "tokio", + "tonic", + "tower-http", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", +] + +[[package]] +name = "openshell-otel-test-support" +version = "0.0.0" +dependencies = [ + "openshell-otel", + "opentelemetry-proto", + "prost", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-subscriber", +] + [[package]] name = "openshell-test-feature-oci-image" version = "0.0.0" @@ -1803,6 +1955,68 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" +dependencies = [ + "http", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "thiserror", + "tokio", + "tonic", + "tonic-types", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", + "tonic-prost", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -1941,6 +2155,12 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "potential_utf" version = "0.1.6" @@ -1997,6 +2217,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.47" @@ -2073,6 +2302,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + [[package]] name = "rcgen" version = "0.14.10" @@ -2451,6 +2689,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2476,6 +2723,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -2806,6 +3059,15 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "time" version = "0.3.55" @@ -2926,6 +3188,7 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", + "flate2", "h2", "http", "http-body", @@ -2935,6 +3198,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", + "rustls-native-certs", "socket2", "sync_wrapper", "tokio", @@ -2957,6 +3221,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -2992,6 +3267,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "uuid", ] [[package]] @@ -3036,6 +3312,64 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +dependencies = [ + "js-sys", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -3119,6 +3453,23 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3200,6 +3551,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "1.0.10" @@ -3237,12 +3598,65 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -3482,6 +3896,12 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.23" From b45d64f86a3692fb88d7bee591034fcc29a0e7b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20Hu=C3=9F?= Date: Wed, 7 Oct 2026 08:24:13 +0200 Subject: [PATCH 2/6] refactor(supervisor): let the OTLP relay exporter take a prepared channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expose Exporter::with_channel as the seam for a collector transport built once in openshell-otel and shared by the gateway, the supervisor's own spans, and the relay. Exporter::new keeps building the plaintext channel for now. Signed-off-by: Roland Huß --- .../src/otlp_relay/exporter.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/openshell-supervisor/src/otlp_relay/exporter.rs b/crates/openshell-supervisor/src/otlp_relay/exporter.rs index 41cfcb9e03..82eb45ffbb 100644 --- a/crates/openshell-supervisor/src/otlp_relay/exporter.rs +++ b/crates/openshell-supervisor/src/otlp_relay/exporter.rs @@ -95,10 +95,19 @@ impl Exporter { .http2_keep_alive_interval(HTTP2_PING_INTERVAL) .keep_alive_timeout(HTTP2_PING_TIMEOUT) .connect_lazy(); - Ok(Self { + Ok(Self::with_channel(channel, endpoint)) + } + + /// Forwards over a channel prepared elsewhere. This is the seam for a + /// shared collector transport (TLS, compression, headers) built once in + /// `openshell-otel` for the gateway, the supervisor's own spans, and this + /// relay; `endpoint` is only used for log messages. + #[must_use] + pub fn with_channel(channel: Channel, endpoint: &str) -> Self { + Self { client: Grpc::new(channel), endpoint: endpoint.to_string(), - }) + } } /// The collector URI this exporter targets. @@ -226,6 +235,22 @@ mod tests { assert!(Exporter::new("not a uri").is_err()); } + #[tokio::test] + async fn exports_over_a_channel_prepared_by_the_caller() { + let collector = OtlpTestServer::start().await; + let channel = Endpoint::from_shared(collector.endpoint().to_string()) + .unwrap() + .connect_lazy(); + let exporter = Exporter::with_channel(channel, collector.endpoint()); + exporter + .export(Bytes::from(encoded_request("via-channel", &[]))) + .await + .expect("export over the supplied channel"); + collector.wait_for_export().await; + let received = collector.shutdown().await; + assert!(received.spans.iter().any(|span| span.name == "via-channel")); + } + #[tokio::test] async fn export_reaches_the_collector_with_attributes_intact() { let collector = OtlpTestServer::start().await; From ede906dabd113f57fb5f69a5ac78031d54191999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20Hu=C3=9F?= Date: Wed, 7 Oct 2026 08:25:12 +0200 Subject: [PATCH 3/6] feat(sandbox): leave caller-set OTLP endpoints alone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inject the relay variables only when the sandbox creation request names no OTLP endpoint, generic or traces-specific. A caller who names one has chosen where the agent exports; overwriting it would silently redirect an agent that deliberately exports elsewhere. A protocol or compression setting alone does not suppress injection. Applies to both spawn paths and to exec children, with the docs updated to match. Signed-off-by: Roland Huß --- crates/openshell-core/src/sandbox_env.rs | 9 +- crates/openshell-sandbox/src/boundary_exec.rs | 11 +- crates/openshell-sandbox/src/child_env.rs | 68 +++++- crates/openshell-sandbox/src/process.rs | 197 ++++++++++++------ docs/how-it-works/gateways/configuration.mdx | 2 +- 5 files changed, 200 insertions(+), 87 deletions(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index e5d212a135..daada1e12f 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -302,10 +302,11 @@ pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER"; /// Standard OpenTelemetry environment variable for the OTLP exporter endpoint. /// /// The sandbox runtime sets it in every workload process to -/// [`OTLP_RELAY_ENDPOINT`], after the environment from the sandbox creation -/// request, so agent SDKs export to the supervisor's relay and a value in the -/// creation request never wins. A per-exec environment is a developer's -/// explicit instruction and still overrides it for that exec only. +/// [`OTLP_RELAY_ENDPOINT`] so agent SDKs export to the supervisor's relay, +/// unless the sandbox creation request already names an OTLP endpoint +/// (generic or traces-specific), in which case the caller's variables are +/// left untouched. A per-exec environment is a developer's explicit +/// instruction and still overrides it for that exec only. pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; /// Standard OpenTelemetry environment variable for the OTLP exporter protocol. diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index 1da8e6adf1..5f6ff12c2d 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -184,10 +184,15 @@ impl LocalBoundaryExec { crate::process::strip_proxy_env_std(&mut command); // Point OTel SDKs at the supervisor's trace relay, after the creation // request's environment and the provider variables, matching the main - // process. A per-exec `spec.env` value applied below still wins for + // process, and only when the creation request named no endpoint of + // its own. A per-exec `spec.env` value applied below still wins for // that exec only. - for (key, value) in crate::child_env::otlp_relay_env_vars() { - command.env(key, value); + if let Some(relay_env) = + crate::child_env::otlp_relay_env_vars_unless_configured(&self.user_environment) + { + for (key, value) in relay_env { + command.env(key, value); + } } for (key, value) in &spec.env { if !key.starts_with(crate::process::RESERVED_ENV_PREFIX) { diff --git a/crates/openshell-sandbox/src/child_env.rs b/crates/openshell-sandbox/src/child_env.rs index 4302482c8a..abf0fcb029 100644 --- a/crates/openshell-sandbox/src/child_env.rs +++ b/crates/openshell-sandbox/src/child_env.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::collections::HashMap; +use std::hash::BuildHasher; use std::path::Path; use openshell_core::sandbox_env::{ @@ -9,17 +11,16 @@ use openshell_core::sandbox_env::{ OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, OTLP_RELAY_ENDPOINT, OTLP_RELAY_TRACES_ENDPOINT, }; -/// OpenTelemetry exporter variables every workload process receives. +/// OpenTelemetry exporter variables a workload process receives when the +/// sandbox creation request names no OTLP endpoint of its own. /// -/// They point agent SDKs at the supervisor's OTLP relay and are applied after -/// the environment from the sandbox creation request, so an endpoint named -/// there never wins: it would either point into the fenced network and fail -/// or bypass attribution. The signal-specific traces variables are set as -/// well because SDKs give them precedence over the generic ones, and -/// compression is disabled because the relay refuses compressed bodies. The -/// values are the same whether or not a collector is configured; without one -/// the relay address refuses connections immediately. A per-exec environment -/// still overrides them for that exec only. +/// They point agent SDKs at the supervisor's OTLP relay. The signal-specific +/// traces variables are set as well because SDKs give them precedence over +/// the generic ones, and compression is disabled because the relay refuses +/// compressed bodies. The values are the same whether or not a collector is +/// configured; without one the relay address refuses connections +/// immediately. Use [`otlp_relay_env_vars_unless_configured`] to respect a +/// caller who deliberately exports elsewhere. pub fn otlp_relay_env_vars() -> [(&'static str, &'static str); 6] { [ (OTEL_EXPORTER_OTLP_ENDPOINT, OTLP_RELAY_ENDPOINT), @@ -34,6 +35,29 @@ pub fn otlp_relay_env_vars() -> [(&'static str, &'static str); 6] { ] } +/// Whether a sandbox creation environment names an OTLP endpoint, generic or +/// traces-specific. A caller who does has chosen where the agent exports. +pub fn names_otlp_endpoint(user_environment: &HashMap) -> bool { + [ + OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + ] + .iter() + .any(|key| user_environment.contains_key(*key)) +} + +/// [`otlp_relay_env_vars`] unless the creation request named an endpoint. +/// +/// Returns `None` when the caller named one, and then none of the caller's +/// `OTEL_EXPORTER_OTLP_*` values are touched: the agent exports where it was +/// told to, subject to the sandbox's network policy and outside relay +/// attribution, instead of being redirected to the relay. +pub fn otlp_relay_env_vars_unless_configured( + user_environment: &HashMap, +) -> Option<[(&'static str, &'static str); 6]> { + (!names_otlp_endpoint(user_environment)).then(otlp_relay_env_vars) +} + pub fn tls_env_vars( ca_cert_path: &Path, combined_bundle_path: &Path, @@ -99,4 +123,28 @@ mod tests { ] ); } + + #[test] + fn a_caller_named_endpoint_disables_relay_injection() { + let none = HashMap::new(); + assert!(otlp_relay_env_vars_unless_configured(&none).is_some()); + + // Protocol or compression alone do not name a destination. + let protocol_only = HashMap::from([( + "OTEL_EXPORTER_OTLP_PROTOCOL".to_string(), + "grpc".to_string(), + )]); + assert!(otlp_relay_env_vars_unless_configured(&protocol_only).is_some()); + + for key in [ + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + ] { + let named = HashMap::from([(key.to_string(), "http://collector.example:4318".into())]); + assert!( + otlp_relay_env_vars_unless_configured(&named).is_none(), + "{key} names a destination" + ); + } + } } diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 430fd78b7e..093ba19c54 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -274,7 +274,8 @@ pub(crate) fn configured_user_environment() -> HashMap { /// sandbox environment minus the reserved `OPENSHELL_` namespace, the /// declared user environment, supervisor-only and proxy variables stripped, /// provider placeholders, TLS trust paths, and finally the OTLP relay -/// endpoint, applied last so a caller-supplied endpoint never wins. +/// variables, applied last so a provider cannot redirect traces, and only +/// when the declared environment names no OTLP endpoint of its own. fn compose_canonical_process_environment( cmd: &mut Command, policy: &SandboxPolicy, @@ -294,8 +295,10 @@ fn compose_canonical_process_environment( cmd.env(key, value); } } - for (key, value) in child_env::otlp_relay_env_vars() { - cmd.env(key, value); + if let Some(relay_env) = child_env::otlp_relay_env_vars_unless_configured(user_environment) { + for (key, value) in relay_env { + cmd.env(key, value); + } } } @@ -1313,11 +1316,13 @@ mod tests { } } - /// The production composition applies the relay endpoint after the - /// declared environment and strips the collector address. Runs in a - /// re-executed copy of the test binary so the collector address can sit - /// in the inherited environment, where `OPENSHELL_OTLP_ENDPOINT` lives - /// for a real supervisor, without touching the harness environment. + /// The production composition injects the relay variables when the + /// declared environment names no OTLP endpoint, leaves a caller's own + /// endpoint alone otherwise, and strips the collector address in both + /// cases. Runs in a re-executed copy of the test binary so the collector + /// address can sit in the inherited environment, where + /// `OPENSHELL_OTLP_ENDPOINT` lives for a real supervisor, without touching + /// the harness environment. #[cfg(unix)] #[test] fn canonical_process_sees_relay_endpoint_and_never_the_collector() { @@ -1337,87 +1342,141 @@ mod tests { run_as_user: Some(current_user.name), run_as_group: None, }); - // Mirror production: inherit the sandbox environment, no env_clear. - let mut cmd = Command::new("/usr/bin/env"); - cmd.stdout(StdStdio::piped()); - let declared = HashMap::from([ - ( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.into(), - "http://example.invalid".into(), - ), - ( + let workspace = ResolvedWorkspace::default(); + let probe = |declared: HashMap, provider_env: HashMap| { + // Mirror production: inherit the sandbox environment, no env_clear. + let mut cmd = Command::new("/usr/bin/env"); + cmd.stdout(StdStdio::piped()); + compose_canonical_process_environment( + &mut cmd, + &policy, + &workspace, + false, + &declared, + &provider_env, + None, + ); + let variables = probe_environment(cmd); + assert!( + !variables.contains_key(openshell_core::sandbox_env::OTLP_ENDPOINT), + "the collector variable reached the workload: {variables:?}" + ); + assert!( + !variables + .values() + .any(|value| value.contains(COLLECTOR_HOST)), + "the collector address reached the workload: {variables:?}" + ); + variables + }; + let get = |variables: &HashMap, key: &str| variables.get(key).cloned(); + + // No endpoint in the creation request: the relay variables are set, + // and a provider cannot redirect traces either. + let variables = probe( + HashMap::from([( openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL.into(), "grpc".into(), - ), - // SDKs prefer the signal-specific variables; a declared one must - // not redirect traces around the relay either. - ( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.into(), - "http://example.invalid/v1/traces".into(), - ), - ( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION.into(), - "gzip".into(), - ), - ]); - // A provider could also try to name an endpoint; the relay must win - // over that layer too. - let provider_env = HashMap::from([( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.to_string(), - "http://provider.invalid".to_string(), - )]); - compose_canonical_process_environment( - &mut cmd, - &policy, - &ResolvedWorkspace::default(), - false, - &declared, - &provider_env, - None, + )]), + HashMap::from([( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.to_string(), + "http://provider.invalid".to_string(), + )]), ); - let variables = probe_environment(cmd); assert_eq!( - variables - .get(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT) - .map(String::as_str), + get( + &variables, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT + ) + .as_deref(), Some(openshell_core::sandbox_env::OTLP_RELAY_ENDPOINT), - "the relay endpoint wins over the declared one: {variables:?}" + "the relay endpoint wins over a provider's: {variables:?}" ); assert_eq!( - variables - .get(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL) - .map(String::as_str), - Some("http/protobuf") + get( + &variables, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL + ) + .as_deref(), + Some("http/protobuf"), + "a declared protocol without an endpoint is replaced: {variables:?}" ); assert_eq!( - variables - .get(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) - .map(String::as_str), + get( + &variables, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + ) + .as_deref(), Some(openshell_core::sandbox_env::OTLP_RELAY_TRACES_ENDPOINT), - "the traces-specific endpoint is forced too: {variables:?}" + "the traces-specific endpoint is set too: {variables:?}" ); assert_eq!( - variables - .get(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION) - .map(String::as_str), + get( + &variables, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION + ) + .as_deref(), Some("none") ); - assert!( - !variables.contains_key(openshell_core::sandbox_env::OTLP_ENDPOINT), - "the collector variable reached the workload: {variables:?}" - ); - assert!( - !variables - .values() - .any(|value| value.contains(COLLECTOR_HOST)), - "the collector address reached the workload: {variables:?}" - ); assert!( !variables .values() .any(|value| value.contains("provider.invalid")), "a provider endpoint overrode the relay: {variables:?}" ); + + // The creation request names its own endpoint: every caller value + // stays as declared and nothing points at the relay. + let variables = probe( + HashMap::from([ + ( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.into(), + "http://own-backend.invalid/v1/traces".into(), + ), + ( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL.into(), + "grpc".into(), + ), + ( + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION.into(), + "gzip".into(), + ), + ]), + HashMap::new(), + ); + assert_eq!( + get( + &variables, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + ) + .as_deref(), + Some("http://own-backend.invalid/v1/traces"), + "a caller's endpoint is left alone: {variables:?}" + ); + assert_eq!( + get( + &variables, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL + ) + .as_deref(), + Some("grpc") + ); + assert_eq!( + get( + &variables, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION + ) + .as_deref(), + Some("gzip") + ); + assert!( + !variables.contains_key(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT), + "nothing is injected when the caller named an endpoint: {variables:?}" + ); + assert!( + !variables.values().any(|value| value.contains("192.0.0.8")), + "the relay address must not appear: {variables:?}" + ); } #[cfg(unix)] diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index f58651e129..314c3542eb 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -489,7 +489,7 @@ Metrics include `openshell_ocsf_otlp_queued_total`, `openshell_ocsf_otlp_exporte ### Agent trace relay -With `[openshell.gateway.otlp].endpoint` set (Helm: `server.otlp.endpoint`), each sandbox also forwards the workload's own traces to the same collector. The sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, the traces-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` and `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, and `OTEL_EXPORTER_OTLP_COMPRESSION=none` (plus its traces-specific variant) in every workload process, after the environment from the sandbox creation request, so a standard OpenTelemetry SDK exports without configuration and a value in the creation request or the image cannot redirect traces around the relay. An environment passed to `openshell sandbox exec` is applied last and may override them for that command. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over OTLP/gRPC. The collector's address is never visible inside the sandbox. +With `[openshell.gateway.otlp].endpoint` set (Helm: `server.otlp.endpoint`), each sandbox also forwards the workload's own traces to the same collector. The sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, the traces-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` and `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, and `OTEL_EXPORTER_OTLP_COMPRESSION=none` (plus its traces-specific variant) in every workload process, so a standard OpenTelemetry SDK exports without configuration and an image cannot redirect traces around the relay. If the sandbox creation request names its own `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, nothing is injected and the caller's OpenTelemetry variables stand: that agent exports where it was told to, subject to the sandbox's network policy, and its traces carry no relay attribution. An environment passed to `openshell sandbox exec` is applied last and may override the injected values for that command. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over OTLP/gRPC. The collector's address is never visible inside the sandbox. Every forwarded resource carries `openshell.sandbox.id`, with the same value as the supervisor's own spans, and `openshell.telemetry.source=agent`. Values the agent supplies for those two keys are replaced at the resource level; span and scope attributes pass through unchanged, so filter and join on the resource attributes. Filter on `openshell.telemetry.source` to separate agent traces from infrastructure traces, and join on `openshell.sandbox.id` to correlate them. From 75378225e93ca26c54eb73378f0caa36292dee81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20Hu=C3=9F?= Date: Fri, 16 Oct 2026 08:13:19 +0200 Subject: [PATCH 4/6] fix(supervisor): address OTLP relay review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inject only the generic OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_EXPORTER_OTLP_PROTOCOL as lowest-precedence defaults, and only while the relay is active. The supervisor now starts the relay before attach and signals it to the sandbox runtime through AgentSpec::otlp_relay_active. An endpoint in the creation request, the image ENV, or an exec environment suppresses the defaults, and a provider-supplied traces endpoint is removed. Harden the relay itself: - skip the exit flush and shutdown wait when the last export failed - count batches still pending on Drop as discarded and log them - validate the collector endpoint with the shared openshell-otel helper - accept Content-Encoding: identity - fix the 413 and shutdown 503 messages and the memory bound docs - narrow the attribution claim to resource attributes Document best-effort delivery and the 404 for metrics and logs. Signed-off-by: Roland Huß --- crates/openshell-core/src/sandbox_env.rs | 39 +--- .../openshell-isolation-interface/src/lib.rs | 3 + crates/openshell-otel/src/lib.rs | 5 +- .../src/boundary_protocol.rs | 31 +++ .../openshell-sandbox-backend/src/protobuf.rs | 20 ++ .../openshell-sandbox-backend/src/runtime.rs | 1 + crates/openshell-sandbox/src/boundary_exec.rs | 78 +++++-- .../openshell-sandbox/src/boundary_server.rs | 4 + crates/openshell-sandbox/src/child_env.rs | 163 +++++++++----- crates/openshell-sandbox/src/delegated.rs | 4 + crates/openshell-sandbox/src/process.rs | 206 +++++++++++------- crates/openshell-supervisor/README.md | 2 +- .../src/backend_setup/tests.rs | 1 + crates/openshell-supervisor/src/lib.rs | 13 +- .../src/otlp_relay/enrichment.rs | 7 +- .../src/otlp_relay/mod.rs | 194 ++++++++++++++++- .../src/otlp_relay/receiver.rs | 43 +++- docs/how-it-works/gateways/configuration.mdx | 6 +- e2e/rust/tests/otlp_relay.rs | 12 +- proto/sandbox_protocol.proto | 4 + 20 files changed, 635 insertions(+), 201 deletions(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index daada1e12f..6aa203deda 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -301,12 +301,12 @@ pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER"; /// Standard OpenTelemetry environment variable for the OTLP exporter endpoint. /// -/// The sandbox runtime sets it in every workload process to -/// [`OTLP_RELAY_ENDPOINT`] so agent SDKs export to the supervisor's relay, -/// unless the sandbox creation request already names an OTLP endpoint -/// (generic or traces-specific), in which case the caller's variables are -/// left untouched. A per-exec environment is a developer's explicit -/// instruction and still overrides it for that exec only. +/// While the supervisor serves the OTLP relay, the sandbox runtime sets it in +/// workload processes to [`OTLP_RELAY_ENDPOINT`] as a lowest-precedence +/// default, so agent SDKs export to the relay without configuration. It is +/// not set when the creation request, the image, or a per-exec environment +/// names an endpoint, generic or traces-specific, and any signal-specific +/// endpoint the workload sets later takes precedence over it in the SDK. pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; /// Standard OpenTelemetry environment variable for the OTLP exporter protocol. @@ -314,23 +314,10 @@ pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT"; /// Set to `http/protobuf` alongside [`OTEL_EXPORTER_OTLP_ENDPOINT`]. pub const OTEL_EXPORTER_OTLP_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL"; -/// Signal-specific endpoint variable. SDKs give it precedence over the -/// generic endpoint, so it is set too (to the full traces URL) or an image -/// that ships one would silently bypass the relay. +/// Signal-specific endpoint variable. Never injected; a layer that sets it +/// has chosen where traces go, so the relay default is left out. pub const OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"; -/// Signal-specific protocol variable, set alongside -/// [`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`] for the same reason. -pub const OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"; - -/// Standard OTLP compression variable. The relay refuses compressed bodies, -/// so both this and the traces-specific variant are set to `none`. -pub const OTEL_EXPORTER_OTLP_COMPRESSION: &str = "OTEL_EXPORTER_OTLP_COMPRESSION"; - -/// Signal-specific compression variable, see -/// [`OTEL_EXPORTER_OTLP_COMPRESSION`]. -pub const OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: &str = "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION"; - /// Reserved destination agent processes export OTLP to. /// /// This address is never routed. A workload `connect()` to any non-loopback @@ -348,10 +335,6 @@ pub const OTLP_RELAY_ADDR: &str = "192.0.0.8:4318"; /// [`OTEL_EXPORTER_OTLP_ENDPOINT`]. pub const OTLP_RELAY_ENDPOINT: &str = "http://192.0.0.8:4318"; -/// The traces URL injected through [`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`], -/// which unlike the generic endpoint names the full path. -pub const OTLP_RELAY_TRACES_ENDPOINT: &str = "http://192.0.0.8:4318/v1/traces"; - // The corporate upstream-proxy configuration deliberately has no reserved // environment variables: it travels on the supervisor's argv // (`--upstream-proxy` and friends), which a sandbox image cannot forge the @@ -361,7 +344,7 @@ pub const OTLP_RELAY_TRACES_ENDPOINT: &str = "http://192.0.0.8:4318/v1/traces"; mod otlp_relay_address_tests { use std::net::{IpAddr, Ipv4Addr, SocketAddr}; - use super::{OTLP_RELAY_ADDR, OTLP_RELAY_ENDPOINT, OTLP_RELAY_TRACES_ENDPOINT}; + use super::{OTLP_RELAY_ADDR, OTLP_RELAY_ENDPOINT}; #[test] fn relay_address_is_an_unroutable_non_loopback_label() { @@ -381,10 +364,6 @@ mod otlp_relay_address_tests { "198.18.0.0/15 is the policy DNS synthetic answer pool" ); assert_eq!(OTLP_RELAY_ENDPOINT, format!("http://{OTLP_RELAY_ADDR}")); - assert_eq!( - OTLP_RELAY_TRACES_ENDPOINT, - format!("{OTLP_RELAY_ENDPOINT}/v1/traces") - ); } } diff --git a/crates/openshell-isolation-interface/src/lib.rs b/crates/openshell-isolation-interface/src/lib.rs index bfa1b0b546..446f6cc244 100644 --- a/crates/openshell-isolation-interface/src/lib.rs +++ b/crates/openshell-isolation-interface/src/lib.rs @@ -30,6 +30,9 @@ pub struct AgentSpec { pub timeout_secs: u64, /// Whether the entrypoint runs interactively (inherits the parent pgrp). pub interactive: bool, + /// Whether the supervisor serves the OTLP relay for this sandbox. The + /// boundary points workload SDKs at the relay only while it is active. + pub otlp_relay_active: bool, } pub mod contract; diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index 914bbea861..81f0f7ff08 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -189,7 +189,10 @@ pub fn resource_for(config: &OtlpExportConfig<'_>) -> Resource { } /// Trim and validate an OTLP endpoint before handing it to an exporter. -fn validated_endpoint(endpoint: &str) -> Result<(&str, http::Uri), SetupError> { +/// +/// Every exporter that reads an operator-supplied endpoint uses this, so a +/// value accepted for one signal is accepted for all of them. +pub fn validated_endpoint(endpoint: &str) -> Result<(&str, http::Uri), SetupError> { let endpoint = endpoint.trim(); if endpoint.is_empty() { return Err(SetupError::EmptyEndpoint); diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index 8402a5e7a5..b35c448774 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -1217,6 +1217,10 @@ pub struct AgentSpecWire { pub workdir: Option, pub timeout_secs: u64, pub interactive: bool, + /// Omitted while false so the request digest of a sandbox without a + /// relay is unchanged. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub otlp_relay_active: bool, } impl From for AgentSpecWire { @@ -1227,6 +1231,7 @@ impl From for AgentSpecWire { workdir: spec.workdir, timeout_secs: spec.timeout_secs, interactive: spec.interactive, + otlp_relay_active: spec.otlp_relay_active, } } } @@ -1641,6 +1646,30 @@ mod tests { ); } + #[test] + fn agent_spec_omits_an_inactive_relay_flag_on_the_wire() { + let mut spec = AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: None, + timeout_secs: 0, + interactive: false, + otlp_relay_active: false, + }; + let encoded = serde_json::to_value(&spec).unwrap(); + assert!( + encoded.get("otlp_relay_active").is_none(), + "a false flag keeps the encoding of earlier releases: {encoded}" + ); + let decoded: AgentSpecWire = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded, spec); + + spec.otlp_relay_active = true; + let decoded: AgentSpecWire = + serde_json::from_value(serde_json::to_value(&spec).unwrap()).unwrap(); + assert!(decoded.otlp_relay_active); + } + #[test] fn request_round_trips_and_redacts_secrets() { let request = RequestEnvelope { @@ -1656,6 +1685,7 @@ mod tests { workdir: Some("/sandbox".to_string()), timeout_secs: 5, interactive: false, + otlp_relay_active: false, }, ca_cert: Some("test certificate".to_string()), ca_bundle: Some("test bundle".to_string()), @@ -1813,6 +1843,7 @@ mod tests { workdir: None, timeout_secs: 5, interactive: false, + otlp_relay_active: false, }, ca_cert: Some("A".repeat(16 * 1024)), ca_bundle: Some("B".repeat(400 * 1024)), diff --git a/crates/openshell-sandbox-backend/src/protobuf.rs b/crates/openshell-sandbox-backend/src/protobuf.rs index 21c52788f1..f370319549 100644 --- a/crates/openshell-sandbox-backend/src/protobuf.rs +++ b/crates/openshell-sandbox-backend/src/protobuf.rs @@ -164,6 +164,7 @@ impl WireMessage for w::AgentSpecWire { nanos: 0, }), interactive: self.interactive, + otlp_relay_active: self.otlp_relay_active, }) } fn from_proto(v: Self::Proto) -> Result { @@ -177,6 +178,7 @@ impl WireMessage for w::AgentSpecWire { workdir: v.workdir, timeout_secs: u64::try_from(timeout.seconds).map_err(invalid)?, interactive: v.interactive, + otlp_relay_active: v.otlp_relay_active, }) } } @@ -843,6 +845,24 @@ mod tests { assert_ne!(changed.payload_digest, request.payload_digest); } + #[test] + fn agent_spec_carries_the_otlp_relay_flag() { + let mut spec = w::AgentSpecWire { + program: "/bin/true".into(), + args: vec![], + workdir: None, + timeout_secs: 0, + interactive: false, + otlp_relay_active: false, + }; + let inactive = spec.to_proto().unwrap(); + assert_eq!(w::AgentSpecWire::from_proto(inactive).unwrap(), spec); + + spec.otlp_relay_active = true; + let decoded = w::AgentSpecWire::from_proto(spec.to_proto().unwrap()).unwrap(); + assert!(decoded.otlp_relay_active); + } + #[test] fn resize_rejects_values_outside_the_rust_contract() { let value = p::RequestEnvelope { diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 8e54aa9832..4cb55a2bde 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -3411,6 +3411,7 @@ mod tests { workdir: Some("/sandbox".to_string()), timeout_secs: 5, interactive: false, + otlp_relay_active: false, }, identity: ResolvedWorkloadIdentity::new( 10_001, diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index 5f6ff12c2d..7a7ef91889 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -37,6 +37,7 @@ pub struct LocalBoundaryExec { provider_credentials: ProviderCredentialState, user_environment: HashMap, runtime: Arc, + otlp_relay_active: bool, #[cfg(target_os = "linux")] launcher: crate::linux::workload_launcher::WorkloadLauncher, } @@ -44,6 +45,7 @@ pub struct LocalBoundaryExec { impl LocalBoundaryExec { /// Construct the executor owned by an active sandbox boundary. #[must_use] + #[allow(clippy::too_many_arguments)] pub fn new( policy: SandboxPolicy, base_workdir: Option, @@ -51,6 +53,7 @@ impl LocalBoundaryExec { provider_credentials: ProviderCredentialState, user_environment: HashMap, runtime: Arc, + otlp_relay_active: bool, #[cfg(target_os = "linux")] launcher: crate::linux::workload_launcher::WorkloadLauncher, ) -> Self { Self { @@ -60,6 +63,7 @@ impl LocalBoundaryExec { provider_credentials, user_environment, runtime, + otlp_relay_active, #[cfg(target_os = "linux")] launcher, } @@ -182,18 +186,17 @@ impl LocalBoundaryExec { } } crate::process::strip_proxy_env_std(&mut command); - // Point OTel SDKs at the supervisor's trace relay, after the creation + // Point OTel SDKs at the supervisor's trace relay after the creation // request's environment and the provider variables, matching the main - // process, and only when the creation request named no endpoint of - // its own. A per-exec `spec.env` value applied below still wins for - // that exec only. - if let Some(relay_env) = - crate::child_env::otlp_relay_env_vars_unless_configured(&self.user_environment) - { - for (key, value) in relay_env { - command.env(key, value); - } - } + // process. An endpoint in the creation request, the image, or this + // exec's `spec.env` suppresses it, and any other `spec.env` value + // applied below still wins for this exec only. + crate::child_env::apply_otlp_relay_env( + &mut command, + self.otlp_relay_active, + &self.user_environment, + &spec.env, + ); for (key, value) in &spec.env { if !key.starts_with(crate::process::RESERVED_ENV_PREFIX) { command.env(key, value); @@ -691,6 +694,10 @@ mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; fn executor() -> LocalBoundaryExec { + executor_with_relay(true) + } + + fn executor_with_relay(otlp_relay_active: bool) -> LocalBoundaryExec { let (launcher, listener) = crate::linux::workload_launcher::start().expect("start test workload launcher"); std::thread::spawn(move || { @@ -726,6 +733,7 @@ mod tests { ), HashMap::new(), crate::boundary_io::BoundaryRuntimeState::new(), + otlp_relay_active, launcher, ) } @@ -820,12 +828,22 @@ mod tests { Some("http/protobuf") ); + assert_eq!( + env_of( + &command, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + ), + None, + "only the generic default is injected" + ); + // A per-exec environment is a developer's explicit instruction and - // is applied last (contracts/sandbox-environment.md). + // is applied last (contracts/sandbox-environment.md). An endpoint in + // it suppresses the whole relay default, protocol included. let command = executor .command(&spec(vec![( openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.to_string(), - "http://127.0.0.1:4318".to_string(), + "http://127.0.0.1:4317".to_string(), )])) .expect("resolve exec"); assert_eq!( @@ -834,8 +852,40 @@ mod tests { openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT ) .as_deref(), - Some("http://127.0.0.1:4318") + Some("http://127.0.0.1:4317") ); + assert_eq!( + env_of( + &command, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL + ), + None + ); + } + + #[test] + fn exec_children_get_no_relay_endpoint_while_the_relay_is_inactive() { + let command = executor_with_relay(false) + .command(&ExecSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + shell: None, + runtime_helper: None, + env: Vec::new(), + workdir: None, + pty: false, + }) + .expect("resolve exec"); + for key in [ + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL, + ] { + assert_eq!( + env_of(&command, key), + None, + "{key} injected without a relay" + ); + } } #[tokio::test] diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 56a974b9a1..439e4a9754 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -3493,6 +3493,7 @@ mod linux { provider_env, ca_file_paths, Some(boundary_runtime.clone()), + spec.otlp_relay_active, )) .map_err(|error| format!("start process supervisor leaf: {error:?}"))?; let signaler = spawned.signaler(); @@ -4372,6 +4373,7 @@ mod linux { workdir: Some("/sandbox".to_string()), timeout_secs: 0, interactive: true, + otlp_relay_active: false, }) .expect("resolve scratch command"); @@ -6368,6 +6370,7 @@ mod linux { workdir: None, timeout_secs: 60, interactive: false, + otlp_relay_active: false, }; assert!(matches!( @@ -6671,6 +6674,7 @@ mod linux { workdir: None, timeout_secs: 5, interactive: false, + otlp_relay_active: false, }; let wire_policy = SandboxPolicyWire::from(policy.clone()); let process = Arc::new( diff --git a/crates/openshell-sandbox/src/child_env.rs b/crates/openshell-sandbox/src/child_env.rs index abf0fcb029..eacd858dde 100644 --- a/crates/openshell-sandbox/src/child_env.rs +++ b/crates/openshell-sandbox/src/child_env.rs @@ -6,56 +6,105 @@ use std::hash::BuildHasher; use std::path::Path; use openshell_core::sandbox_env::{ - OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, - OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, OTLP_RELAY_ENDPOINT, OTLP_RELAY_TRACES_ENDPOINT, + OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + OTLP_RELAY_ENDPOINT, }; -/// OpenTelemetry exporter variables a workload process receives when the -/// sandbox creation request names no OTLP endpoint of its own. +/// Variables that name where an OpenTelemetry SDK exports traces. A layer of +/// the workload environment that sets one has chosen a destination. +const OTLP_ENDPOINT_VARS: [&str; 2] = [ + OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, +]; + +/// OpenTelemetry exporter variables that point agent SDKs at the +/// supervisor's OTLP relay. /// -/// They point agent SDKs at the supervisor's OTLP relay. The signal-specific -/// traces variables are set as well because SDKs give them precedence over -/// the generic ones, and compression is disabled because the relay refuses -/// compressed bodies. The values are the same whether or not a collector is -/// configured; without one the relay address refuses connections -/// immediately. Use [`otlp_relay_env_vars_unless_configured`] to respect a -/// caller who deliberately exports elsewhere. -pub fn otlp_relay_env_vars() -> [(&'static str, &'static str); 6] { +/// Only the generic variables are set. Signal-specific variables take +/// precedence over them in every SDK, so an endpoint the workload sets later +/// at any layer, generic or traces-specific, wins over this default. +pub fn otlp_relay_env_vars() -> [(&'static str, &'static str); 2] { [ (OTEL_EXPORTER_OTLP_ENDPOINT, OTLP_RELAY_ENDPOINT), (OTEL_EXPORTER_OTLP_PROTOCOL, "http/protobuf"), - ( - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, - OTLP_RELAY_TRACES_ENDPOINT, - ), - (OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, "http/protobuf"), - (OTEL_EXPORTER_OTLP_COMPRESSION, "none"), - (OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, "none"), ] } -/// Whether a sandbox creation environment names an OTLP endpoint, generic or -/// traces-specific. A caller who does has chosen where the agent exports. -pub fn names_otlp_endpoint(user_environment: &HashMap) -> bool { - [ - OTEL_EXPORTER_OTLP_ENDPOINT, - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, - ] - .iter() - .any(|key| user_environment.contains_key(*key)) +/// Whether one layer of the workload environment names an OTLP endpoint, +/// generic or traces-specific. +pub fn names_otlp_endpoint(environment: &HashMap) -> bool { + OTLP_ENDPOINT_VARS + .iter() + .any(|key| environment.contains_key(*key)) } -/// [`otlp_relay_env_vars`] unless the creation request named an endpoint. +/// Whether the relay default applies: the relay is active and no layer the +/// caller controls names an endpoint. `inherited` reports whether the +/// sandbox's own environment, which carries the image `ENV`, sets a variable. +fn relay_default_applies( + relay_active: bool, + user_environment: &HashMap, + exec_environment: &[(String, String)], + inherited: impl Fn(&str) -> bool, +) -> bool { + relay_active + && !names_otlp_endpoint(user_environment) + && !exec_environment + .iter() + .any(|(key, _)| OTLP_ENDPOINT_VARS.contains(&key.as_str())) + && !OTLP_ENDPOINT_VARS.iter().any(|key| inherited(key)) +} + +/// Applies the relay default to `cmd` after every other layer except a +/// per-exec environment. /// -/// Returns `None` when the caller named one, and then none of the caller's -/// `OTEL_EXPORTER_OTLP_*` values are touched: the agent exports where it was -/// told to, subject to the sandbox's network policy and outside relay -/// attribution, instead of being redirected to the relay. -pub fn otlp_relay_env_vars_unless_configured( +/// Nothing is injected while the relay is inactive, or when the creation +/// request (`user_environment`), the image, or `exec_environment` names an +/// endpoint: the agent then exports where it was told to, subject to the +/// sandbox's network policy and outside relay attribution. When the default +/// is injected, a traces-specific endpoint can only have come from a +/// provider, and is removed so a provider cannot redirect traces. +pub fn apply_otlp_relay_env( + cmd: &mut impl EnvTarget, + relay_active: bool, user_environment: &HashMap, -) -> Option<[(&'static str, &'static str); 6]> { - (!names_otlp_endpoint(user_environment)).then(otlp_relay_env_vars) + exec_environment: &[(String, String)], +) { + if !relay_default_applies(relay_active, user_environment, exec_environment, |key| { + std::env::var_os(key).is_some() + }) { + return; + } + cmd.remove_var(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT); + for (key, value) in otlp_relay_env_vars() { + cmd.set_var(key, value); + } +} + +/// The environment-editing surface shared by `std` and `tokio` commands. +pub trait EnvTarget { + fn set_var(&mut self, key: &str, value: &str); + fn remove_var(&mut self, key: &str); +} + +impl EnvTarget for std::process::Command { + fn set_var(&mut self, key: &str, value: &str) { + self.env(key, value); + } + + fn remove_var(&mut self, key: &str) { + self.env_remove(key); + } +} + +impl EnvTarget for tokio::process::Command { + fn set_var(&mut self, key: &str, value: &str) { + self.env(key, value); + } + + fn remove_var(&mut self, key: &str) { + self.env_remove(key); + } } pub fn tls_env_vars( @@ -113,37 +162,49 @@ mod tests { [ ("OTEL_EXPORTER_OTLP_ENDPOINT", "http://192.0.0.8:4318"), ("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"), - ( - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", - "http://192.0.0.8:4318/v1/traces" - ), - ("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "http/protobuf"), - ("OTEL_EXPORTER_OTLP_COMPRESSION", "none"), - ("OTEL_EXPORTER_OTLP_TRACES_COMPRESSION", "none"), ] ); } #[test] - fn a_caller_named_endpoint_disables_relay_injection() { - let none = HashMap::new(); - assert!(otlp_relay_env_vars_unless_configured(&none).is_some()); + fn relay_default_applies_only_while_active_and_unconfigured() { + let none: HashMap = HashMap::new(); + let nothing_inherited = |_: &str| false; + assert!(relay_default_applies(true, &none, &[], nothing_inherited)); + assert!( + !relay_default_applies(false, &none, &[], nothing_inherited), + "an inactive relay gets no default" + ); // Protocol or compression alone do not name a destination. let protocol_only = HashMap::from([( "OTEL_EXPORTER_OTLP_PROTOCOL".to_string(), "grpc".to_string(), )]); - assert!(otlp_relay_env_vars_unless_configured(&protocol_only).is_some()); + assert!(relay_default_applies( + true, + &protocol_only, + &[], + nothing_inherited + )); for key in [ "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", ] { - let named = HashMap::from([(key.to_string(), "http://collector.example:4318".into())]); + let value = "http://collector.example:4318".to_string(); + let declared = HashMap::from([(key.to_string(), value.clone())]); + assert!( + !relay_default_applies(true, &declared, &[], nothing_inherited), + "{key} in the creation request names a destination" + ); + assert!( + !relay_default_applies(true, &none, &[(key.to_string(), value)], nothing_inherited), + "{key} in the exec environment names a destination" + ); assert!( - otlp_relay_env_vars_unless_configured(&named).is_none(), - "{key} names a destination" + !relay_default_applies(true, &none, &[], |inherited| inherited == key), + "{key} in the image environment names a destination" ); } } diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index 45639c4c00..2dc7899103 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -41,6 +41,7 @@ pub async fn spawn_workload( provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, boundary_runtime: Option>, + otlp_relay_active: bool, ) -> Result { // Driver-selected workspaces are the sandbox identity's home. This keeps // canonical and later exec processes consistent for image WorkingDir and @@ -83,6 +84,7 @@ pub async fn spawn_workload( provider_credentials, user_environment, boundary_runtime.clone(), + otlp_relay_active, launcher.clone(), )); @@ -96,6 +98,7 @@ pub async fn spawn_workload( policy, ca_file_paths.as_ref(), &provider_env, + otlp_relay_active, ) .wrap_err("spawn delegated workload process")?; #[cfg(not(target_os = "linux"))] @@ -107,6 +110,7 @@ pub async fn spawn_workload( policy, ca_file_paths.as_ref(), &provider_env, + otlp_relay_active, )?; entrypoint_pid.store(handle.pid(), Ordering::Release); diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 093ba19c54..3f3a2cc9d3 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -274,8 +274,10 @@ pub(crate) fn configured_user_environment() -> HashMap { /// sandbox environment minus the reserved `OPENSHELL_` namespace, the /// declared user environment, supervisor-only and proxy variables stripped, /// provider placeholders, TLS trust paths, and finally the OTLP relay -/// variables, applied last so a provider cannot redirect traces, and only -/// when the declared environment names no OTLP endpoint of its own. +/// default, applied last so a provider cannot redirect traces, and only +/// while the relay is active and neither the declared environment nor the +/// image names an OTLP endpoint of its own. +#[allow(clippy::too_many_arguments)] fn compose_canonical_process_environment( cmd: &mut Command, policy: &SandboxPolicy, @@ -284,6 +286,7 @@ fn compose_canonical_process_environment( user_environment: &HashMap, provider_env: &HashMap, ca_paths: Option<&(PathBuf, PathBuf)>, + otlp_relay_active: bool, ) { apply_canonical_process_environment(cmd, policy, workspace, interactive, user_environment); strip_supervisor_only_env(cmd); @@ -295,11 +298,7 @@ fn compose_canonical_process_environment( cmd.env(key, value); } } - if let Some(relay_env) = child_env::otlp_relay_env_vars_unless_configured(user_environment) { - for (key, value) in relay_env { - cmd.env(key, value); - } - } + child_env::apply_otlp_relay_env(cmd, otlp_relay_active, user_environment, &[]); } #[cfg(unix)] @@ -462,6 +461,7 @@ impl ProcessHandle { policy: &SandboxPolicy, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, + otlp_relay_active: bool, ) -> Result { Self::spawn_impl( launcher, @@ -472,6 +472,7 @@ impl ProcessHandle { policy, ca_paths, provider_env, + otlp_relay_active, ) } @@ -490,6 +491,7 @@ impl ProcessHandle { policy: &SandboxPolicy, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, + otlp_relay_active: bool, ) -> Result { Self::spawn_impl( program, @@ -499,6 +501,7 @@ impl ProcessHandle { policy, ca_paths, provider_env, + otlp_relay_active, ) } @@ -513,6 +516,7 @@ impl ProcessHandle { policy: &SandboxPolicy, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, + otlp_relay_active: bool, ) -> Result { let mut cmd = Command::new(program); cmd.args(args).kill_on_drop(true); @@ -552,6 +556,7 @@ impl ProcessHandle { &configured_user_environment(), provider_env, ca_paths, + otlp_relay_active, ); if let Some(dir) = workspace.root() { @@ -663,6 +668,7 @@ impl ProcessHandle { policy: &SandboxPolicy, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, + otlp_relay_active: bool, ) -> Result { let mut cmd = Command::new(program); cmd.args(args).kill_on_drop(true); @@ -702,6 +708,7 @@ impl ProcessHandle { &configured_user_environment(), provider_env, ca_paths, + otlp_relay_active, ); if let Some(dir) = workspace.root() { @@ -1173,6 +1180,14 @@ mod tests { command .args(["--exact", test_path, "--nocapture"]) .env(marker, "1"); + // A harness that exports OTLP endpoints must not look like an image + // that ships one. + for key in [ + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT, + openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + ] { + command.env_remove(key); + } for (key, value) in env { command.env(key, value); } @@ -1316,16 +1331,21 @@ mod tests { } } - /// The production composition injects the relay variables when the - /// declared environment names no OTLP endpoint, leaves a caller's own - /// endpoint alone otherwise, and strips the collector address in both - /// cases. Runs in a re-executed copy of the test binary so the collector - /// address can sit in the inherited environment, where - /// `OPENSHELL_OTLP_ENDPOINT` lives for a real supervisor, without touching - /// the harness environment. + /// The production composition injects the relay default only while the + /// relay is active and the declared environment names no OTLP endpoint, + /// leaves a caller's own endpoint alone otherwise, and strips the + /// collector address in every case. Runs in a re-executed copy of the + /// test binary so the collector address can sit in the inherited + /// environment, where `OPENSHELL_OTLP_ENDPOINT` lives for a real + /// supervisor, without touching the harness environment. #[cfg(unix)] #[test] fn canonical_process_sees_relay_endpoint_and_never_the_collector() { + use openshell_core::sandbox_env::{ + OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTLP_RELAY_ENDPOINT, + }; + const OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: &str = "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION"; const COLLECTOR_HOST: &str = "collector.invalid"; if run_in_isolated_child( "OPENSHELL_TEST_RELAY_ENV_CHILD", @@ -1343,7 +1363,9 @@ mod tests { run_as_group: None, }); let workspace = ResolvedWorkspace::default(); - let probe = |declared: HashMap, provider_env: HashMap| { + let probe = |declared: HashMap, + provider_env: HashMap, + relay_active: bool| { // Mirror production: inherit the sandbox environment, no env_clear. let mut cmd = Command::new("/usr/bin/env"); cmd.stdout(StdStdio::piped()); @@ -1355,6 +1377,7 @@ mod tests { &declared, &provider_env, None, + relay_active, ); let variables = probe_environment(cmd); assert!( @@ -1370,53 +1393,39 @@ mod tests { variables }; let get = |variables: &HashMap, key: &str| variables.get(key).cloned(); + let provider_redirect = || { + HashMap::from([ + ( + OTEL_EXPORTER_OTLP_ENDPOINT.to_string(), + "http://provider.invalid".to_string(), + ), + ( + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.to_string(), + "http://provider.invalid/v1/traces".to_string(), + ), + ]) + }; - // No endpoint in the creation request: the relay variables are set, - // and a provider cannot redirect traces either. + // Relay active, no endpoint in the creation request: only the + // generic default is set, and a provider cannot redirect traces. let variables = probe( - HashMap::from([( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL.into(), - "grpc".into(), - )]), - HashMap::from([( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT.to_string(), - "http://provider.invalid".to_string(), - )]), + HashMap::from([(OTEL_EXPORTER_OTLP_PROTOCOL.into(), "grpc".into())]), + provider_redirect(), + true, ); assert_eq!( - get( - &variables, - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT - ) - .as_deref(), - Some(openshell_core::sandbox_env::OTLP_RELAY_ENDPOINT), + get(&variables, OTEL_EXPORTER_OTLP_ENDPOINT).as_deref(), + Some(OTLP_RELAY_ENDPOINT), "the relay endpoint wins over a provider's: {variables:?}" ); assert_eq!( - get( - &variables, - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL - ) - .as_deref(), + get(&variables, OTEL_EXPORTER_OTLP_PROTOCOL).as_deref(), Some("http/protobuf"), "a declared protocol without an endpoint is replaced: {variables:?}" ); - assert_eq!( - get( - &variables, - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - ) - .as_deref(), - Some(openshell_core::sandbox_env::OTLP_RELAY_TRACES_ENDPOINT), - "the traces-specific endpoint is set too: {variables:?}" - ); - assert_eq!( - get( - &variables, - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION - ) - .as_deref(), - Some("none") + assert!( + !variables.contains_key(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT), + "no signal-specific endpoint is injected, and a provider's is removed: {variables:?}" ); assert!( !variables @@ -1425,52 +1434,46 @@ mod tests { "a provider endpoint overrode the relay: {variables:?}" ); + // Relay inactive: nothing points at the relay address. + let variables = probe(HashMap::new(), HashMap::new(), false); + assert!( + !variables.contains_key(OTEL_EXPORTER_OTLP_ENDPOINT), + "no endpoint is injected while the relay is inactive: {variables:?}" + ); + assert!( + !variables.values().any(|value| value.contains("192.0.0.8")), + "the relay address must not appear: {variables:?}" + ); + // The creation request names its own endpoint: every caller value // stays as declared and nothing points at the relay. let variables = probe( HashMap::from([ ( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.into(), + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.into(), "http://own-backend.invalid/v1/traces".into(), ), - ( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL.into(), - "grpc".into(), - ), - ( - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION.into(), - "gzip".into(), - ), + (OTEL_EXPORTER_OTLP_PROTOCOL.into(), "grpc".into()), + (OTEL_EXPORTER_OTLP_TRACES_COMPRESSION.into(), "gzip".into()), ]), HashMap::new(), + true, ); assert_eq!( - get( - &variables, - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - ) - .as_deref(), + get(&variables, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT).as_deref(), Some("http://own-backend.invalid/v1/traces"), "a caller's endpoint is left alone: {variables:?}" ); assert_eq!( - get( - &variables, - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_PROTOCOL - ) - .as_deref(), + get(&variables, OTEL_EXPORTER_OTLP_PROTOCOL).as_deref(), Some("grpc") ); assert_eq!( - get( - &variables, - openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_TRACES_COMPRESSION - ) - .as_deref(), + get(&variables, OTEL_EXPORTER_OTLP_TRACES_COMPRESSION).as_deref(), Some("gzip") ); assert!( - !variables.contains_key(openshell_core::sandbox_env::OTEL_EXPORTER_OTLP_ENDPOINT), + !variables.contains_key(OTEL_EXPORTER_OTLP_ENDPOINT), "nothing is injected when the caller named an endpoint: {variables:?}" ); assert!( @@ -1479,6 +1482,55 @@ mod tests { ); } + /// An endpoint in the image `ENV`, which reaches the canonical process + /// through the inherited sandbox environment, is left alone too. + #[cfg(unix)] + #[test] + fn canonical_process_keeps_an_image_endpoint() { + use openshell_core::sandbox_env::{ + OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_PROTOCOL, + }; + if run_in_isolated_child( + "OPENSHELL_TEST_RELAY_IMAGE_ENV_CHILD", + "process::tests::canonical_process_keeps_an_image_endpoint", + &[( + OTEL_EXPORTER_OTLP_ENDPOINT, + "http://image-collector.invalid:4318".to_string(), + )], + ) { + return; + } + let current_user = User::from_uid(nix::unistd::geteuid()).unwrap().unwrap(); + let policy = policy_with_process(ProcessPolicy { + run_as_user: Some(current_user.name), + run_as_group: None, + }); + let mut cmd = Command::new("/usr/bin/env"); + cmd.stdout(StdStdio::piped()); + compose_canonical_process_environment( + &mut cmd, + &policy, + &ResolvedWorkspace::default(), + false, + &HashMap::new(), + &HashMap::new(), + None, + true, + ); + let variables = probe_environment(cmd); + assert_eq!( + variables + .get(OTEL_EXPORTER_OTLP_ENDPOINT) + .map(String::as_str), + Some("http://image-collector.invalid:4318"), + "the image endpoint is left alone: {variables:?}" + ); + assert!( + !variables.contains_key(OTEL_EXPORTER_OTLP_PROTOCOL), + "no relay protocol is injected next to an image endpoint: {variables:?}" + ); + } + #[cfg(unix)] #[allow(unsafe_code)] fn probe_hardened_child(probe: unsafe fn() -> i64) -> i64 { diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 46e74225a9..2cd827d235 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -92,7 +92,7 @@ Console JSON is installed independently of optional file appenders and uses an I The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_ENDPOINT`, the same one the supervisor uses for its own spans. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards batches over OTLP/gRPC with a lazy connection. -Limits are fixed: 2 MiB per request body, at most 512 `ResourceSpans` entries, 256 KiB of encoded `Resource` fields, and 8192 resource attributes per request (each of these answers 413), a 10 second header-read timeout, 16 concurrent connections, 32 buffered batches, and a 5 second deadline per export attempt. Only the resources are decoded; spans are copied byte for byte, at most two requests at a time. Worst-case memory attributable to the agent is about 67 MiB of buffered batches, plus 32 MiB of bodies awaiting enrichment across the 16 connections, plus about 12 MiB for the two enrichments in progress. A full buffer answers 503 with `Retry-After`. Without a collector endpoint nothing starts and the proxy refuses the relay address. After the agent exits, `run_sandbox_with_backend` flushes buffered batches for up to 5 seconds before it reports the exit to the gateway, then logs a final summary of rejected and failed counts. +Limits are fixed: 2 MiB per request body, at most 512 `ResourceSpans` entries, 256 KiB of encoded `Resource` fields, and 8192 resource attributes per request (each of these answers 413), a 10 second header-read timeout, 16 concurrent connections, 32 buffered batches, and a 5 second deadline per export attempt. Only the resources are decoded; spans are copied byte for byte, at most two requests at a time. Worst-case memory attributable to the agent is about 67 MiB of buffered batches, plus 32 MiB of bodies awaiting enrichment across the 16 connections, plus about 12 MiB for the two enrichments in progress, plus about 4 MiB for the batch being exported and its encoded copy. A full buffer answers 503 with `Retry-After`. Without a collector endpoint nothing starts and the proxy refuses the relay address. After the agent exits, `run_sandbox_with_backend` flushes buffered batches for up to 5 seconds before it reports the exit to the gateway, then logs a final summary of rejected and failed counts. Flushes skip the wait while the most recent export failed, and batches still pending when the relay is dropped without a flush count as failed. The supervisor tells the sandbox runtime through `AgentSpec::otlp_relay_active` whether the relay runs, and the runtime injects the generic `OTEL_EXPORTER_OTLP_*` defaults only then. ## Safe runtime and parent liveness diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index a2ad0bdb68..88823bf619 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -212,6 +212,7 @@ fn agent() -> AgentSpec { workdir: Some("/test-workspace".into()), timeout_secs: 15, interactive: false, + otlp_relay_active: false, } } diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 0c96fdcdf5..7c22a6f0a8 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -1159,6 +1159,13 @@ pub async fn run_sandbox_instance( // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0)); + // The agent trace relay must exist before networking starts so the + // handler is installed when the first staged connection arrives, and + // before attach so the agent spec can tell the boundary whether to point + // workload SDKs at it. + let (mut otlp_relay, otlp_reserved_destination) = + otlp_relay::activate(otlp_endpoint, sandbox_id.clone()); + // Construct the selected client only after shared policy and credential // setup has completed. Keep live credentials and later CA publication in // supervisor-owned handles shared with the backend. @@ -1178,6 +1185,7 @@ pub async fn run_sandbox_instance( workdir: workspace, timeout_secs, interactive, + otlp_relay_active: otlp_relay.is_some(), }, ) .instrument(tracing::info_span!(parent: &startup, "supervisor.boundary.attach")) @@ -1266,11 +1274,6 @@ pub async fn run_sandbox_instance( (ready, backend_name, ca_file_paths) }; - // The agent trace relay must exist before networking starts so the - // handler is installed when the first staged connection arrives. - let (mut otlp_relay, otlp_reserved_destination) = - otlp_relay::activate(otlp_endpoint, sandbox_id.clone()); - let mut networking = Some( openshell_supervisor_network::run::run_networking( &policy, diff --git a/crates/openshell-supervisor/src/otlp_relay/enrichment.rs b/crates/openshell-supervisor/src/otlp_relay/enrichment.rs index 03801e25b1..7fce2483d2 100644 --- a/crates/openshell-supervisor/src/otlp_relay/enrichment.rs +++ b/crates/openshell-supervisor/src/otlp_relay/enrichment.rs @@ -4,8 +4,11 @@ //! Resource attribution for relayed agent traces. //! //! Every forwarded `ResourceSpans` entry carries exactly two supervisor-owned -//! attributes. Agent-supplied values for those keys are always discarded so -//! a workload cannot impersonate another sandbox or pose as infrastructure. +//! resource attributes. Agent-supplied values for those keys are always +//! discarded at the resource level, so a workload cannot attribute its +//! resources to another sandbox or to infrastructure. Scope and span +//! attributes are copied unchanged and may carry the same keys with any +//! value; only the resource attributes are authoritative. //! //! Enrichment works at the protobuf wire level. Only each entry's `resource` //! field is decoded and re-encoded, which drops unknown fields inside it; diff --git a/crates/openshell-supervisor/src/otlp_relay/mod.rs b/crates/openshell-supervisor/src/otlp_relay/mod.rs index 95134d52f1..c6bdfa3634 100644 --- a/crates/openshell-supervisor/src/otlp_relay/mod.rs +++ b/crates/openshell-supervisor/src/otlp_relay/mod.rs @@ -44,7 +44,9 @@ pub const MAX_BODY_BYTES: usize = 2 * 1024 * 1024; /// [`MAX_BODY_BYTES`] (32 MiB); and the working set of the /// [`ENRICHMENT_CONCURRENCY`] enrichments in progress, each about twice the /// body plus at most about 1 MiB of decoded resource elements (about -/// 12 MiB). Every allocation is sized to its content before it is held. +/// 12 MiB). On top of that, the export task holds the batch it took from +/// the buffer and tonic encodes a copy of it, one batch each (about 4 MiB). +/// Every allocation is sized to its content before it is held. pub const MAX_ENRICHED_BYTES: usize = MAX_BODY_BYTES + enrichment::MAX_RESOURCE_SPANS * enrichment::ATTRIBUTION_BYTES_PER_ENTRY; /// How long a connection may idle before a complete request header arrives. @@ -92,6 +94,9 @@ pub struct RelayCounters { exported: AtomicU64, rejected: AtomicU64, export_failures: AtomicU64, + /// Whether the most recent export attempt failed. Cleared by the next + /// success; flushes skip waiting while it is set. + last_export_failed: AtomicBool, first_rejection_logged: AtomicBool, first_failure_logged: AtomicBool, final_summary_logged: AtomicBool, @@ -119,6 +124,7 @@ impl RelayCounters { /// first occurrence only. pub fn record_export_failure(&self, endpoint: &str, error: &dyn std::fmt::Display) { self.export_failures.fetch_add(1, Ordering::Relaxed); + self.last_export_failed.store(true, Ordering::Relaxed); if !self.first_failure_logged.swap(true, Ordering::Relaxed) { warn!(endpoint, %error, "OTLP relay export to collector failed"); } @@ -127,6 +133,12 @@ impl RelayCounters { /// Records one batch delivered to the collector. pub fn record_export_success(&self) { self.exported.fetch_add(1, Ordering::Relaxed); + self.last_export_failed.store(false, Ordering::Relaxed); + } + + /// Whether the most recent export attempt failed. + fn last_export_failed(&self) -> bool { + self.last_export_failed.load(Ordering::Relaxed) } /// Current counts. @@ -189,8 +201,11 @@ pub struct OtlpRelayConfig { /// The relay could not be started; it stays inactive. #[derive(Debug)] pub enum StartError { - /// The collector endpoint is not a URI tonic accepts. - InvalidEndpoint(tonic::transport::Error), + /// The collector endpoint is empty or not a URI, judged the same way as + /// for the supervisor's own exporter. + InvalidEndpoint(openshell_otel::SetupError), + /// The collector endpoint is a URI tonic does not accept. + Transport(tonic::transport::Error), /// The collector endpoint uses a scheme the relay does not speak. TLS to /// the collector is a tracked follow-up; the supervisor's own exporter /// rejects `https` the same way. @@ -201,6 +216,7 @@ impl std::fmt::Display for StartError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::InvalidEndpoint(error) => write!(f, "invalid collector endpoint: {error}"), + Self::Transport(error) => write!(f, "invalid collector endpoint: {error}"), Self::UnsupportedScheme(scheme) => write!( f, "unsupported collector endpoint scheme {scheme:?}: the relay speaks plaintext OTLP/gRPC (http) only" @@ -226,6 +242,19 @@ impl Drop for OtlpRelay { fn drop(&mut self) { self.export_task.abort(); self.summary_task.abort(); + // A lifecycle path that drops the relay without `shutdown` discards + // whatever is still buffered or in flight; count it so the final + // summary reports the loss. After `shutdown` nothing is pending. + let discarded = self.counters.pending(); + if discarded > 0 { + self.counters + .export_failures + .fetch_add(discarded, Ordering::Relaxed); + warn!( + discarded, + "OTLP relay stopped without a flush; discarding buffered batches" + ); + } self.counters.log_final_summary(); } } @@ -254,15 +283,17 @@ fn relay_addr() -> SocketAddr { /// /// Returns the relay handle and the reserved destination to install in the /// network proxy. Fails only when the endpoint is not a valid plaintext URI; -/// an unreachable collector is handled lazily by the exporter. +/// an unreachable collector is handled lazily by the exporter. Surrounding +/// whitespace is trimmed, as for the supervisor's own exporter. pub fn start(config: OtlpRelayConfig) -> Result<(OtlpRelay, ReservedDestination), StartError> { - if let Ok(uri) = config.endpoint.parse::() - && let Some(scheme) = uri.scheme_str() + let (endpoint, uri) = openshell_otel::validated_endpoint(&config.endpoint) + .map_err(StartError::InvalidEndpoint)?; + if let Some(scheme) = uri.scheme_str() && !scheme.eq_ignore_ascii_case("http") { return Err(StartError::UnsupportedScheme(scheme.to_string())); } - let exporter = Exporter::new(&config.endpoint).map_err(StartError::InvalidEndpoint)?; + let exporter = Exporter::new(endpoint).map_err(StartError::Transport)?; if config.sandbox_id.len() > enrichment::MAX_SANDBOX_ID_BYTES { // The attribution growth bound assumes a shorter id; batches near // the wire limit with many entries will be refused with 413. @@ -278,7 +309,7 @@ pub fn start(config: OtlpRelayConfig) -> Result<(OtlpRelay, ReservedDestination) .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "enabled") - .unmapped("endpoint", config.endpoint) + .unmapped("endpoint", endpoint.to_string()) .message("OTLP agent trace relay enabled") .build() ); @@ -416,7 +447,21 @@ impl OtlpRelay { /// while the relay keeps serving. Used before the exit report when the /// supervisor stays up for exec sessions afterwards, so their traces /// are still relayed; [`Self::shutdown`] runs at final teardown. + /// + /// Returns at once when the most recent export failed: a collector that + /// is down would only run the clock out, and trace delivery must not + /// delay the exit report. pub async fn flush(&self, bound: Duration) { + if self.counters.last_export_failed() { + let pending = self.counters.pending(); + if pending > 0 { + warn!( + pending, + "OTLP relay flush skipped: the last export to the collector failed" + ); + } + return; + } let started = Instant::now(); let exported_before = self.counters.snapshot().exported; while self.counters.pending() > 0 && started.elapsed() < bound { @@ -440,7 +485,8 @@ impl OtlpRelay { /// Stops accepting agent connections, flushes buffered batches to the /// collector for at most `flush_bound`, and logs the final summary. /// Never fails; batches still pending at the deadline are discarded and - /// counted as export failures. + /// counted as export failures. Like [`Self::flush`], it does not wait for + /// the collector when the most recent export failed. pub async fn shutdown(mut self, flush_bound: Duration) { let started = Instant::now(); let exported_before = self.counters.snapshot().exported; @@ -451,6 +497,24 @@ impl OtlpRelay { } self.server.close_buffer(); + if self.counters.last_export_failed() { + self.export_task.abort(); + let _ = (&mut self.export_task).await; + let discarded = self.counters.pending(); + if discarded > 0 { + self.counters + .export_failures + .fetch_add(discarded, Ordering::Relaxed); + warn!( + discarded, + "OTLP relay flush skipped: the last export to the collector failed; discarding buffered batches" + ); + } + self.summary_task.abort(); + self.counters.log_final_summary(); + return; + } + let remaining = flush_bound.saturating_sub(started.elapsed()); match tokio::time::timeout(remaining, &mut self.export_task).await { Ok(Ok(())) => { @@ -724,12 +788,42 @@ mod tests { .err() .expect("unparsable endpoint fails"); assert!(matches!(error, StartError::InvalidEndpoint(_)), "{error}"); + for blank in ["", " "] { + let error = start(OtlpRelayConfig { + endpoint: blank.into(), + sandbox_id: "sb-test".into(), + }) + .err() + .expect("blank endpoint fails"); + assert!( + matches!( + error, + StartError::InvalidEndpoint(openshell_otel::SetupError::EmptyEndpoint) + ), + "{error}" + ); + } let after = tokio::runtime::Handle::current() .metrics() .num_alive_tasks(); assert_eq!(before, after, "nothing is spawned for an inactive relay"); } + #[tokio::test] + async fn endpoint_whitespace_is_trimmed_like_the_supervisor_exporter() { + let collector = OtlpTestServer::start().await; + let (relay, _reserved) = start(OtlpRelayConfig { + endpoint: format!(" {}\n", collector.endpoint()), + sandbox_id: "sb-test".into(), + }) + .expect("a padded endpoint starts the relay"); + assert_eq!(post(&relay, "padded").await, 200); + collector.wait_for_export().await; + relay.shutdown(FLUSH_TIMEOUT).await; + let received = collector.shutdown().await; + assert!(received.spans.iter().any(|span| span.name == "padded")); + } + #[tokio::test] async fn activate_without_endpoint_starts_nothing() { let before = tokio::runtime::Handle::current() @@ -813,6 +907,57 @@ mod tests { assert!(received.spans.iter().any(|span| span.name == "after-flush")); } + #[tokio::test] + async fn flush_and_shutdown_skip_waiting_after_a_failed_export() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let collector = SilentCollector::start().await; + // The gate stays closed, so the batches stay buffered. + let (relay, _reserved, _gate) = gated(&collector.endpoint); + for span in ["one", "two"] { + assert_eq!(post(&relay, span).await, 200); + } + relay + .counters + .record_export_failure(&collector.endpoint, &"collector down"); + let counters = Arc::clone(&relay.counters); + assert_eq!(counters.pending(), 1); + + let started = Instant::now(); + relay.flush(FLUSH_TIMEOUT).await; + assert!( + started.elapsed() < Duration::from_secs(1), + "flush waited {:?} for a collector that is down", + started.elapsed() + ); + assert_eq!( + logs.count( + tracing::Level::WARN, + "OTLP relay flush skipped: the last export to the collector failed" + ), + 1 + ); + + let started = Instant::now(); + relay.shutdown(FLUSH_TIMEOUT).await; + assert!( + started.elapsed() < RECEIVER_SHUTDOWN_TIMEOUT + Duration::from_secs(1), + "shutdown waited {:?} for a collector that is down", + started.elapsed() + ); + assert_eq!(counters.pending(), 0); + assert_eq!(counters.snapshot().export_failures, 2); + } + + #[tokio::test] + async fn a_successful_export_clears_the_failure_flag() { + let counters = RelayCounters::default(); + counters.record_export_failure("http://collector:4317", &"boom"); + assert!(counters.last_export_failed()); + counters.record_export_success(); + assert!(!counters.last_export_failed()); + } + #[tokio::test] async fn dropping_the_relay_without_shutdown_logs_the_final_summary_once() { let logs = LogCapture::default(); @@ -826,6 +971,37 @@ mod tests { drop(collector.shutdown().await); } + #[tokio::test] + async fn dropping_the_relay_counts_buffered_batches_as_discarded() { + let logs = LogCapture::default(); + let _guard = logs.install(); + let collector = OtlpTestServer::start().await; + // The gate stays closed, so both batches are still buffered. + let (relay, _reserved, _gate) = gated(collector.endpoint()); + for span in ["one", "two"] { + assert_eq!(post(&relay, span).await, 200); + } + let counters = Arc::clone(&relay.counters); + drop(relay); + assert_eq!(counters.snapshot().export_failures, 2); + assert_eq!(counters.pending(), 0); + assert_eq!( + logs.count( + tracing::Level::WARN, + "OTLP relay stopped without a flush; discarding buffered batches" + ), + 1 + ); + let summaries = logs.events(tracing::Level::INFO, "OTLP relay summary"); + assert_eq!( + summaries.len(), + 1, + "a loss alone triggers the final summary" + ); + assert_eq!(summaries[0].field("export_failures"), Some(2)); + drop(collector.shutdown().await); + } + #[tokio::test] async fn first_rejection_warns_once() { let logs = LogCapture::default(); diff --git a/crates/openshell-supervisor/src/otlp_relay/receiver.rs b/crates/openshell-supervisor/src/otlp_relay/receiver.rs index f579c3160c..e540d2980c 100644 --- a/crates/openshell-supervisor/src/otlp_relay/receiver.rs +++ b/crates/openshell-supervisor/src/otlp_relay/receiver.rs @@ -188,7 +188,17 @@ impl OtlpConnectionServer { "not found; the relay serves POST /v1/traces only", )); } - if request.headers().contains_key(CONTENT_ENCODING) { + // `identity` declares an uncompressed body; every other coding is + // refused rather than decoded. + let compressed = request + .headers() + .get(CONTENT_ENCODING) + .is_some_and(|value| { + !value + .to_str() + .is_ok_and(|coding| coding.trim().eq_ignore_ascii_case("identity")) + }); + if compressed { return Ok(rejected( StatusCode::UNSUPPORTED_MEDIA_TYPE, "compressed body", @@ -291,7 +301,7 @@ impl OtlpConnectionServer { return Ok(rejected( StatusCode::PAYLOAD_TOO_LARGE, "enriched body exceeds limit", - "request body exceeds the 2 MiB limit after attribution", + "request exceeds the relay's batch limit after attribution", )); } @@ -309,11 +319,13 @@ impl OtlpConnectionServer { } Err(SendError::BufferFull) => { self.counters.record_rejection(); - Ok(unavailable()) + Ok(unavailable( + "relay buffer is full; retry after the Retry-After interval", + )) } Err(SendError::BufferClosed) => { debug!("OTLP relay refused an export: relay is shutting down"); - Ok(unavailable()) + Ok(unavailable("relay is shutting down")) } } } @@ -407,11 +419,11 @@ fn rejected(status: StatusCode, reason: &str, message: &str) -> Response Response> { +fn unavailable(message: &str) -> Response> { let mut response = status_response( StatusCode::SERVICE_UNAVAILABLE, tonic::Code::Unavailable, - "relay buffer is full; retry after the Retry-After interval", + message, ); response.headers_mut().insert( RETRY_AFTER, @@ -710,6 +722,24 @@ mod tests { ); } + #[tokio::test] + async fn identity_content_encoding_is_accepted() { + let (server, _handle, mut rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + let head = send( + &mut client, + &request_with_headers( + "POST", + "/v1/traces", + &[("Content-Type", PROTOBUF), ("Content-Encoding", "Identity")], + &body(), + ), + ) + .await; + assert_eq!(head.status, 200); + assert!(rx.recv().await.is_some()); + } + #[tokio::test(start_paused = true)] async fn stalled_body_is_timed_out_and_frees_its_slot() { let (server, _handle, _rx) = start(BUFFER_CAPACITY); @@ -1159,6 +1189,7 @@ mod tests { ) .await; assert_eq!(head.status, 503); + assert_eq!(head.rpc_status().message, "relay is shutting down"); let rejections = server.counters.snapshot().rejected; assert_eq!(rejections, 0, "shutdown refusals are not back-pressure"); } diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 314c3542eb..d55c35d1e9 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -489,11 +489,13 @@ Metrics include `openshell_ocsf_otlp_queued_total`, `openshell_ocsf_otlp_exporte ### Agent trace relay -With `[openshell.gateway.otlp].endpoint` set (Helm: `server.otlp.endpoint`), each sandbox also forwards the workload's own traces to the same collector. The sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, the traces-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` and `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL`, and `OTEL_EXPORTER_OTLP_COMPRESSION=none` (plus its traces-specific variant) in every workload process, so a standard OpenTelemetry SDK exports without configuration and an image cannot redirect traces around the relay. If the sandbox creation request names its own `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, nothing is injected and the caller's OpenTelemetry variables stand: that agent exports where it was told to, subject to the sandbox's network policy, and its traces carry no relay attribution. An environment passed to `openshell sandbox exec` is applied last and may override the injected values for that command. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over OTLP/gRPC. The collector's address is never visible inside the sandbox. +With `[openshell.gateway.otlp].endpoint` set (Helm: `server.otlp.endpoint`), each sandbox also forwards the workload's own traces to the same collector. While the relay is active, the sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` in every workload process, so a standard OpenTelemetry SDK exports without configuration. These are lowest-precedence defaults. They are not set when the sandbox creation request, the image `ENV`, or an environment passed to `openshell sandbox exec` names `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, and an endpoint the workload sets itself, for example with a shell `export`, takes precedence over them. A workload that exports elsewhere does so subject to the sandbox's network policy, and its traces carry no relay attribution. Provider credentials cannot redirect traces. Without a configured collector, nothing is injected. Because the generic endpoint is set, an SDK that also exports metrics or logs sends them to the relay, which answers 404 to those paths. Disable those exporters or point them elsewhere with `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` and `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over OTLP/gRPC. The collector's address is never visible inside the sandbox. Every forwarded resource carries `openshell.sandbox.id`, with the same value as the supervisor's own spans, and `openshell.telemetry.source=agent`. Values the agent supplies for those two keys are replaced at the resource level; span and scope attributes pass through unchanged, so filter and join on the resource attributes. Filter on `openshell.telemetry.source` to separate agent traces from infrastructure traces, and join on `openshell.sandbox.id` to correlate them. -Agents must export traces as OTLP/HTTP protobuf with compression disabled. The relay answers 415 to JSON bodies and to any `Content-Encoding`, 404 to the metrics and logs paths, 413 to bodies larger than 2 MiB, to requests with more than 512 `ResourceSpans` entries, and to requests whose resource attributes exceed 8192 in number or 256 KiB in total, 408 to requests whose body does not arrive within 30 seconds, and 503 with `Retry-After` while its buffer of 32 batches is full, which standard SDKs retry with backoff. Error responses carry a protobuf `google.rpc.Status` whose message states the reason. Without a configured collector, connections to the relay address are refused immediately. The relay keeps serving `openshell sandbox exec` sessions after the main process exits and flushes once more when the sandbox is torn down. +Agents must export traces as OTLP/HTTP protobuf with compression disabled. The relay answers 415 to JSON bodies and to any `Content-Encoding` other than `identity`, 404 to the metrics and logs paths, 413 to bodies larger than 2 MiB, to requests with more than 512 `ResourceSpans` entries, and to requests whose resource attributes exceed 8192 in number or 256 KiB in total, 408 to requests whose body does not arrive within 30 seconds, and 503 with `Retry-After` while its buffer of 32 batches is full, which standard SDKs retry with backoff. Error responses carry a protobuf `google.rpc.Status` whose message states the reason. Without a configured collector, connections to the relay address are refused immediately. + +Delivery is best-effort. The relay answers 200 once a batch is buffered and makes one export attempt per batch, so a batch is lost if the collector is unavailable at that moment. The supervisor logs the first failed export and a periodic summary of failures. Before the sandbox reports that its main process exited, the relay flushes buffered batches for up to 5 seconds, and skips that wait when its most recent export failed. It keeps serving `openshell sandbox exec` sessions after the main process exits and flushes once more when the sandbox is torn down. The relayed hop does not yet support TLS or authentication to the collector; an `https` endpoint leaves the relay inactive with a logged reason. Protect the hop at the network layer, for example with a collector-side ingress policy. diff --git a/e2e/rust/tests/otlp_relay.rs b/e2e/rust/tests/otlp_relay.rs index e8625aa7f0..5eff3e541d 100644 --- a/e2e/rust/tests/otlp_relay.rs +++ b/e2e/rust/tests/otlp_relay.rs @@ -56,11 +56,13 @@ raise SystemExit(0 if response.status_code == 200 else 1) ) } -/// Same export, but reports the failure class instead of raising, for the -/// inert case. +/// For the inert case: reports whether an endpoint was injected, then posts +/// to the relay address directly and reports the failure class instead of +/// raising. const INERT_SCRIPT: &str = r#" import os, requests -url = os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] + "/v1/traces" +print("ENDPOINT", os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "unset")) +url = "http://192.0.0.8:4318/v1/traces" try: response = requests.post(url, data=b"", headers={"Content-Type": "application/x-protobuf"}, timeout=3) print("STATUS", response.status_code) @@ -319,6 +321,10 @@ async fn relay_is_inert_without_collector() { .await .expect("sandbox attempts an export without a collector"); let output = sandbox.create_output.clone(); + assert!( + output.contains("ENDPOINT unset"), + "no relay endpoint is injected while the relay is inactive:\n{output}" + ); assert!( output.contains("ERROR ConnectionError"), "the export must fail with a connection error, not a timeout:\n{output}" diff --git a/proto/sandbox_protocol.proto b/proto/sandbox_protocol.proto index 40a35cf096..3973e7db43 100644 --- a/proto/sandbox_protocol.proto +++ b/proto/sandbox_protocol.proto @@ -255,6 +255,10 @@ message AgentSpec { // Omitted or zero means no timeout; otherwise whole nonnegative seconds. google.protobuf.Duration timeout = 4; bool interactive = 5; + // True while the supervisor relays agent OTLP traces, so the runtime may + // point workload SDKs at the relay by default. False keeps the workload + // environment unchanged. + bool otlp_relay_active = 6; } message ExecSpec { string program = 1; From 80a2f8a76cbe76b427446f3e39a155d9d55fcb17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20Hu=C3=9F?= Date: Fri, 23 Oct 2026 10:48:24 +0200 Subject: [PATCH 5/6] feat(supervisor): gate the agent trace relay behind [otlp.agent] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The relay started whenever a sandbox supervisor had a trace endpoint, so enabling gateway trace export also changed every workload's environment. Make it opt-in: `[openshell.gateway.otlp.agent]` with `enabled = true` turns it on, and its optional `endpoint` falls back to the shared `[openshell.gateway.otlp]` endpoint, like the OCSF table. The gateway validates the endpoint and drivers pass it to supervisors as OPENSHELL_OTLP_AGENT_ENDPOINT, separate from OPENSHELL_OTLP_ENDPOINT. Docker, Podman (including the Podman Machine host alias), Kubernetes and VM drivers carry it, in-process and as standalone binaries. The shared Kubernetes supervisor still does not relay. The relay now acknowledges OTLP metrics and logs exports with an empty success and discards them, so agent SDKs that export every signal to the generic endpoint stop retrying. Discards are counted and logged once. Local dev gateway tasks, the Skaffold values and the e2e harness enable the table; OPENSHELL_E2E_OTLP_AGENT=0 runs the inert case with trace export still on. Signed-off-by: Roland Huß --- .agents/skills/helm-dev-environment/SKILL.md | 7 +- crates/openshell-core/src/sandbox_env.rs | 6 + crates/openshell-driver-docker/src/lib.rs | 23 +++- crates/openshell-driver-docker/src/main.rs | 6 + crates/openshell-driver-docker/src/tests.rs | 11 +- .../openshell-driver-kubernetes/src/config.rs | 5 + .../openshell-driver-kubernetes/src/driver.rs | 26 +++- .../openshell-driver-kubernetes/src/main.rs | 6 + crates/openshell-driver-podman/src/config.rs | 5 + .../openshell-driver-podman/src/container.rs | 22 ++++ crates/openshell-driver-podman/src/driver.rs | 35 +++++ crates/openshell-driver-podman/src/main.rs | 6 + crates/openshell-driver-vm/src/driver.rs | 12 ++ crates/openshell-driver-vm/src/main.rs | 6 + crates/openshell-gateway/src/lib.rs | 12 ++ crates/openshell-gateway/src/vm.rs | 42 +++++- crates/openshell-server/src/cli.rs | 1 + crates/openshell-server/src/config_file.rs | 121 ++++++++++++++++-- crates/openshell-server/src/otel_tracing.rs | 30 ++++- .../openshell-supervisor-network/src/proxy.rs | 4 +- crates/openshell-supervisor/README.md | 2 +- .../src/backend_setup/tests.rs | 2 +- .../src/legacy_startup_regression_tests.rs | 2 +- crates/openshell-supervisor/src/lib.rs | 8 +- crates/openshell-supervisor/src/main.rs | 32 ++++- .../src/otlp_relay/mod.rs | 37 +++++- .../src/otlp_relay/receiver.rs | 70 +++++++++- .../openshell-supervisor/src/shared_host.rs | 2 +- deploy/helm/openshell/ci/values-skaffold.yaml | 2 + docs/how-it-works/gateways/configuration.mdx | 20 ++- e2e/rust/tests/otlp_relay.rs | 14 +- e2e/support/podman-gateway-config.sh | 5 + e2e/with-docker-gateway.sh | 5 + skills/debug-openshell-cluster/SKILL.md | 2 +- tasks/scripts/gateway-docker.sh | 7 +- tasks/scripts/gateway-podman.sh | 7 +- tasks/scripts/gateway-vm.sh | 7 +- tasks/scripts/gateway.sh | 3 + 38 files changed, 556 insertions(+), 57 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index bd499f4030..e8b4bf654e 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -170,9 +170,10 @@ trace-context annotation. The same command exposes OTLP/gRPC on `http://127.0.0.1:4317` and, when deployed, the Kubernetes gateway on `http://127.0.0.1:8090`. The local `gateway:docker`, `gateway:podman`, and `gateway:vm` tasks detect the collector listener at startup and enable trace -export only while it is reachable. Spans an agent exports from inside a sandbox -arrive through the supervisor's relay with `openshell.telemetry.source=agent` -and share `openshell.sandbox.id` with that sandbox's supervisor spans. +export, OCSF log export, and the agent trace relay only while it is reachable. +Spans an agent exports from inside a sandbox arrive through the supervisor's +relay with `openshell.telemetry.source=agent` and share `openshell.sandbox.id` +with that sandbox's supervisor spans. The Skaffold profile for HA reverse-proxy development is available from `deploy/helm/openshell/`: diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 6aa203deda..009ebad7fe 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -32,6 +32,12 @@ pub const OTLP_ENDPOINT: &str = "OPENSHELL_OTLP_ENDPOINT"; /// OTLP/gRPC collector endpoint for supervisor OCSF event export. pub const OTLP_OCSF_ENDPOINT: &str = "OPENSHELL_OTLP_OCSF_ENDPOINT"; +/// OTLP/gRPC collector endpoint that supervisors relay workload traces to. +/// +/// Set only when the gateway enables `[openshell.gateway.otlp.agent]`; absent +/// keeps the relay inactive and leaves the workload environment unchanged. +pub const OTLP_AGENT_ENDPOINT: &str = "OPENSHELL_OTLP_AGENT_ENDPOINT"; + /// OCSF schema version that supervisors downgrade exported events to. pub const OTLP_OCSF_SCHEMA_VERSION: &str = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION"; diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index c306377141..d244dc5c0c 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -212,6 +212,11 @@ pub struct DockerComputeConfig { #[serde(skip)] pub supervisor_ocsf_export: Option, + /// OTLP/gRPC collector that supervisors relay workload traces to. The + /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it. + #[serde(skip)] + pub supervisor_otlp_agent_endpoint: Option, + /// Gateway name passed to supervisors for their telemetry. The gateway /// supplies its own name; driver TOML cannot set it. #[serde(skip)] @@ -319,6 +324,7 @@ impl Default for DockerComputeConfig { supervisor_network_mode: DockerSupervisorNetworkMode::Host, host_gateway_ip: None, supervisor_ocsf_export: None, + supervisor_otlp_agent_endpoint: None, supervisor_gateway_name: None, sandbox_runtime_image: None, supervisor_bin: None, @@ -360,6 +366,7 @@ struct DockerDriverRuntimeConfig { bridge_default_endpoint: Option, host_gateway_ip: Option, supervisor_ocsf_export: Option, + supervisor_otlp_agent_endpoint: Option, supervisor_gateway_name: Option, ssh_socket_path: String, guest_tls: Option, @@ -1018,6 +1025,9 @@ impl DockerComputeDriver { bridge_default_endpoint, host_gateway_ip, supervisor_ocsf_export: docker_config.supervisor_ocsf_export.clone(), + supervisor_otlp_agent_endpoint: docker_config + .supervisor_otlp_agent_endpoint + .clone(), supervisor_gateway_name: docker_config.supervisor_gateway_name.clone(), ssh_socket_path: docker_config.ssh_socket_path.clone(), guest_tls, @@ -5830,7 +5840,8 @@ fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap } /// Environment that lets the supervisor export spans, joining the current -/// trace, and OCSF events, identifying the gateway that launched it. +/// trace, OCSF events, and relayed workload traces, identifying the gateway +/// that launched it. fn supervisor_otlp_environment(config: &DockerDriverRuntimeConfig) -> Vec { let traces = config.supervisor_otlp_endpoint.iter().flat_map(|endpoint| { std::iter::once((openshell_core::sandbox_env::OTLP_ENDPOINT, endpoint.clone())) @@ -5840,12 +5851,22 @@ fn supervisor_otlp_environment(config: &DockerDriverRuntimeConfig) -> Vec, + /// OTLP/gRPC collector endpoint that supervisors relay workload traces + /// to. Absent keeps the agent trace relay off. + #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")] + otlp_agent_endpoint: Option, + #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, } @@ -90,6 +95,7 @@ async fn main() -> Result<()> { docker_config.supervisor_image = Some(image); } docker_config.supervisor_otlp_endpoint = args.otlp_endpoint.clone(); + docker_config.supervisor_otlp_agent_endpoint = args.otlp_agent_endpoint.clone(); docker_config.supervisor_gateway_name = args.gateway_name.clone(); docker_config.supervisor_ocsf_export = args.otlp_ocsf_endpoint.clone().map(|endpoint| { openshell_core::sandbox_env::OcsfOtlpExport { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 2fcc7b8690..19cd6d3ecd 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -330,6 +330,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { bridge_default_endpoint: Some("https://host.docker.internal:8443".to_string()), host_gateway_ip: None, supervisor_ocsf_export: None, + supervisor_otlp_agent_endpoint: None, supervisor_gateway_name: None, ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), guest_tls: Some(DockerGuestTlsPaths { @@ -1513,7 +1514,8 @@ fn docker_child_environment_strips_supervisor_control_keys() { #[test] fn supervisor_otlp_environment_carries_only_configured_signals() { use openshell_core::sandbox_env::{ - GATEWAY_NAME, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, OTLP_OCSF_SCHEMA_VERSION, OcsfOtlpExport, + GATEWAY_NAME, OTLP_AGENT_ENDPOINT, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, + OTLP_OCSF_SCHEMA_VERSION, OcsfOtlpExport, }; let mut config = runtime_config(); @@ -1538,6 +1540,13 @@ fn supervisor_otlp_environment_carries_only_configured_signals() { ] ); + config.supervisor_ocsf_export = None; + config.supervisor_otlp_agent_endpoint = Some("http://agents:4317".to_string()); + assert_eq!( + supervisor_otlp_environment(&config), + [format!("{OTLP_AGENT_ENDPOINT}=http://agents:4317")] + ); + config.supervisor_gateway_name = Some("prod-west".to_string()); assert_eq!( supervisor_otlp_environment(&config).last(), diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 598e0b20ce..a983459477 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -372,6 +372,10 @@ pub struct KubernetesComputeConfig { /// export; driver TOML cannot set it. #[serde(skip)] pub supervisor_ocsf_export: Option, + /// OTLP/gRPC collector that supervisors relay workload traces to. The + /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it. + #[serde(skip)] + pub supervisor_otlp_agent_endpoint: Option, /// Gateway name passed to supervisors for their telemetry. The gateway /// supplies its own name; driver TOML cannot set it. @@ -502,6 +506,7 @@ impl Default for KubernetesComputeConfig { grpc_endpoint: String::new(), supervisor_otlp_endpoint: None, supervisor_ocsf_export: None, + supervisor_otlp_agent_endpoint: None, supervisor_gateway_name: None, ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), client_tls_secret_name: String::new(), diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 91a1ff416b..90e71bb635 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -912,7 +912,8 @@ impl std::fmt::Debug for KubernetesComputeDriver { impl KubernetesComputeDriver { /// Environment that lets a supervisor export spans, joining the current - /// trace, and OCSF events, identifying the gateway that launched it. + /// trace, OCSF events, and relayed workload traces, identifying the + /// gateway that launched it. fn supervisor_otlp_environment(&self) -> Vec<(&'static str, String)> { let mut environment = Vec::new(); if let Some(endpoint) = self.config.supervisor_otlp_endpoint.as_deref() { @@ -925,6 +926,12 @@ impl KubernetesComputeDriver { if let Some(export) = &self.config.supervisor_ocsf_export { environment.extend(export.environment()); } + if let Some(endpoint) = &self.config.supervisor_otlp_agent_endpoint { + environment.push(( + openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT, + endpoint.clone(), + )); + } if let Some(name) = &self.config.supervisor_gateway_name { environment.push((openshell_core::sandbox_env::GATEWAY_NAME, name.clone())); } @@ -1078,10 +1085,11 @@ impl KubernetesComputeDriver { } if shared_supervisor.is_some() && (config.supervisor_otlp_endpoint.is_some() - || config.supervisor_ocsf_export.is_some()) + || config.supervisor_ocsf_export.is_some() + || config.supervisor_otlp_agent_endpoint.is_some()) { warn!( - "the gateway supervisor topology does not export supervisor OTLP traces or OCSF events; shared sandbox instances keep console logs and gateway log push" + "the gateway supervisor topology does not export supervisor OTLP traces or OCSF events or relay workload traces; shared sandbox instances keep console logs and gateway log push" ); } let driver = Self { @@ -8757,6 +8765,18 @@ mod tests { ] ); + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { + supervisor_otlp_agent_endpoint: Some("http://agents:4317".into()), + ..Default::default() + }); + assert_eq!( + driver.supervisor_otlp_environment(), + [( + openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT, + "http://agents:4317".to_string() + )] + ); + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig { supervisor_gateway_name: Some("prod-west".into()), ..Default::default() diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 209d286370..1cb70c6955 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -57,6 +57,11 @@ struct Args { )] otlp_ocsf_schema_version: Option, + /// OTLP/gRPC collector endpoint that supervisors relay workload traces + /// to. Absent keeps the agent trace relay off. + #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")] + otlp_agent_endpoint: Option, + #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, @@ -298,6 +303,7 @@ async fn main() -> Result<()> { proxy_ca_bundle: args.proxy_ca_bundle, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), supervisor_otlp_endpoint: args.otlp_endpoint.clone(), + supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), supervisor_gateway_name: args.gateway_name.clone(), supervisor_ocsf_export: args.otlp_ocsf_endpoint.clone().map(|endpoint| { openshell_core::sandbox_env::OcsfOtlpExport { diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 2aa8b9c0d1..712f48c325 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -51,6 +51,10 @@ pub struct PodmanComputeConfig { /// export; driver TOML cannot set it. #[serde(skip)] pub supervisor_ocsf_export: Option, + /// OTLP/gRPC collector that supervisors relay workload traces to. The + /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it. + #[serde(skip)] + pub supervisor_otlp_agent_endpoint: Option, /// Gateway name passed to supervisors for their telemetry. The gateway /// supplies its own name; driver TOML cannot set it. @@ -540,6 +544,7 @@ impl Default for PodmanComputeConfig { grpc_endpoint: String::new(), supervisor_otlp_endpoint: None, supervisor_ocsf_export: None, + supervisor_otlp_agent_endpoint: None, supervisor_gateway_name: None, gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 8e7a9f4228..ccd365872d 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -717,6 +717,12 @@ fn build_env( .map(|(name, value)| (name.to_string(), value)), ); } + if let Some(endpoint) = &config.supervisor_otlp_agent_endpoint { + env.insert( + openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT.into(), + endpoint.clone(), + ); + } if let Some(name) = &config.supervisor_gateway_name { env.insert( openshell_core::sandbox_env::GATEWAY_NAME.into(), @@ -2774,6 +2780,22 @@ mod tests { spec["env"][openshell_core::sandbox_env::OTLP_OCSF_SCHEMA_VERSION], "1.1" ); + assert!( + spec["env"] + .get(openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT) + .is_none(), + "the agent trace relay stays off unless the gateway enables it" + ); + + let config = PodmanComputeConfig { + supervisor_otlp_agent_endpoint: Some("http://agents:4317".to_string()), + ..test_config() + }; + let spec = build_container_spec(&sandbox, &config); + assert_eq!( + spec["env"][openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT], + "http://agents:4317" + ); let config = PodmanComputeConfig { supervisor_gateway_name: Some("prod-west".to_string()), diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 06d4d7b6fd..bb4ec70f50 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -87,6 +87,16 @@ fn select_supervisor_otlp_endpoint( .map(|endpoint| supervisor_collector_endpoint(endpoint, environment)) } +fn select_supervisor_otlp_agent_endpoint( + config: &PodmanComputeConfig, + environment: PodmanEndpointEnvironment, +) -> Option { + config + .supervisor_otlp_agent_endpoint + .as_deref() + .map(|endpoint| supervisor_collector_endpoint(endpoint, environment)) +} + fn select_supervisor_ocsf_export( config: &PodmanComputeConfig, environment: PodmanEndpointEnvironment, @@ -561,6 +571,8 @@ impl PodmanComputeDriver { select_supervisor_otlp_endpoint(&config, PodmanEndpointEnvironment::current()); config.supervisor_ocsf_export = select_supervisor_ocsf_export(&config, PodmanEndpointEnvironment::current()); + config.supervisor_otlp_agent_endpoint = + select_supervisor_otlp_agent_endpoint(&config, PodmanEndpointEnvironment::current()); if endpoint_was_selected { info!( grpc_endpoint = %config.grpc_endpoint, @@ -2864,6 +2876,29 @@ mod tests { ); } + #[test] + fn supervisor_agent_endpoint_uses_host_alias_for_loopback_on_podman_machine() { + let config = PodmanComputeConfig { + supervisor_otlp_agent_endpoint: Some("http://127.0.0.1:4317".to_string()), + ..PodmanComputeConfig::default() + }; + assert_eq!( + select_supervisor_otlp_agent_endpoint( + &config, + PodmanEndpointEnvironment::PodmanMachine + ) + .as_deref(), + Some("http://host.containers.internal:4317") + ); + assert_eq!( + select_supervisor_otlp_agent_endpoint( + &PodmanComputeConfig::default(), + PodmanEndpointEnvironment::PodmanMachine + ), + None + ); + } + #[test] fn supervisor_otlp_endpoint_keeps_routable_hosts_on_podman_machine() { assert_eq!( diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index c0298197ad..53df64e112 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -54,6 +54,11 @@ struct Args { )] otlp_ocsf_schema_version: Option, + /// OTLP/gRPC collector endpoint that supervisors relay workload traces + /// to. Absent keeps the agent trace relay off. + #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")] + otlp_agent_endpoint: Option, + #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, @@ -235,6 +240,7 @@ async fn main() -> Result<()> { image_pull_policy: args.sandbox_image_pull_policy, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), supervisor_otlp_endpoint: args.otlp_endpoint.clone(), + supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), supervisor_gateway_name: args.gateway_name.clone(), supervisor_ocsf_export: args.otlp_ocsf_endpoint.clone().map(|endpoint| { openshell_core::sandbox_env::OcsfOtlpExport { diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 0a1176e7c9..62671a8f16 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -309,6 +309,10 @@ pub struct VmDriverConfig { /// export; driver TOML cannot set it. #[serde(skip)] pub supervisor_ocsf_export: Option, + /// OTLP/gRPC collector that supervisors relay workload traces to. The + /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it. + #[serde(skip)] + pub supervisor_otlp_agent_endpoint: Option, /// Gateway name passed to supervisors for their telemetry. pub supervisor_gateway_name: Option, pub state_dir: PathBuf, @@ -369,6 +373,10 @@ impl std::fmt::Debug for VmDriverConfig { .field("grpc_endpoint", &self.grpc_endpoint) .field("supervisor_otlp_endpoint", &self.supervisor_otlp_endpoint) .field("supervisor_ocsf_export", &self.supervisor_ocsf_export) + .field( + "supervisor_otlp_agent_endpoint", + &self.supervisor_otlp_agent_endpoint, + ) .field("supervisor_gateway_name", &self.supervisor_gateway_name) .field("state_dir", &self.state_dir) .field("launcher_bin", &self.launcher_bin) @@ -436,6 +444,7 @@ impl Default for VmDriverConfig { grpc_endpoint: String::new(), supervisor_otlp_endpoint: None, supervisor_ocsf_export: None, + supervisor_otlp_agent_endpoint: None, supervisor_gateway_name: None, allow_driver_config: false, resource_admission: @@ -1048,6 +1057,9 @@ impl VmDriver { if let Some(export) = &self.config.supervisor_ocsf_export { command.envs(export.environment()); } + if let Some(endpoint) = &self.config.supervisor_otlp_agent_endpoint { + command.env(openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT, endpoint); + } if let Some(name) = &self.config.supervisor_gateway_name { command.env(openshell_core::sandbox_env::GATEWAY_NAME, name); } diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index fb7c4accf2..a30c7b83d5 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -112,6 +112,11 @@ struct Args { )] otlp_ocsf_schema_version: Option, + /// OTLP/gRPC collector endpoint that supervisors relay workload traces + /// to. Absent keeps the agent trace relay off. + #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")] + otlp_agent_endpoint: Option, + #[arg(long, env = "OPENSHELL_GATEWAY_NAME")] gateway_name: Option, @@ -319,6 +324,7 @@ async fn main() -> Result<()> { .grpc_endpoint .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, supervisor_otlp_endpoint: args.otlp_endpoint.clone(), + supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(), supervisor_gateway_name: args.gateway_name.clone(), supervisor_ocsf_export: args.otlp_ocsf_endpoint.clone().map(|endpoint| { openshell_core::sandbox_env::OcsfOtlpExport { diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 3f060ae9f7..5d7e961b4a 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -284,6 +284,10 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { .and_then(OtlpConfig::traces_endpoint) .map(str::to_string); config.supervisor_ocsf_export = supervisor_ocsf_export(context.otlp_config()); + config.supervisor_otlp_agent_endpoint = context + .otlp_config() + .and_then(OtlpConfig::agent_endpoint) + .map(str::to_string); config.supervisor_gateway_name = Some(context.gateway_name().to_string()); // With the gateway supervisor topology, the gateway owns the shared // supervisor's assignments and token; the driver adapts them to @@ -362,6 +366,10 @@ impl openshell_server::ComputeDriverFactory for DockerFactory { .and_then(OtlpConfig::traces_endpoint) .map(str::to_string); config.supervisor_ocsf_export = supervisor_ocsf_export(context.otlp_config()); + config.supervisor_otlp_agent_endpoint = context + .otlp_config() + .and_then(OtlpConfig::agent_endpoint) + .map(str::to_string); config.supervisor_gateway_name = Some(context.gateway_name().to_string()); require_guest_tls_for_local_driver(&context, "docker")?; apply_guest_tls(&mut config.guest_tls_ca, context.guest_tls_ca()); @@ -429,6 +437,10 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { .and_then(OtlpConfig::traces_endpoint) .map(str::to_string); config.supervisor_ocsf_export = supervisor_ocsf_export(context.otlp_config()); + config.supervisor_otlp_agent_endpoint = context + .otlp_config() + .and_then(OtlpConfig::agent_endpoint) + .map(str::to_string); config.supervisor_gateway_name = Some(context.gateway_name().to_string()); require_guest_tls_for_local_driver(&context, "podman")?; apply_guest_tls(&mut config.guest_tls_ca, context.guest_tls_ca()); diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 149b143ee1..9668a33fab 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -681,9 +681,13 @@ fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gat .arg(version.as_str()); } } + let agent_endpoint = otlp_config.and_then(OtlpConfig::agent_endpoint); + if let Some(endpoint) = agent_endpoint { + command.arg("--otlp-agent-endpoint").arg(endpoint); + } // The driver names its own spans with the gateway name, and supervisors - // name their spans and OCSF records with it. - if traces_endpoint.is_some() || ocsf.is_some() { + // name their spans, OCSF records, and relayed traces with it. + if traces_endpoint.is_some() || ocsf.is_some() || agent_endpoint.is_some() { command.arg("--gateway-name").arg(gateway_name); } } @@ -837,6 +841,7 @@ mod tests { endpoint: "http://collector.internal:4317".to_string(), }), ocsf: None, + agent: None, }), "production-us-west", ); @@ -869,6 +874,7 @@ mod tests { endpoint: "http://siem.internal:4317".to_string(), schema_version: Some(OcsfSchemaVersion::V1_3), }), + agent: None, }), "production-us-west", ); @@ -891,6 +897,38 @@ mod tests { ); } + #[test] + fn vm_driver_command_includes_the_agent_trace_relay() { + let mut command = tokio::process::Command::new("openshell-driver-vm"); + append_otlp_args( + &mut command, + Some(&OtlpConfig { + service_name: None, + traces: None, + ocsf: None, + agent: Some(openshell_server::config_file::AgentOtlpConfig { + endpoint: "http://agents.internal:4317".to_string(), + }), + }), + "production-us-west", + ); + + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args, + [ + "--otlp-agent-endpoint", + "http://agents.internal:4317", + "--gateway-name", + "production-us-west" + ] + ); + } + #[test] fn vm_driver_command_forwards_corporate_proxy_settings() { let mut command = tokio::process::Command::new("openshell-driver-vm"); diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 5b2ce01c13..eca145ea8f 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -749,6 +749,7 @@ async fn run_from_args( info!( traces_endpoint = otlp.traces_endpoint().map(tracing::field::display), ocsf_endpoint = otlp.ocsf_endpoint().map(tracing::field::display), + agent_endpoint = otlp.agent_endpoint().map(tracing::field::display), "OTLP exporting enabled" ); } diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 817a6a02ad..f1aa693a7b 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -288,8 +288,9 @@ impl TryFrom for OcsfLogConfig { /// `[openshell.gateway.otlp]` section and its per-signal tables. /// /// `endpoint` is the collector for every enabled signal, and on its own -/// enables trace export. `[otlp.traces]` overrides the trace endpoint, and -/// `[otlp.ocsf]` with `enabled = true` exports OCSF events as log records, +/// enables trace export. `[otlp.traces]` overrides the trace endpoint, +/// `[otlp.ocsf]` with `enabled = true` exports OCSF events as log records, and +/// `[otlp.agent]` with `enabled = true` relays workload traces, each /// optionally to a different endpoint. SDK tuning knobs are deliberately absent — see /// [`crate::otel_tracing`] for what this table owns and what the `OTEL_*` /// environment variables own. @@ -313,33 +314,55 @@ pub struct OtlpConfig { serialize_with = "serialize_enabled_ocsf" )] pub ocsf: Option, + + /// Relay of workload traces from sandboxes, when enabled. + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_enabled" + )] + pub agent: Option, } /// Writes `enabled = true` so that a serialized config parses back to the /// same export. -fn serialize_enabled_ocsf( - ocsf: &Option, +fn serialize_enabled( + table: &Option, serializer: S, ) -> Result { #[derive(Serialize)] - struct Enabled<'a> { + struct Enabled<'a, T> { enabled: bool, #[serde(flatten)] - ocsf: &'a OcsfOtlpConfig, + table: &'a T, } - ocsf.as_ref() - .map(|ocsf| Enabled { + table + .as_ref() + .map(|table| Enabled { enabled: true, - ocsf, + table, }) .serialize(serializer) } +fn serialize_enabled_ocsf( + ocsf: &Option, + serializer: S, +) -> Result { + serialize_enabled(ocsf, serializer) +} + impl OtlpConfig { /// The collector that receives traces, when trace export is enabled. pub fn traces_endpoint(&self) -> Option<&str> { self.traces.as_ref().map(|traces| traces.endpoint.as_str()) } + + /// The collector that receives relayed workload traces, when the agent + /// trace relay is enabled. + pub fn agent_endpoint(&self) -> Option<&str> { + self.agent.as_ref().map(|agent| agent.endpoint.as_str()) + } } /// `[openshell.gateway.otlp.traces]`. @@ -361,6 +384,14 @@ pub struct OcsfOtlpConfig { pub schema_version: Option, } +/// `[openshell.gateway.otlp.agent]`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentOtlpConfig { + /// OTLP/gRPC collector endpoint that sandbox supervisors forward + /// workload traces to. + pub endpoint: String, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct RawOtlpConfig { @@ -372,6 +403,17 @@ struct RawOtlpConfig { traces: Option, #[serde(default)] ocsf: Option, + #[serde(default)] + agent: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawAgentOtlp { + #[serde(default)] + enabled: bool, + #[serde(default)] + endpoint: Option, } #[derive(Deserialize)] @@ -416,13 +458,26 @@ impl TryFrom for OtlpConfig { }) }) .transpose()?; - if traces.is_none() && ocsf.is_none() { - return Err("otlp needs an endpoint, or otlp.ocsf with enabled = true".to_string()); + let agent = raw + .agent + .filter(|agent| agent.enabled) + .map(|agent| { + Ok::<_, String>(AgentOtlpConfig { + endpoint: endpoint_for("agent", agent.endpoint)?, + }) + }) + .transpose()?; + if traces.is_none() && ocsf.is_none() && agent.is_none() { + return Err( + "otlp needs an endpoint, or otlp.ocsf or otlp.agent with enabled = true" + .to_string(), + ); } Ok(Self { service_name: raw.service_name, traces, ocsf, + agent, }) } } @@ -1332,11 +1387,53 @@ service_name = "openshell-gateway-dev" } } + #[test] + fn otlp_agent_relay_is_off_unless_enabled() { + for agent in ["", "enabled = false\n", "endpoint = 'http://agents:4317'\n"] { + let otlp = otlp(&format!( + "[openshell.gateway.otlp]\nendpoint = 'http://127.0.0.1:4317'\n\n\ + [openshell.gateway.otlp.agent]\n{agent}" + )); + assert_eq!( + otlp.agent_endpoint(), + None, + "enabled the relay with {agent:?}" + ); + } + let otlp = otlp("[openshell.gateway.otlp]\nendpoint = 'http://127.0.0.1:4317'\n"); + assert_eq!(otlp.agent_endpoint(), None); + } + + #[test] + fn otlp_agent_relay_defaults_to_the_shared_endpoint() { + let otlp = otlp( + "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317'\n\n\ + [openshell.gateway.otlp.traces]\nendpoint = 'http://traces:4317'\n\n\ + [openshell.gateway.otlp.agent]\nenabled = true\n", + ); + assert_eq!(otlp.traces_endpoint(), Some("http://traces:4317")); + assert_eq!(otlp.agent_endpoint(), Some("http://shared:4317")); + } + + #[test] + fn otlp_agent_relay_uses_its_own_endpoint() { + let otlp = otlp( + "[openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'http://agents:4317'\n", + ); + assert_eq!(otlp.traces_endpoint(), None); + assert_eq!(otlp.agent_endpoint(), Some("http://agents:4317")); + + let tmp = write_tmp("[openshell.gateway.otlp.agent]\nenabled = true\n"); + let error = load(tmp.path()).expect_err("an enabled relay needs an endpoint"); + assert!(format!("{error:#}").contains("otlp.agent needs an endpoint")); + } + #[test] fn otlp_config_round_trips() { let tmp = write_tmp( "[openshell.gateway.otlp]\nendpoint = 'http://shared:4317'\nservice_name = 'gw'\n\n\ - [openshell.gateway.otlp.ocsf]\nenabled = true\nschema_version = '1.3'\n", + [openshell.gateway.otlp.ocsf]\nenabled = true\nschema_version = '1.3'\n\n\ + [openshell.gateway.otlp.agent]\nenabled = true\nendpoint = 'http://agents:4317'\n", ); let file = load(tmp.path()).unwrap(); let encoded = toml::to_string(&file).unwrap(); diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index 183de04c5b..0d147c0e83 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -118,6 +118,9 @@ fn build_provider( pub struct OtlpExport { traces: Option>, ocsf: Option>, + /// Validated collector that supervisors relay workload traces to. The + /// gateway itself exports nothing for this signal. + agent_endpoint: Option, } /// One signal's exporter and the endpoint it sends to. @@ -176,7 +179,21 @@ impl OtlpExport { }) }) .transpose()?; - Ok(Self { traces, ocsf }) + let agent_endpoint = cfg + .agent_endpoint() + .map(|endpoint| target(endpoint).map(|target| target.endpoint().to_string())) + .transpose()?; + Ok(Self { + traces, + ocsf, + agent_endpoint, + }) + } + + /// The collector that supervisors relay workload traces to, when the + /// agent trace relay is enabled. + pub fn agent_endpoint(&self) -> Option<&str> { + self.agent_endpoint.as_deref() } /// The collector receiving traces, when trace export is enabled. @@ -574,6 +591,7 @@ mod tests { endpoint: endpoint.into(), }), ocsf: None, + agent: None, } } @@ -887,7 +905,15 @@ mod tests { async fn misconfigured_endpoint_disables_export_without_failing_startup() { let mut bad_ocsf = config(); bad_ocsf.ocsf = Some(ocsf_config("definitely not a url")); - for cfg in [with_traces_endpoint("definitely not a url"), bad_ocsf] { + let mut bad_agent = config(); + bad_agent.agent = Some(crate::config_file::AgentOtlpConfig { + endpoint: "definitely not a url".into(), + }); + for cfg in [ + with_traces_endpoint("definitely not a url"), + bad_ocsf, + bad_agent, + ] { let (export, err) = OtlpExport::start(Some(&cfg), GatewayResourceAttributes::default()); assert!( export.is_none(), diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 39a71a5aeb..f5aa8a63a1 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -678,12 +678,12 @@ async fn preauthorize_transparent_open( if RELAY_DENIAL_WARNED.swap(true, Ordering::SeqCst) { debug!( %destination, - "agent OTLP export attempted but no collector is configured" + "agent OTLP export attempted but the agent trace relay is not enabled" ); } else { warn!( %destination, - "agent OTLP export attempted but no collector is configured" + "agent OTLP export attempted but the agent trace relay is not enabled" ); } let _ = completion.send(TcpOpenDecision::Denied(TcpOpenDenial::PolicyDenied)); diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 2cd827d235..630601f2ac 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -90,7 +90,7 @@ Console JSON is installed independently of optional file appenders and uses an I ## OTLP agent trace relay -The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_ENDPOINT`, the same one the supervisor uses for its own spans. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards batches over OTLP/gRPC with a lazy connection. +The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_AGENT_ENDPOINT`. The gateway sets that variable only when `[openshell.gateway.otlp.agent]` is enabled, so trace export alone leaves the relay inactive. Metrics and logs exports to the relay are acknowledged and discarded. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards batches over OTLP/gRPC with a lazy connection. Limits are fixed: 2 MiB per request body, at most 512 `ResourceSpans` entries, 256 KiB of encoded `Resource` fields, and 8192 resource attributes per request (each of these answers 413), a 10 second header-read timeout, 16 concurrent connections, 32 buffered batches, and a 5 second deadline per export attempt. Only the resources are decoded; spans are copied byte for byte, at most two requests at a time. Worst-case memory attributable to the agent is about 67 MiB of buffered batches, plus 32 MiB of bodies awaiting enrichment across the 16 connections, plus about 12 MiB for the two enrichments in progress, plus about 4 MiB for the batch being exported and its encoded copy. A full buffer answers 503 with `Retry-After`. Without a collector endpoint nothing starts and the proxy refuses the relay address. After the agent exits, `run_sandbox_with_backend` flushes buffered batches for up to 5 seconds before it reports the exit to the gateway, then logs a final summary of rejected and failed counts. Flushes skip the wait while the most recent export failed, and batches still pending when the relay is dropped without a flush count as failed. The supervisor tells the sandbox runtime through `AgentSpec::otlp_relay_active` whether the relay runs, and the runtime injects the generic `OTEL_EXPORTER_OTLP_*` defaults only then. diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 88823bf619..751f3483d6 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -705,7 +705,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { provider_workload_api_socket: None, // The binary loads the proxy trust directory from its environment. proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), - otlp_endpoint: None, + agent_otlp_endpoint: None, }; let bindings = crate::SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index 6e90be6537..65d9c4e0c3 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -304,7 +304,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl trace_context: None, provider_workload_api_socket: None, proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), - otlp_endpoint: None, + agent_otlp_endpoint: None, }; let bindings = SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 7c22a6f0a8..c54d582643 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -795,7 +795,7 @@ pub struct SandboxInstanceConfig { pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, /// OTLP/gRPC collector for relayed agent traces; `None` keeps the relay /// inactive. - pub otlp_endpoint: Option, + pub agent_otlp_endpoint: Option, } impl SandboxInstanceConfig { @@ -944,7 +944,7 @@ pub async fn run_sandbox_instance( trace_context, provider_workload_api_socket, proxy_tls, - otlp_endpoint, + agent_otlp_endpoint, } = config; // The owner binds the event context and log sink to the instance's // threads; the instance itself uses the rest. @@ -1164,7 +1164,7 @@ pub async fn run_sandbox_instance( // before attach so the agent spec can tell the boundary whether to point // workload SDKs at it. let (mut otlp_relay, otlp_reserved_destination) = - otlp_relay::activate(otlp_endpoint, sandbox_id.clone()); + otlp_relay::activate(agent_otlp_endpoint, sandbox_id.clone()); // Construct the selected client only after shared policy and credential // setup has completed. Keep live credentials and later CA publication in @@ -4398,7 +4398,7 @@ mod tests { trace_context: None, provider_workload_api_socket: None, proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), - otlp_endpoint: None, + agent_otlp_endpoint: None, }; let bindings = SandboxInstanceBindings::dedicated( &config, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 8fd0382019..2504f95130 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -133,6 +133,11 @@ struct Args { #[arg(long, env = openshell_core::sandbox_env::OTLP_OCSF_SCHEMA_VERSION)] otlp_ocsf_schema_version: Option, + /// OTLP/gRPC collector endpoint that workload traces are relayed to. + /// Absent keeps the relay inactive. + #[arg(long, env = openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT)] + otlp_agent_endpoint: Option, + /// Name of the gateway that launched this sandbox, for telemetry. #[arg(long, env = openshell_core::sandbox_env::GATEWAY_NAME)] gateway_name: Option, @@ -596,7 +601,7 @@ fn main() -> Result<()> { .filter(|path| !path.is_empty()) .map(PathBuf::from), proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::from_environment(), - otlp_endpoint: args.otlp_endpoint.clone(), + agent_otlp_endpoint: args.otlp_agent_endpoint.clone(), }; // Log push already shares these credentials and the logging // layers these OCSF JSON settings. @@ -905,6 +910,31 @@ mod tests { ); } + #[test] + fn agent_relay_endpoint_is_separate_from_trace_export() { + let args = Args::try_parse_from([ + "openshell-supervisor", + "--otlp-endpoint", + "http://traces:4317", + ]) + .expect("parse trace export"); + assert_eq!( + args.otlp_agent_endpoint, None, + "trace export alone leaves the relay inactive" + ); + let args = Args::try_parse_from([ + "openshell-supervisor", + "--otlp-agent-endpoint", + "http://agents:4317", + ]) + .expect("parse agent relay"); + assert_eq!( + args.otlp_agent_endpoint.as_deref(), + Some("http://agents:4317") + ); + assert_eq!(args.otlp_endpoint, None); + } + #[tokio::test] async fn ocsf_export_starts_only_with_a_valid_endpoint() { let config = |endpoint| openshell_otel::OtlpExportConfig { diff --git a/crates/openshell-supervisor/src/otlp_relay/mod.rs b/crates/openshell-supervisor/src/otlp_relay/mod.rs index c6bdfa3634..1c304597db 100644 --- a/crates/openshell-supervisor/src/otlp_relay/mod.rs +++ b/crates/openshell-supervisor/src/otlp_relay/mod.rs @@ -8,7 +8,8 @@ //! broker stages that connection to the supervisor, the network proxy hands //! the stream to [`receiver::OtlpConnectionServer`], and the accepted batch //! is enriched with the sandbox id, buffered, and forwarded over OTLP/gRPC to -//! the collector the supervisor already uses for its own spans. +//! the collector the gateway names in `OPENSHELL_OTLP_AGENT_ENDPOINT`. The +//! gateway sets it only when `[openshell.gateway.otlp.agent]` is enabled. pub mod buffer; pub mod enrichment; @@ -94,10 +95,14 @@ pub struct RelayCounters { exported: AtomicU64, rejected: AtomicU64, export_failures: AtomicU64, + /// Metrics and logs exports acknowledged and dropped, since the relay + /// forwards traces only. + discarded_signals: AtomicU64, /// Whether the most recent export attempt failed. Cleared by the next /// success; flushes skip waiting while it is set. last_export_failed: AtomicBool, first_rejection_logged: AtomicBool, + first_discard_logged: AtomicBool, first_failure_logged: AtomicBool, final_summary_logged: AtomicBool, } @@ -120,6 +125,24 @@ impl RelayCounters { } } + /// Records one metrics or logs export that was acknowledged and dropped. + /// Logs the first occurrence only, since an agent SDK sends one per + /// export interval for as long as it runs. + pub fn record_discarded_signal(&self, signal: &'static str) { + self.discarded_signals.fetch_add(1, Ordering::Relaxed); + if !self.first_discard_logged.swap(true, Ordering::Relaxed) { + info!( + signal, + "OTLP relay discarded an agent export: the relay forwards traces only" + ); + } + } + + /// Metrics and logs exports discarded so far. + pub fn discarded_signals(&self) -> u64 { + self.discarded_signals.load(Ordering::Relaxed) + } + /// Records one export attempt that failed or timed out. Warns on the /// first occurrence only. pub fn record_export_failure(&self, endpoint: &str, error: &dyn std::fmt::Display) { @@ -169,7 +192,13 @@ impl RelayCounters { rejected, export_failures, } = self.snapshot(); - info!(exported, rejected, export_failures, "OTLP relay summary"); + info!( + exported, + rejected, + export_failures, + discarded_signals = self.discarded_signals(), + "OTLP relay summary" + ); } /// Logs the final summary once, and only when something was rejected or @@ -192,7 +221,7 @@ impl RelayCounters { /// Input to [`start`]. #[derive(Debug, Clone)] pub struct OtlpRelayConfig { - /// OTLP/gRPC collector URI (`OPENSHELL_OTLP_ENDPOINT`). + /// OTLP/gRPC collector URI (`OPENSHELL_OTLP_AGENT_ENDPOINT`). pub endpoint: String, /// Sandbox identifier attached to every forwarded resource. pub sandbox_id: String, @@ -334,7 +363,7 @@ pub fn activate( .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Disabled, "disabled") - .message("OTLP agent trace relay inactive: no collector endpoint configured") + .message("OTLP agent trace relay inactive: not enabled by the gateway") .build() ); return (None, None); diff --git a/crates/openshell-supervisor/src/otlp_relay/receiver.rs b/crates/openshell-supervisor/src/otlp_relay/receiver.rs index e540d2980c..ea664698db 100644 --- a/crates/openshell-supervisor/src/otlp_relay/receiver.rs +++ b/crates/openshell-supervisor/src/otlp_relay/receiver.rs @@ -181,6 +181,22 @@ impl OtlpConnectionServer { &self, request: Request, ) -> Result>, Infallible> { + // The relay forwards traces only. Agents whose SDK also exports + // metrics or logs to the generic endpoint get an OTLP success, so they + // do not retry forever, and the body is drained without being kept. + if request.method() == Method::POST + && let Some(signal) = discarded_signal(request.uri().path()) + { + // A body that is not delivered in time is abandoned; hyper closes + // the connection because the request was not consumed. + let _ = tokio::time::timeout(BODY_READ_TIMEOUT, drain_body(request.into_body())).await; + self.counters.record_discarded_signal(signal); + return Ok(Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, "application/x-protobuf") + .body(Full::new(Bytes::new())) + .expect("static response builds")); + } if request.method() != Method::POST || request.uri().path() != "/v1/traces" { return Ok(rejected( StatusCode::NOT_FOUND, @@ -342,6 +358,30 @@ enum BodyRead { Failed(hyper::Error), } +/// The OTLP signal a path exports when the relay accepts and discards it. +fn discarded_signal(path: &str) -> Option<&'static str> { + match path { + "/v1/metrics" => Some("metrics"), + "/v1/logs" => Some("logs"), + _ => None, + } +} + +/// Reads and drops a discarded export body, up to [`OVERSIZE_DRAIN_CAP`], so +/// the response is delivered on a connection the client has finished writing +/// to. Nothing is buffered. +async fn drain_body(mut body: Incoming) { + let mut total = 0usize; + while let Some(Ok(frame)) = body.frame().await { + if let Some(data) = frame.data_ref() { + total += data.len(); + if total > OVERSIZE_DRAIN_CAP { + break; + } + } + } +} + /// Reads the body up to [`MAX_BODY_BYTES`]. Once the limit is crossed, or /// when the declared length already exceeds it, the rest is discarded up /// to [`OVERSIZE_DRAIN_CAP`] so the response can be delivered on a @@ -653,10 +693,38 @@ mod tests { async fn other_paths_and_methods_are_not_found() { let (server, _handle, _rx) = start(BUFFER_CAPACITY); let mut client = connect(&server); - let head = send(&mut client, &request("POST", "/v1/metrics", PROTOBUF, b"")).await; + let head = send(&mut client, &request("POST", "/v1/profiles", PROTOBUF, b"")).await; assert_eq!(head.status, 404); let head = send(&mut client, &request("GET", "/v1/traces", PROTOBUF, b"")).await; assert_eq!(head.status, 404); + let head = send(&mut client, &request("GET", "/v1/metrics", PROTOBUF, b"")).await; + assert_eq!(head.status, 404); + } + + #[tokio::test] + async fn metrics_and_logs_are_acknowledged_and_discarded() { + let (server, _handle, mut rx) = start(BUFFER_CAPACITY); + let mut client = connect(&server); + for path in ["/v1/metrics", "/v1/logs"] { + let head = send(&mut client, &request("POST", path, PROTOBUF, &[1, 2, 3])).await; + assert_eq!(head.status, 200, "{path}"); + assert_eq!(head.header("content-type"), Some(PROTOBUF)); + } + // The same connection still serves a trace export afterwards, so the + // discarded bodies were consumed. + let head = send( + &mut client, + &request("POST", "/v1/traces", PROTOBUF, &body()), + ) + .await; + assert_eq!(head.status, 200); + assert!(rx.recv().await.is_some(), "the trace batch is buffered"); + assert_eq!( + server.counters.accepted.load(Ordering::Relaxed), + 1, + "metrics and logs are not buffered" + ); + assert_eq!(server.counters.discarded_signals(), 2); } #[tokio::test] diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index f68195f40a..0f7cbe2e89 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -390,7 +390,7 @@ impl SharedHost { }, // The shared host has no per-instance collector endpoint yet, so // hosted instances do not relay agent traces. - otlp_endpoint: None, + agent_otlp_endpoint: None, }; crate::sandbox_instance_host::sandbox_instance_launch( Box::new(crate::backend_setup::RoutedBackendSetup::new( diff --git a/deploy/helm/openshell/ci/values-skaffold.yaml b/deploy/helm/openshell/ci/values-skaffold.yaml index e34fd2a8eb..1ff2bb2fc4 100644 --- a/deploy/helm/openshell/ci/values-skaffold.yaml +++ b/deploy/helm/openshell/ci/values-skaffold.yaml @@ -7,6 +7,8 @@ gatewayConfig: image_pull_policy: if_not_present openshell.gateway.otlp: endpoint: http://openshell-collector.observability.svc.cluster.local:4317 + openshell.gateway.otlp.agent: + enabled: true openshell.gateway.auth: allow_unauthenticated_users: true diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index d55c35d1e9..2c6dc7115b 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -400,7 +400,7 @@ Allow enough retention and disk space for shipper outages. Graceful shutdown all ## OTLP Export -`[openshell.gateway.otlp]` enables OpenTelemetry export over OTLP/gRPC. Omit the table to disable export. Its `endpoint` is the collector for every signal the gateway exports and, on its own, enables trace export. `[openshell.gateway.otlp.traces]` sends spans to a different collector. [OCSF log export](#ocsf-log-export) is off until `[openshell.gateway.otlp.ocsf]` sets `enabled = true`, and can use its own collector. +`[openshell.gateway.otlp]` enables OpenTelemetry export over OTLP/gRPC. Omit the table to disable export. Its `endpoint` is the collector for every signal the gateway exports and, on its own, enables trace export. `[openshell.gateway.otlp.traces]` sends spans to a different collector. [OCSF log export](#ocsf-log-export) is off until `[openshell.gateway.otlp.ocsf]` sets `enabled = true`, and the [agent trace relay](#agent-trace-relay) is off until `[openshell.gateway.otlp.agent]` sets `enabled = true`. Each can use its own collector. The gateway already uses the Rust `tracing` framework for structured logs sent to stdout and the sandbox log stream. Enabling this section adds an OpenTelemetry layer to the same tracing subscriber. It exports span trees to an OTLP collector without exporting, replacing, or redirecting the existing diagnostic log events. @@ -428,8 +428,8 @@ The gateway passes the trace endpoint to compute drivers and sandbox supervisors For Helm deployments, set `server.otlp.endpoint` to render this table. The optional `server.otlp.serviceName` value overrides the gateway service name; -driver service names remain fixed. Set the `traces` and `ocsf` tables through -`gatewayConfig`, which renders each key as a TOML table: +driver service names remain fixed. Set the `traces`, `ocsf`, and `agent` tables +through `gatewayConfig`, which renders each key as a TOML table: ```yaml gatewayConfig: @@ -437,6 +437,8 @@ gatewayConfig: endpoint: https://traces-backend:4317 openshell.gateway.otlp.ocsf: enabled: true + openshell.gateway.otlp.agent: + enabled: true ``` The local `mise run helm:k3s:create` workflow installs a trace collector and UI, @@ -489,11 +491,19 @@ Metrics include `openshell_ocsf_otlp_queued_total`, `openshell_ocsf_otlp_exporte ### Agent trace relay -With `[openshell.gateway.otlp].endpoint` set (Helm: `server.otlp.endpoint`), each sandbox also forwards the workload's own traces to the same collector. While the relay is active, the sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` in every workload process, so a standard OpenTelemetry SDK exports without configuration. These are lowest-precedence defaults. They are not set when the sandbox creation request, the image `ENV`, or an environment passed to `openshell sandbox exec` names `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, and an endpoint the workload sets itself, for example with a shell `export`, takes precedence over them. A workload that exports elsewhere does so subject to the sandbox's network policy, and its traces carry no relay attribution. Provider credentials cannot redirect traces. Without a configured collector, nothing is injected. Because the generic endpoint is set, an SDK that also exports metrics or logs sends them to the relay, which answers 404 to those paths. Disable those exporters or point them elsewhere with `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` and `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over OTLP/gRPC. The collector's address is never visible inside the sandbox. +Set `enabled = true` in `[openshell.gateway.otlp.agent]` to have each sandbox forward the workload's own traces. It defaults to `false`, so trace export alone never changes the workload environment. Traces go to the table's `endpoint`, or to `[openshell.gateway.otlp]` `endpoint` when it is omitted, and that endpoint must be reachable from sandboxes. + +```toml +[openshell.gateway.otlp.agent] +enabled = true +endpoint = "http://agent-collector.observability.svc:4317" # optional +``` + +While the relay is active, the sandbox runtime sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://192.0.0.8:4318` and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` in every workload process, so a standard OpenTelemetry SDK exports without configuration. These are lowest-precedence defaults. They are not set when the sandbox creation request, the image `ENV`, or an environment passed to `openshell sandbox exec` names `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, and an endpoint the workload sets itself, for example with a shell `export`, takes precedence over them. A workload that exports elsewhere does so subject to the sandbox's network policy, and its traces carry no relay attribution. Provider credentials cannot redirect traces. While the relay is off, nothing is injected. Because the generic endpoint is set, an SDK that also exports metrics or logs sends them to the relay, which answers with an empty OTLP success and discards them. To keep metrics and logs, point those exporters elsewhere with `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` and `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`. The address is a reserved label rather than a network destination: the supervisor serves it inside the sandbox boundary and forwards accepted batches to the collector over OTLP/gRPC. The collector's address is never visible inside the sandbox. Every forwarded resource carries `openshell.sandbox.id`, with the same value as the supervisor's own spans, and `openshell.telemetry.source=agent`. Values the agent supplies for those two keys are replaced at the resource level; span and scope attributes pass through unchanged, so filter and join on the resource attributes. Filter on `openshell.telemetry.source` to separate agent traces from infrastructure traces, and join on `openshell.sandbox.id` to correlate them. -Agents must export traces as OTLP/HTTP protobuf with compression disabled. The relay answers 415 to JSON bodies and to any `Content-Encoding` other than `identity`, 404 to the metrics and logs paths, 413 to bodies larger than 2 MiB, to requests with more than 512 `ResourceSpans` entries, and to requests whose resource attributes exceed 8192 in number or 256 KiB in total, 408 to requests whose body does not arrive within 30 seconds, and 503 with `Retry-After` while its buffer of 32 batches is full, which standard SDKs retry with backoff. Error responses carry a protobuf `google.rpc.Status` whose message states the reason. Without a configured collector, connections to the relay address are refused immediately. +Agents must export traces as OTLP/HTTP protobuf with compression disabled. The relay answers 415 to JSON bodies and to any `Content-Encoding` other than `identity`, 413 to bodies larger than 2 MiB, to requests with more than 512 `ResourceSpans` entries, and to requests whose resource attributes exceed 8192 in number or 256 KiB in total, 408 to requests whose body does not arrive within 30 seconds, and 503 with `Retry-After` while its buffer of 32 batches is full, which standard SDKs retry with backoff. Error responses carry a protobuf `google.rpc.Status` whose message states the reason. While the relay is off, connections to the relay address are refused immediately. Delivery is best-effort. The relay answers 200 once a batch is buffered and makes one export attempt per batch, so a batch is lost if the collector is unavailable at that moment. The supervisor logs the first failed export and a periodic summary of failures. Before the sandbox reports that its main process exited, the relay flushes buffered batches for up to 5 seconds, and skips that wait when its most recent export failed. It keeps serving `openshell sandbox exec` sessions after the main process exits and flushes once more when the sandbox is torn down. diff --git a/e2e/rust/tests/otlp_relay.rs b/e2e/rust/tests/otlp_relay.rs index 5eff3e541d..fe53630044 100644 --- a/e2e/rust/tests/otlp_relay.rs +++ b/e2e/rust/tests/otlp_relay.rs @@ -4,10 +4,11 @@ //! Supervisor-direct OTLP trace relay. //! //! The gateway wrapper writes `[openshell.gateway.otlp].endpoint` from -//! `OPENSHELL_E2E_OTLP_ENDPOINT`. These tests bind a collector stub on that -//! endpoint's port and prove that a span exported from inside a sandbox +//! `OPENSHELL_E2E_OTLP_ENDPOINT` and enables `[openshell.gateway.otlp.agent]` +//! unless `OPENSHELL_E2E_OTLP_AGENT=0`. These tests bind a collector stub on +//! that endpoint's port and prove that a span exported from inside a sandbox //! arrives with the supervisor's attribution. With the variable unset or -//! empty, the inert case runs instead. +//! empty, or with the relay disabled, the inert case runs instead. #![cfg(feature = "e2e-local-container-driver")] @@ -70,10 +71,13 @@ except Exception as error: print("ERROR", type(error).__name__) "#; +/// The collector the relay forwards to, when the harness enabled both OTLP +/// export and the agent trace relay. fn collector_endpoint() -> Option { + let relay_enabled = std::env::var("OPENSHELL_E2E_OTLP_AGENT").as_deref() != Ok("0"); std::env::var("OPENSHELL_E2E_OTLP_ENDPOINT") .ok() - .filter(|value| !value.is_empty()) + .filter(|value| !value.is_empty() && relay_enabled) } fn collector_bind_addr(endpoint: &str) -> SocketAddr { @@ -312,7 +316,7 @@ async fn agent_span_reaches_collector_with_attribution() { #[serial] async fn relay_is_inert_without_collector() { if collector_endpoint().is_some() { - eprintln!("OPENSHELL_E2E_OTLP_ENDPOINT is set; skipping the inert case"); + eprintln!("the agent trace relay is enabled; skipping the inert case"); return; } diff --git a/e2e/support/podman-gateway-config.sh b/e2e/support/podman-gateway-config.sh index a645aaf391..580ad0673d 100755 --- a/e2e/support/podman-gateway-config.sh +++ b/e2e/support/podman-gateway-config.sh @@ -89,6 +89,11 @@ e2e_write_podman_gateway_config() { if [ -n "${OPENSHELL_E2E_OTLP_ENDPOINT:-}" ]; then printf '\n[openshell.gateway.otlp]\n' printf 'endpoint = %s\n' "$(e2e_podman_toml_string "${OPENSHELL_E2E_OTLP_ENDPOINT}")" + # The agent trace relay is opt-in; OPENSHELL_E2E_OTLP_AGENT=0 runs the + # relay's inert case with trace export still on. + if [ "${OPENSHELL_E2E_OTLP_AGENT:-1}" != "0" ]; then + printf '\n[openshell.gateway.otlp.agent]\nenabled = true\n' + fi fi } >>"${output}" } diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 2c9cb0978c..87e998e901 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -619,6 +619,11 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" if [ -n "${OPENSHELL_E2E_OTLP_ENDPOINT:-}" ]; then printf '[openshell.gateway.otlp]\n' printf 'endpoint = %s\n\n' "$(toml_string "${OPENSHELL_E2E_OTLP_ENDPOINT}")" + # The agent trace relay is opt-in; OPENSHELL_E2E_OTLP_AGENT=0 runs the + # relay's inert case with trace export still on. + if [ "${OPENSHELL_E2E_OTLP_AGENT:-1}" != "0" ]; then + printf '[openshell.gateway.otlp.agent]\nenabled = true\n\n' + fi fi printf '[openshell.drivers.docker]\n' if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 9ec64d915f..5b102fd85d 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1266,7 +1266,7 @@ credential failures. | Collector receives no OCSF log records, or `openshell.ocsf.gap` records | `[openshell.gateway.otlp.ocsf]` missing or without `enabled = true`, an unreachable or non-gRPC endpoint, or export failures and queue pressure | Check `[openshell.gateway.otlp.ocsf]` for `enabled = true` and the endpoint it resolves to, the gateway's `OTLP exporting enabled` startup log for `ocsf_endpoint`, export warnings, and `openshell_ocsf_otlp_*` metrics. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | Gateway OCSF JSONL stops growing, has gaps, or is rejected by a SIEM | File errors, queue pressure, shipper falling behind retention, or a schema mismatch | Inspect gateway warnings and `openshell_ocsf_log_*` metrics; check `[openshell.gateway.ocsf_log]`, `schema_version`, directory permissions, free space, per-replica paths, and shipper rotation checkpoints. See the published [gateway configuration reference](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration). | | `BatchSpanProcessor.ExportError` repeatedly reports connection refused on `127.0.0.1:4317` | The local gateway started with OTLP configured but the collector forwarding task later stopped, or the config was created manually | Restart `gateway:docker`, `gateway:podman`, or `gateway:vm` so it re-detects the listener; inspect the generated `gateway.toml` for `[openshell.gateway.otlp]` | -| Sandbox log shows `OTLP agent trace relay inactive: no collector endpoint configured` at startup, then `agent OTLP export attempted but no collector is configured` when the agent exports | The gateway has no `[openshell.gateway.otlp].endpoint`, so the supervisor refuses the workload's OTLP relay address | Set `endpoint` (Helm: `server.otlp.endpoint`) and recreate the sandbox; `openshell logs --source sandbox` then shows `OTLP agent trace relay enabled` | +| Sandbox log shows `OTLP agent trace relay inactive: not enabled by the gateway` at startup, then `agent OTLP export attempted but the agent trace relay is not enabled` when the agent exports | The gateway does not enable `[openshell.gateway.otlp.agent]`, so the supervisor refuses the workload's OTLP relay address | Set `enabled = true` in `[openshell.gateway.otlp.agent]` (Helm: `gatewayConfig."openshell.gateway.otlp.agent".enabled`) with an endpoint from that table or `[openshell.gateway.otlp]`, and recreate the sandbox; `openshell logs --source sandbox` then shows `OTLP agent trace relay enabled` | | Gateway starts but sandbox create fails | Compute driver cannot reach runtime | Docker/Podman/Kubernetes/VM driver logs | | Docker or Podman sandbox never registers | Wrong gateway endpoint, unavailable host networking, or supervisor startup failure | Gateway logs and supervisor container logs | | CPU-only VM startup reports that read-only `/proc` cannot be removed | Older supervisors can infer GPU requirements from host devices | Check gateway, VM driver, and bundled supervisor versions; use matching updated artifacts. CPU-only VMs do not require read-write `/proc`. | diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 40605d05cc..b3e33489a2 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -83,7 +83,7 @@ ensure_docker_runtime_image() { append_local_otlp_config_if_available() { local config_path=$1 if ! port_is_in_use 4317; then - echo "OTLP collector not detected on 127.0.0.1:4317; trace and OCSF log export disabled." + echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled." return fi @@ -94,8 +94,11 @@ endpoint = "http://127.0.0.1:4317" [openshell.gateway.otlp.ocsf] enabled = true + +[openshell.gateway.otlp.agent] +enabled = true EOF - echo "OTLP trace and OCSF log export enabled for http://127.0.0.1:4317." + echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317." } register_gateway_metadata() { diff --git a/tasks/scripts/gateway-podman.sh b/tasks/scripts/gateway-podman.sh index afc7e5404a..c032cdad80 100644 --- a/tasks/scripts/gateway-podman.sh +++ b/tasks/scripts/gateway-podman.sh @@ -115,7 +115,7 @@ port_is_in_use() { append_local_otlp_config_if_available() { local config_path=$1 if ! port_is_in_use 4317; then - echo "OTLP collector not detected on 127.0.0.1:4317; trace and OCSF log export disabled." + echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled." return fi @@ -126,8 +126,11 @@ endpoint = "http://127.0.0.1:4317" [openshell.gateway.otlp.ocsf] enabled = true + +[openshell.gateway.otlp.agent] +enabled = true EOF - echo "OTLP trace and OCSF log export enabled for http://127.0.0.1:4317." + echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317." } register_gateway_metadata() { diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index f6db561d32..beb9eb1235 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -90,7 +90,7 @@ port_is_in_use() { append_local_otlp_config_if_available() { local config_path=$1 if ! port_is_in_use 4317; then - echo "OTLP collector not detected on 127.0.0.1:4317; trace and OCSF log export disabled." + echo "OTLP collector not detected on 127.0.0.1:4317; trace export, OCSF log export, and the agent trace relay disabled." return fi @@ -101,8 +101,11 @@ endpoint = "http://127.0.0.1:4317" [openshell.gateway.otlp.ocsf] enabled = true + +[openshell.gateway.otlp.agent] +enabled = true EOF - echo "OTLP trace and OCSF log export enabled for http://127.0.0.1:4317." + echo "OTLP trace export, OCSF log export, and the agent trace relay enabled for http://127.0.0.1:4317." } invoking_user() { diff --git a/tasks/scripts/gateway.sh b/tasks/scripts/gateway.sh index ffe373d645..3b04ee2684 100644 --- a/tasks/scripts/gateway.sh +++ b/tasks/scripts/gateway.sh @@ -268,6 +268,9 @@ endpoint = "http://127.0.0.1:4317" [openshell.gateway.otlp.ocsf] enabled = true +[openshell.gateway.otlp.agent] +enabled = true + [openshell.gateway.auth] allow_unauthenticated_users = true From 2c6fd429b368b5bac466e9c343c660ec8adfada6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20Hu=C3=9F?= Date: Mon, 26 Oct 2026 11:20:46 +0100 Subject: [PATCH 6/6] docs(supervisor): describe the agent trace relay architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an "Agent traces" section to the architecture page and document the relay in the crate READMEs: the reserved-destination hook in the network supervisor, the OTEL defaults the sandbox runtime injects, and how the Docker, Podman, Kubernetes, and VM drivers pass the agent endpoint. Signed-off-by: Roland Huß --- crates/openshell-driver-docker/README.md | 6 +++++- crates/openshell-driver-kubernetes/README.md | 9 ++++++--- crates/openshell-driver-podman/README.md | 4 ++++ crates/openshell-driver-vm/README.md | 2 +- crates/openshell-sandbox/README.md | 9 +++++++++ crates/openshell-supervisor-network/README.md | 15 +++++++++++++++ crates/openshell-supervisor/README.md | 2 +- docs/about/architecture.mdx | 11 +++++++++++ 8 files changed, 52 insertions(+), 6 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 2f64fd05d1..7f2222bb34 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -11,7 +11,11 @@ When the gateway configures `[openshell.gateway.otlp]`, the in-process driver exports spans to the same OTLP/gRPC collector as `openshell-driver-docker`. The standalone driver accepts `OPENSHELL_OTLP_ENDPOINT`, continues W3C trace context from gateway RPC -metadata, and flushes spans during graceful shutdown. +metadata, and flushes spans during graceful shutdown. When +`[openshell.gateway.otlp.agent]` is enabled, the driver also sets +`OPENSHELL_OTLP_AGENT_ENDPOINT` on the supervisor container, which starts the +agent trace relay. The standalone driver takes that endpoint as +`--otlp-agent-endpoint`. ## Runtime Model diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 7dc2478d34..dc5abcaf48 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -91,7 +91,9 @@ When the gateway exports OTLP traces, the driver sets `OPENSHELL_OTLP_ENDPOINT` on the supervisor Pod to the gateway's endpoint and `TRACEPARENT` to the trace context of the operation that created the Pod. The supervisor exports its spans there and parents its startup span on that -context. The endpoint is not configurable in driver TOML. +context. When `[openshell.gateway.otlp.agent]` is enabled, the driver also +sets `OPENSHELL_OTLP_AGENT_ENDPOINT`, which starts the supervisor's agent trace +relay. Neither endpoint is configurable in driver TOML. The driver creates one namespace-wide `NetworkPolicy` before it releases any workload Pod. It selects every OpenShell workload, denies all workload egress, @@ -257,8 +259,9 @@ fails closed. The topology does not support upstream proxies or provider SPIFFE identity, and the shared supervisor is a single point of failure for the sandboxes it hosts. -Shared sandbox instances export no supervisor OTLP traces or OCSF events; the driver -warns at startup when the gateway configures that export. +Shared sandbox instances export no supervisor OTLP traces or OCSF events and do +not relay agent traces; the driver warns at startup when the gateway configures +any of them. When the setting is removed, reconciliation suspends shared generations, and a restart creates the dedicated generation's boundary Service the shared one never had. diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index f582cb03b3..6d5d7a20b3 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -156,3 +156,7 @@ authenticated health signal. Gateway OTLP configuration continues to export compute-driver spans under the `openshell-driver-podman` service, preserving gateway trace context. +When `[openshell.gateway.otlp.agent]` is enabled, the driver sets +`OPENSHELL_OTLP_AGENT_ENDPOINT` on the supervisor container, which starts the +agent trace relay. On Podman Machine it rewrites a loopback agent endpoint to +the desktop host, as it does for the supervisor's trace endpoint. diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index deb4ca9dae..a0371bacfb 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -70,7 +70,7 @@ By default `mise run gateway:vm`: - Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`. - Places the VM driver state (per-sandbox `overlay.ext4`, image cache, and `run/compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`. - Writes `.cache/gateway-vm/gateway.toml` with `[openshell.drivers.vm].driver_dir = "$PWD/target/debug"` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`. -- Enables OTLP trace export to `http://127.0.0.1:4317` only when a local collector is listening there. Otherwise, it omits the OTLP configuration to avoid repeated export failures. +- Enables OTLP trace export, OCSF log export, and the agent trace relay to `http://127.0.0.1:4317` only when a local collector is listening there. Otherwise, it omits the OTLP configuration to avoid repeated export failures. For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges: diff --git a/crates/openshell-sandbox/README.md b/crates/openshell-sandbox/README.md index 1cde277dd8..9394ccc900 100644 --- a/crates/openshell-sandbox/README.md +++ b/crates/openshell-sandbox/README.md @@ -26,3 +26,12 @@ and dials a replacement when the supervisor starts HTTP/2 on it or the spare closes. Live connections keep the same disconnect, freeze, and recovery-deadline handling as listener mode, and the workload still cannot connect to the control port. + +When `AgentSpec::otlp_relay_active` is set, `child_env::apply_otlp_relay_env` +gives every workload process (main, exec, and SSH) `OTEL_EXPORTER_OTLP_ENDPOINT` +pointing at the relay address and `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, +and removes any inherited `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`. These defaults +apply after the provider variables. If the creation request, the image, or a +per-exec environment names a generic or traces-specific OTLP endpoint, the +runtime injects nothing, and the workload exports to that endpoint under the +sandbox's network policy. diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 7ce403a15e..cf842d5041 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -26,3 +26,18 @@ HTTP client; a token acquired for one sandbox is never served to another. Without a Workload API socket, token grants fail closed. A process hosting several instances serves `/v1/denials` from a per-instance `SandboxInstanceDenialLog` instead of the shared shorthand log files. + +## Reserved destinations + +`preauthorize_transparent_open` matches a staged TCP open against the reserved +OTLP relay address (`openshell_core::sandbox_env::OTLP_RELAY_ADDR`) before host +mapping, policy evaluation, and SSRF checks. The address is an unroutable +label, so the proxy never dials it. When `run_networking` receives a +`ReservedDestination`, the proxy hands the workload stream to its +`ReservedStreamHandler`; a handler that refuses, for example at its connection +cap, turns the open into a resource-exhausted denial. Without a handler, the +proxy denies the open at once. It logs the first attempt at warn and later ones +at debug, and emits no OCSF denial event: a standard OpenTelemetry SDK retries +for the life of the agent, and one event per retry would flood the OCSF log. +The supervisor installs its OTLP relay receiver as the handler while the agent +trace relay is active. diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 630601f2ac..bbe1af000f 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -92,7 +92,7 @@ Console JSON is installed independently of optional file appenders and uses an I The `otlp_relay` module forwards the workload's OTLP/HTTP protobuf trace exports to the collector named by `OPENSHELL_OTLP_AGENT_ENDPOINT`. The gateway sets that variable only when `[openshell.gateway.otlp.agent]` is enabled, so trace export alone leaves the relay inactive. Metrics and logs exports to the relay are acknowledged and discarded. The workload exports to the reserved address `192.0.0.8:4318`; the sandbox broker stages that connection to the supervisor, and the network proxy hands the stream to the relay's HTTP/1.1 receiver through the generic reserved-destination hook instead of evaluating policy. The receiver replaces the `openshell.sandbox.id` and `openshell.telemetry.source=agent` resource attributes, pushes the re-encoded batch into a bounded buffer, and a single export task forwards batches over OTLP/gRPC with a lazy connection. -Limits are fixed: 2 MiB per request body, at most 512 `ResourceSpans` entries, 256 KiB of encoded `Resource` fields, and 8192 resource attributes per request (each of these answers 413), a 10 second header-read timeout, 16 concurrent connections, 32 buffered batches, and a 5 second deadline per export attempt. Only the resources are decoded; spans are copied byte for byte, at most two requests at a time. Worst-case memory attributable to the agent is about 67 MiB of buffered batches, plus 32 MiB of bodies awaiting enrichment across the 16 connections, plus about 12 MiB for the two enrichments in progress, plus about 4 MiB for the batch being exported and its encoded copy. A full buffer answers 503 with `Retry-After`. Without a collector endpoint nothing starts and the proxy refuses the relay address. After the agent exits, `run_sandbox_with_backend` flushes buffered batches for up to 5 seconds before it reports the exit to the gateway, then logs a final summary of rejected and failed counts. Flushes skip the wait while the most recent export failed, and batches still pending when the relay is dropped without a flush count as failed. The supervisor tells the sandbox runtime through `AgentSpec::otlp_relay_active` whether the relay runs, and the runtime injects the generic `OTEL_EXPORTER_OTLP_*` defaults only then. +Limits are fixed: 2 MiB per request body, at most 512 `ResourceSpans` entries, 256 KiB of encoded `Resource` fields, and 8192 resource attributes per request (each of these answers 413), a 10 second header-read timeout, 16 concurrent connections, 32 buffered batches, and a 5 second deadline per export attempt. Only the resources are decoded; spans are copied byte for byte, at most two requests at a time. Worst-case memory attributable to the agent is about 67 MiB of buffered batches, plus 32 MiB of bodies awaiting enrichment across the 16 connections, plus about 12 MiB for the two enrichments in progress, plus about 4 MiB for the batch being exported and its encoded copy. A full buffer answers 503 with `Retry-After`. Without an agent endpoint nothing starts and the proxy refuses the relay address. After the agent exits, `run_sandbox_with_backend` flushes buffered batches for up to 5 seconds before it reports the exit to the gateway, then logs a final summary of rejected and failed counts. Flushes skip the wait while the most recent export failed, and batches still pending when the relay is dropped without a flush count as failed. The supervisor tells the sandbox runtime through `AgentSpec::otlp_relay_active` whether the relay runs, and the runtime injects the generic `OTEL_EXPORTER_OTLP_*` defaults only then. ## Safe runtime and parent liveness diff --git a/docs/about/architecture.mdx b/docs/about/architecture.mdx index 5862988396..a63e5257ed 100644 --- a/docs/about/architecture.mdx +++ b/docs/about/architecture.mdx @@ -91,6 +91,17 @@ The mediated channel to the **supervisor** is the workload's only allowed egress path. The outer fence denies everything else, so the agent can't reach a service, the gateway, DNS, or another private address directly. +### Agent traces + +When an operator enables the +[agent trace relay](/how-it-works/gateways/configuration#agent-trace-relay), +the **sandbox** points the agent's OpenTelemetry SDK at the reserved address +`192.0.0.8:4318`. Connections to it take the same mediated channel, but the +**supervisor** answers them itself instead of opening an upstream connection. +It tags each trace batch with the sandbox ID and forwards it to the operator's +collector. The agent never learns the collector's address and needs no egress +rule to reach it. + ## How Each Runtime Builds the Boundary Every runtime follows the same contract, but each one uses the tools it already