feat(supervisor): relay agent OTLP traces to the collector

.agents/skills/helm-dev-environment/SKILL.md

@@ -170,7 +170,10 @@ trace-context annotation. The same command exposes OTLP/gRPC on
 `http://127.0.0.1:4317` and, when deployed, the Kubernetes gateway on
 `http://127.0.0.1:8090`. The local `gateway:docker`, `gateway:podman`, and
 `gateway:vm` tasks detect the collector listener at startup and enable trace
-export only while it is reachable.
+export, OCSF log export, and the agent trace relay only while it is reachable.
+Spans an agent exports from inside a sandbox arrive through the supervisor's
+relay with `openshell.telemetry.source=agent` and share `openshell.sandbox.id`
+with that sandbox's supervisor spans.
 
 The Skaffold profile for HA reverse-proxy development is available from
 `deploy/helm/openshell/`:

Cargo.toml

@@ -71,6 +71,7 @@ tracing-appender = "0.2"
 
 # OpenTelemetry — OTLP/gRPC export. Kept in lockstep with the workspace's
 # tonic 0.14 / prost 0.14 via opentelemetry-proto's `grpc-tonic` feature.
+opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] }
 opentelemetry = "0.32"
 opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "gzip-tonic", "tls-aws-lc", "tls-roots", "trace"] }

crates/openshell-core/src/sandbox_env.rs

@@ -32,6 +32,12 @@ pub const OTLP_ENDPOINT: &str = "OPENSHELL_OTLP_ENDPOINT";
 /// OTLP/gRPC collector endpoint for supervisor OCSF event export.
 pub const OTLP_OCSF_ENDPOINT: &str = "OPENSHELL_OTLP_OCSF_ENDPOINT";
 
+/// OTLP/gRPC collector endpoint that supervisors relay workload traces to.
+///
+/// Set only when the gateway enables `[openshell.gateway.otlp.agent]`; absent
+/// keeps the relay inactive and leaves the workload environment unchanged.
+pub const OTLP_AGENT_ENDPOINT: &str = "OPENSHELL_OTLP_AGENT_ENDPOINT";
+
 /// OCSF schema version that supervisors downgrade exported events to.
 pub const OTLP_OCSF_SCHEMA_VERSION: &str = "OPENSHELL_OTLP_OCSF_SCHEMA_VERSION";
 
@@ -299,11 +305,74 @@ pub const DEFAULT_SANDBOX_GID: u32 = 1000;
 /// OCI only for the former contract.
 pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER";
 
+/// Standard OpenTelemetry environment variable for the OTLP exporter endpoint.
+///
+/// While the supervisor serves the OTLP relay, the sandbox runtime sets it in
+/// workload processes to [`OTLP_RELAY_ENDPOINT`] as a lowest-precedence
+/// default, so agent SDKs export to the relay without configuration. It is
+/// not set when the creation request, the image, or a per-exec environment
+/// names an endpoint, generic or traces-specific, and any signal-specific
+/// endpoint the workload sets later takes precedence over it in the SDK.
+pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
+
+/// Standard OpenTelemetry environment variable for the OTLP exporter protocol.
+///
+/// Set to `http/protobuf` alongside [`OTEL_EXPORTER_OTLP_ENDPOINT`].
+pub const OTEL_EXPORTER_OTLP_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL";
+
+/// Signal-specific endpoint variable. Never injected; a layer that sets it
+/// has chosen where traces go, so the relay default is left out.
+pub const OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT";
+
+/// Reserved destination agent processes export OTLP to.
+///
+/// This address is never routed. A workload `connect()` to any non-loopback
+/// address is intercepted by the sandbox seccomp broker and staged for the
+/// supervisor, which recognises this destination and serves the OTLP
+/// receiver on the staged stream itself instead of dialing upstream. So the
+/// value is a label the supervisor switches on, and it must stay outside
+/// loopback (loopback connects complete locally without mediation) and
+/// outside `198.18.0.0/15`, which the policy DNS runtime uses for synthetic
+/// answers. `192.0.0.8` is the IPv4 dummy address (RFC 7600): reserved,
+/// unroutable, and never assigned to a real service.
+pub const OTLP_RELAY_ADDR: &str = "192.0.0.8:4318";
+
+/// [`OTLP_RELAY_ADDR`] as the URL injected through
+/// [`OTEL_EXPORTER_OTLP_ENDPOINT`].
+pub const OTLP_RELAY_ENDPOINT: &str = "http://192.0.0.8:4318";
+
 // The corporate upstream-proxy configuration deliberately has no reserved
 // environment variables: it travels on the supervisor's argv
 // (`--upstream-proxy` and friends), which a sandbox image cannot forge the
 // way it could bake `ENV` values.
 
+#[cfg(test)]
+mod otlp_relay_address_tests {
+    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+    use super::{OTLP_RELAY_ADDR, OTLP_RELAY_ENDPOINT};
+
+    #[test]
+    fn relay_address_is_an_unroutable_non_loopback_label() {
+        let addr: SocketAddr = OTLP_RELAY_ADDR.parse().expect("relay address parses");
+        assert_eq!(addr.port(), 4318, "OTLP HTTP default port");
+        let IpAddr::V4(ip) = addr.ip() else {
+            panic!("relay address must be IPv4");
+        };
+        assert!(
+            !ip.is_loopback(),
+            "loopback would bypass seccomp mediation and never reach the supervisor"
+        );
+        assert_eq!(ip, Ipv4Addr::new(192, 0, 0, 8), "RFC 7600 dummy address");
+        let [a, b, ..] = ip.octets();
+        assert!(
+            !(a == 198 && (b == 18 || b == 19)),
+            "198.18.0.0/15 is the policy DNS synthetic answer pool"
+        );
+        assert_eq!(OTLP_RELAY_ENDPOINT, format!("http://{OTLP_RELAY_ADDR}"));
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/openshell-driver-docker/README.md

@@ -11,7 +11,11 @@ When the gateway configures `[openshell.gateway.otlp]`, the in-process driver
 exports spans to the same OTLP/gRPC collector as
 `openshell-driver-docker`. The standalone driver accepts
 `OPENSHELL_OTLP_ENDPOINT`, continues W3C trace context from gateway RPC
-metadata, and flushes spans during graceful shutdown.
+metadata, and flushes spans during graceful shutdown. When
+`[openshell.gateway.otlp.agent]` is enabled, the driver also sets
+`OPENSHELL_OTLP_AGENT_ENDPOINT` on the supervisor container, which starts the
+agent trace relay. The standalone driver takes that endpoint as
+`--otlp-agent-endpoint`.
 
 ## Runtime Model
 

crates/openshell-driver-docker/src/lib.rs

@@ -212,6 +212,11 @@ pub struct DockerComputeConfig {
     #[serde(skip)]
     pub supervisor_ocsf_export: Option<openshell_core::sandbox_env::OcsfOtlpExport>,
 
+    /// OTLP/gRPC collector that supervisors relay workload traces to. The
+    /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it.
+    #[serde(skip)]
+    pub supervisor_otlp_agent_endpoint: Option<String>,
+
     /// Gateway name passed to supervisors for their telemetry. The gateway
     /// supplies its own name; driver TOML cannot set it.
     #[serde(skip)]
@@ -319,6 +324,7 @@ impl Default for DockerComputeConfig {
             supervisor_network_mode: DockerSupervisorNetworkMode::Host,
             host_gateway_ip: None,
             supervisor_ocsf_export: None,
+            supervisor_otlp_agent_endpoint: None,
             supervisor_gateway_name: None,
             sandbox_runtime_image: None,
             supervisor_bin: None,
@@ -360,6 +366,7 @@ struct DockerDriverRuntimeConfig {
     bridge_default_endpoint: Option<String>,
     host_gateway_ip: Option<IpAddr>,
     supervisor_ocsf_export: Option<openshell_core::sandbox_env::OcsfOtlpExport>,
+    supervisor_otlp_agent_endpoint: Option<String>,
     supervisor_gateway_name: Option<String>,
     ssh_socket_path: String,
     guest_tls: Option<DockerGuestTlsPaths>,
@@ -1018,6 +1025,9 @@ impl DockerComputeDriver {
                 bridge_default_endpoint,
                 host_gateway_ip,
                 supervisor_ocsf_export: docker_config.supervisor_ocsf_export.clone(),
+                supervisor_otlp_agent_endpoint: docker_config
+                    .supervisor_otlp_agent_endpoint
+                    .clone(),
                 supervisor_gateway_name: docker_config.supervisor_gateway_name.clone(),
                 ssh_socket_path: docker_config.ssh_socket_path.clone(),
                 guest_tls,
@@ -5830,7 +5840,8 @@ fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap<String, String>
 }
 
 /// Environment that lets the supervisor export spans, joining the current
-/// trace, and OCSF events, identifying the gateway that launched it.
+/// trace, OCSF events, and relayed workload traces, identifying the gateway
+/// that launched it.
 fn supervisor_otlp_environment(config: &DockerDriverRuntimeConfig) -> Vec<String> {
     let traces = config.supervisor_otlp_endpoint.iter().flat_map(|endpoint| {
         std::iter::once((openshell_core::sandbox_env::OTLP_ENDPOINT, endpoint.clone()))
@@ -5840,12 +5851,22 @@ fn supervisor_otlp_environment(config: &DockerDriverRuntimeConfig) -> Vec<String
         .supervisor_ocsf_export
         .iter()
         .flat_map(openshell_core::sandbox_env::OcsfOtlpExport::environment);
+    let agent = config
+        .supervisor_otlp_agent_endpoint
+        .iter()
+        .map(|endpoint| {
+            (
+                openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT,
+                endpoint.clone(),
+            )
+        });
     let gateway_name = config
         .supervisor_gateway_name
         .iter()
         .map(|name| (openshell_core::sandbox_env::GATEWAY_NAME, name.clone()));
     traces
         .chain(ocsf)
+        .chain(agent)
         .chain(gateway_name)
         .map(|(name, value)| format!("{name}={value}"))
         .collect()

crates/openshell-driver-docker/src/main.rs

@@ -59,6 +59,11 @@ struct Args {
     )]
     otlp_ocsf_schema_version: Option<String>,
 
+    /// OTLP/gRPC collector endpoint that supervisors relay workload traces
+    /// to. Absent keeps the agent trace relay off.
+    #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")]
+    otlp_agent_endpoint: Option<String>,
+
     #[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
     gateway_name: Option<String>,
 }
@@ -90,6 +95,7 @@ async fn main() -> Result<()> {
         docker_config.supervisor_image = Some(image);
     }
     docker_config.supervisor_otlp_endpoint = args.otlp_endpoint.clone();
+    docker_config.supervisor_otlp_agent_endpoint = args.otlp_agent_endpoint.clone();
     docker_config.supervisor_gateway_name = args.gateway_name.clone();
     docker_config.supervisor_ocsf_export = args.otlp_ocsf_endpoint.clone().map(|endpoint| {
         openshell_core::sandbox_env::OcsfOtlpExport {

crates/openshell-driver-docker/src/tests.rs

@@ -330,6 +330,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig {
         bridge_default_endpoint: Some("https://host.docker.internal:8443".to_string()),
         host_gateway_ip: None,
         supervisor_ocsf_export: None,
+        supervisor_otlp_agent_endpoint: None,
         supervisor_gateway_name: None,
         ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(),
         guest_tls: Some(DockerGuestTlsPaths {
@@ -1513,7 +1514,8 @@ fn docker_child_environment_strips_supervisor_control_keys() {
 #[test]
 fn supervisor_otlp_environment_carries_only_configured_signals() {
     use openshell_core::sandbox_env::{
-        GATEWAY_NAME, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT, OTLP_OCSF_SCHEMA_VERSION, OcsfOtlpExport,
+        GATEWAY_NAME, OTLP_AGENT_ENDPOINT, OTLP_ENDPOINT, OTLP_OCSF_ENDPOINT,
+        OTLP_OCSF_SCHEMA_VERSION, OcsfOtlpExport,
     };
 
     let mut config = runtime_config();
@@ -1538,6 +1540,13 @@ fn supervisor_otlp_environment_carries_only_configured_signals() {
         ]
     );
 
+    config.supervisor_ocsf_export = None;
+    config.supervisor_otlp_agent_endpoint = Some("http://agents:4317".to_string());
+    assert_eq!(
+        supervisor_otlp_environment(&config),
+        [format!("{OTLP_AGENT_ENDPOINT}=http://agents:4317")]
+    );
+
     config.supervisor_gateway_name = Some("prod-west".to_string());
     assert_eq!(
         supervisor_otlp_environment(&config).last(),

crates/openshell-driver-kubernetes/README.md

@@ -91,7 +91,9 @@ When the gateway exports OTLP traces, the driver sets
 `OPENSHELL_OTLP_ENDPOINT` on the supervisor Pod to the gateway's endpoint and
 `TRACEPARENT` to the trace context of the operation that created the Pod. The
 supervisor exports its spans there and parents its startup span on that
-context. The endpoint is not configurable in driver TOML.
+context. When `[openshell.gateway.otlp.agent]` is enabled, the driver also
+sets `OPENSHELL_OTLP_AGENT_ENDPOINT`, which starts the supervisor's agent trace
+relay. Neither endpoint is configurable in driver TOML.
 
 The driver creates one namespace-wide `NetworkPolicy` before it releases any
 workload Pod. It selects every OpenShell workload, denies all workload egress,
@@ -257,8 +259,9 @@ fails closed.
 
 The topology does not support upstream proxies or provider SPIFFE identity, and
 the shared supervisor is a single point of failure for the sandboxes it hosts.
-Shared sandbox instances export no supervisor OTLP traces or OCSF events; the driver
-warns at startup when the gateway configures that export.
+Shared sandbox instances export no supervisor OTLP traces or OCSF events and do
+not relay agent traces; the driver warns at startup when the gateway configures
+any of them.
 When the setting is removed, reconciliation suspends shared generations, and a
 restart creates the dedicated generation's boundary Service the shared one never
 had.

crates/openshell-driver-kubernetes/src/config.rs

@@ -372,6 +372,10 @@ pub struct KubernetesComputeConfig {
     /// export; driver TOML cannot set it.
     #[serde(skip)]
     pub supervisor_ocsf_export: Option<openshell_core::sandbox_env::OcsfOtlpExport>,
+    /// OTLP/gRPC collector that supervisors relay workload traces to. The
+    /// gateway supplies it from `[otlp.agent]`; driver TOML cannot set it.
+    #[serde(skip)]
+    pub supervisor_otlp_agent_endpoint: Option<String>,
 
     /// Gateway name passed to supervisors for their telemetry. The gateway
     /// supplies its own name; driver TOML cannot set it.
@@ -502,6 +506,7 @@ impl Default for KubernetesComputeConfig {
             grpc_endpoint: String::new(),
             supervisor_otlp_endpoint: None,
             supervisor_ocsf_export: None,
+            supervisor_otlp_agent_endpoint: None,
             supervisor_gateway_name: None,
             ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(),
             client_tls_secret_name: String::new(),

crates/openshell-driver-kubernetes/src/driver.rs

@@ -912,7 +912,8 @@ impl std::fmt::Debug for KubernetesComputeDriver {
 
 impl KubernetesComputeDriver {
     /// Environment that lets a supervisor export spans, joining the current
-    /// trace, and OCSF events, identifying the gateway that launched it.
+    /// trace, OCSF events, and relayed workload traces, identifying the
+    /// gateway that launched it.
     fn supervisor_otlp_environment(&self) -> Vec<(&'static str, String)> {
         let mut environment = Vec::new();
         if let Some(endpoint) = self.config.supervisor_otlp_endpoint.as_deref() {
@@ -925,6 +926,12 @@ impl KubernetesComputeDriver {
         if let Some(export) = &self.config.supervisor_ocsf_export {
             environment.extend(export.environment());
         }
+        if let Some(endpoint) = &self.config.supervisor_otlp_agent_endpoint {
+            environment.push((
+                openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT,
+                endpoint.clone(),
+            ));
+        }
         if let Some(name) = &self.config.supervisor_gateway_name {
             environment.push((openshell_core::sandbox_env::GATEWAY_NAME, name.clone()));
         }
@@ -1078,10 +1085,11 @@ impl KubernetesComputeDriver {
         }
         if shared_supervisor.is_some()
             && (config.supervisor_otlp_endpoint.is_some()
-                || config.supervisor_ocsf_export.is_some())
+                || config.supervisor_ocsf_export.is_some()
+                || config.supervisor_otlp_agent_endpoint.is_some())
         {
             warn!(
-                "the gateway supervisor topology does not export supervisor OTLP traces or OCSF events; shared sandbox instances keep console logs and gateway log push"
+                "the gateway supervisor topology does not export supervisor OTLP traces or OCSF events or relay workload traces; shared sandbox instances keep console logs and gateway log push"
             );
         }
         let driver = Self {
@@ -8757,6 +8765,18 @@ mod tests {
             ]
         );
 
+        let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig {
+            supervisor_otlp_agent_endpoint: Some("http://agents:4317".into()),
+            ..Default::default()
+        });
+        assert_eq!(
+            driver.supervisor_otlp_environment(),
+            [(
+                openshell_core::sandbox_env::OTLP_AGENT_ENDPOINT,
+                "http://agents:4317".to_string()
+            )]
+        );
+
         let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig {
             supervisor_gateway_name: Some("prod-west".into()),
             ..Default::default()

crates/openshell-driver-kubernetes/src/main.rs

@@ -57,6 +57,11 @@ struct Args {
     )]
     otlp_ocsf_schema_version: Option<String>,
 
+    /// OTLP/gRPC collector endpoint that supervisors relay workload traces
+    /// to. Absent keeps the agent trace relay off.
+    #[arg(long, env = "OPENSHELL_OTLP_AGENT_ENDPOINT")]
+    otlp_agent_endpoint: Option<String>,
+
     #[arg(long, env = "OPENSHELL_GATEWAY_NAME")]
     gateway_name: Option<String>,
 
@@ -298,6 +303,7 @@ async fn main() -> Result<()> {
             proxy_ca_bundle: args.proxy_ca_bundle,
             grpc_endpoint: args.grpc_endpoint.unwrap_or_default(),
             supervisor_otlp_endpoint: args.otlp_endpoint.clone(),
+            supervisor_otlp_agent_endpoint: args.otlp_agent_endpoint.clone(),
             supervisor_gateway_name: args.gateway_name.clone(),
             supervisor_ocsf_export: args.otlp_ocsf_endpoint.clone().map(|endpoint| {
                 openshell_core::sandbox_env::OcsfOtlpExport {