feat(observability): send configured headers to every OTLP collector

crates/openshell-core/src/jwt.rs

@@ -406,10 +406,18 @@ mod session {
         /// Never delivered to the workload. Missing only in older bundles.
         #[serde(default, skip_serializing_if = "Option::is_none")]
         pub ssh_host_private_key: Option<SecretSshHostKey>,
+        /// Headers for the supervisor's trace collector, present only when
+        /// the gateway has them. Never delivered to the workload.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        pub traces_otlp_headers: Option<SecretOtlpHeaders>,
         /// Headers for the OCSF OTLP collector, present only when the gateway
         /// has them. Never delivered to the workload.
         #[serde(default, skip_serializing_if = "Option::is_none")]
         pub ocsf_otlp_headers: Option<SecretOtlpHeaders>,
+        /// Headers for the agent trace relay's collector, present only when
+        /// the gateway configures them. Never delivered to the workload.
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        pub agent_otlp_headers: Option<SecretOtlpHeaders>,
     }
 
     /// Gateway-created authentication input trusted by a compute driver.
@@ -496,7 +504,9 @@ mod session {
                 .field("sandbox_token", &"[REDACTED]")
                 .field("sandbox_expires_at", &self.sandbox_expires_at)
                 .field("ssh_host_private_key", &self.ssh_host_private_key)
+                .field("traces_otlp_headers", &self.traces_otlp_headers)
                 .field("ocsf_otlp_headers", &self.ocsf_otlp_headers)
+                .field("agent_otlp_headers", &self.agent_otlp_headers)
                 .finish()
         }
     }
@@ -1625,7 +1635,9 @@ mod tests {
                 sandbox_token: pair.sandbox.token,
                 sandbox_expires_at: pair.sandbox.expires_at,
                 ssh_host_private_key: None,
+                traces_otlp_headers: None,
                 ocsf_otlp_headers: None,
+                agent_otlp_headers: None,
             };
             bundle.validate().expect("non-expiring auth bundle");
         }
@@ -1764,9 +1776,15 @@ mod tests {
                 ssh_host_private_key: Some(SecretSshHostKey::new(
                     "private-ssh-host-key".to_string(),
                 )),
+                traces_otlp_headers: Some(SecretOtlpHeaders::new(
+                    "authorization=traces-token".to_string(),
+                )),
                 ocsf_otlp_headers: Some(SecretOtlpHeaders::new(
                     "authorization=collector-token".to_string(),
                 )),
+                agent_otlp_headers: Some(SecretOtlpHeaders::new(
+                    "authorization=agent-token".to_string(),
+                )),
             };
 
             let encoded = serde_json::to_vec(&bundle).expect("serialize auth bundle");
@@ -1787,22 +1805,36 @@ mod tests {
             assert!(!debug.contains(bundle.sandbox_token.expose_secret()));
             assert!(!debug.contains("private-ssh-host-key"));
             assert!(!debug.contains("collector-token"));
-            assert_eq!(debug.matches("[REDACTED]").count(), 4);
+            assert!(!debug.contains("agent-token"));
+            assert!(!debug.contains("traces-token"));
+            assert_eq!(debug.matches("[REDACTED]").count(), 6);
             assert_eq!(
                 decoded.ssh_host_private_key.unwrap().expose_secret(),
                 "private-ssh-host-key"
             );
+            assert_eq!(
+                decoded.traces_otlp_headers.unwrap().expose_secret(),
+                "authorization=traces-token"
+            );
             assert_eq!(
                 decoded.ocsf_otlp_headers.unwrap().expose_secret(),
                 "authorization=collector-token"
             );
+            assert_eq!(
+                decoded.agent_otlp_headers.unwrap().expose_secret(),
+                "authorization=agent-token"
+            );
             let mut legacy = serde_json::to_value(&bundle).unwrap();
             let fields = legacy.as_object_mut().unwrap();
             fields.remove("ssh_host_private_key");
+            fields.remove("traces_otlp_headers");
             fields.remove("ocsf_otlp_headers");
+            fields.remove("agent_otlp_headers");
             let legacy: SupervisorAuthBundle = serde_json::from_value(legacy).unwrap();
             assert!(legacy.ssh_host_private_key.is_none());
+            assert!(legacy.traces_otlp_headers.is_none());
             assert!(legacy.ocsf_otlp_headers.is_none());
+            assert!(legacy.agent_otlp_headers.is_none());
         }
 
         #[derive(Serialize)]

crates/openshell-driver-docker/src/tests.rs

@@ -64,7 +64,9 @@ fn test_launch_authentication() -> Vec<u8> {
             sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),
             sandbox_expires_at: i64::MAX,
             ssh_host_private_key: None,
+            traces_otlp_headers: None,
             ocsf_otlp_headers: None,
+            agent_otlp_headers: None,
         },
         gateway_id: "gateway-test".to_string(),
         verification_keys: vec![SessionVerificationKey {

crates/openshell-driver-podman/src/driver.rs

@@ -2663,7 +2663,9 @@ mod tests {
                 sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),
                 sandbox_expires_at: i64::MAX,
                 ssh_host_private_key: None,
+                traces_otlp_headers: None,
                 ocsf_otlp_headers: None,
+                agent_otlp_headers: None,
             },
             gateway_id: "gateway-test".to_string(),
             verification_keys: vec![SessionVerificationKey {

crates/openshell-driver-podman/src/isolation.rs

@@ -471,7 +471,9 @@ mod tests {
                 sandbox_token: SecretJwt::parse("sandbox.token.value").unwrap(),
                 sandbox_expires_at: i64::MAX,
                 ssh_host_private_key: None,
+                traces_otlp_headers: None,
                 ocsf_otlp_headers: None,
+                agent_otlp_headers: None,
             },
             gateway_id: "gateway-test".to_string(),
             verification_keys: vec![SessionVerificationKey {

crates/openshell-driver-vm/src/driver.rs

@@ -11242,7 +11242,9 @@ mod tests {
                 sandbox_token: SecretJwt::parse(format!("sandbox-{label}")).expect("sandbox token"),
                 sandbox_expires_at: 1,
                 ssh_host_private_key: None,
+                traces_otlp_headers: None,
                 ocsf_otlp_headers: None,
+                agent_otlp_headers: None,
             },
             gateway_id: "gateway-a".to_string(),
             verification_keys: vec![SessionVerificationKey {

crates/openshell-gateway/src/lib.rs

@@ -749,3 +749,7 @@ mod tests {
         assert!(<VmFactory as openshell_server::ComputeDriverFactory>::supports_network_supervisor_trust(&VmFactory));
     }
 }
+
+#[cfg(test)]
+#[path = "../../../tests/support/environment.rs"]
+mod test_environment;

crates/openshell-gateway/src/vm.rs

@@ -729,6 +729,11 @@ fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gat
         sandbox_env::OTLP_AGENT_ENDPOINT_FLAG,
         sandbox_env::OTLP_AGENT_PROTOCOL_FLAG,
     );
+    // The driver's own trace exporter reads this like any OTLP SDK; keeping
+    // it out of the arguments keeps the headers out of process listings.
+    if let Some(headers) = openshell_server::gateway_trace_headers(otlp_config) {
+        command.env("OTEL_EXPORTER_OTLP_TRACES_HEADERS", headers.expose_secret());
+    }
     // The driver names its own spans with the gateway name, and supervisors
     // name their spans, OCSF records, and relayed traces with it.
     if !otlp.is_empty() {
@@ -833,7 +838,8 @@ mod tests {
     use openshell_core::sandbox_env::OtlpProtocol;
     use openshell_core::{NetworkSupervisorTrustBundle, UpstreamProxyConfig};
     use openshell_server::config_file::{
-        AgentOtlpConfig, OcsfOtlpConfig, OtlpCollectorConfig, OtlpConfig, OtlpTracesConfig,
+        AgentOtlpConfig, OcsfOtlpConfig, OtlpCollectorConfig, OtlpConfig, OtlpHeaders,
+        OtlpTracesConfig,
     };
     use std::os::unix::fs::PermissionsExt;
     use std::os::unix::net::UnixListener as StdUnixListener;
@@ -876,6 +882,36 @@ mod tests {
         .expect("an explicit bootstrap image is valid");
     }
 
+    #[test]
+    fn vm_driver_receives_trace_headers_in_its_environment_only() {
+        crate::test_environment::Environment::new()
+            .remove("OTEL_EXPORTER_OTLP_HEADERS")
+            .remove("OTEL_EXPORTER_OTLP_TRACES_HEADERS")
+            .run(|| {
+                let otlp: OtlpConfig = toml::from_str(
+                    "endpoint = 'http://collector.internal:4318'\n\
+                 headers = { authorization = 'Bearer secret-token' }\n",
+                )
+                .unwrap();
+                let mut command = tokio::process::Command::new("openshell-driver-vm");
+                append_otlp_args(&mut command, Some(&otlp), "production-us-west");
+
+                let env = command
+                    .as_std()
+                    .get_envs()
+                    .find(|(name, _)| *name == "OTEL_EXPORTER_OTLP_TRACES_HEADERS")
+                    .and_then(|(_, value)| value)
+                    .map(|value| value.to_string_lossy().into_owned());
+                assert_eq!(env.as_deref(), Some("authorization=Bearer%20secret-token"));
+                assert!(
+                    command
+                        .as_std()
+                        .get_args()
+                        .all(|arg| !arg.to_string_lossy().contains("secret-token"))
+                );
+            });
+    }
+
     #[test]
     fn vm_driver_command_includes_gateway_otlp_configuration() {
         let mut command = tokio::process::Command::new("openshell-driver-vm");
@@ -887,6 +923,8 @@ mod tests {
                     collector: OtlpCollectorConfig {
                         endpoint: "http://collector.internal:4317".to_string(),
                         protocol: OtlpProtocol::Grpc,
+                        headers: OtlpHeaders::default(),
+                        headers_file: None,
                     },
                 }),
                 ocsf: None,
@@ -919,6 +957,8 @@ mod tests {
                     collector: OtlpCollectorConfig {
                         endpoint: "http://collector.internal:4318".to_string(),
                         protocol: OtlpProtocol::HttpProtobuf,
+                        headers: OtlpHeaders::default(),
+                        headers_file: None,
                     },
                 }),
                 ocsf: None,
@@ -954,6 +994,8 @@ mod tests {
                     collector: OtlpCollectorConfig {
                         endpoint: "http://siem.internal:4318".to_string(),
                         protocol: OtlpProtocol::HttpProtobuf,
+                        headers: OtlpHeaders::default(),
+                        headers_file: None,
                     },
                     schema_version: Some("2.0".into()),
                 }),
@@ -993,6 +1035,8 @@ mod tests {
                     collector: OtlpCollectorConfig {
                         endpoint: "http://agents.internal:4318".to_string(),
                         protocol: OtlpProtocol::HttpProtobuf,
+                        headers: OtlpHeaders::default(),
+                        headers_file: None,
                     },
                 }),
             }),

crates/openshell-otel-test-support/src/lib.rs

@@ -109,6 +109,7 @@ pub async fn assert_compute_driver_tracing(
     let (provider, error) = descriptor.provider_for(
         Some(collector.endpoint()),
         openshell_otel::Protocol::Grpc,
+        &::http::HeaderMap::new(),
         service_version,
         Some("test-gateway"),
         Some(descriptor.compute_driver()),
@@ -142,6 +143,8 @@ pub struct ReceivedTraces {
     pub spans: Vec<Span>,
     /// String bodies of exported log records.
     pub log_bodies: Vec<String>,
+    /// `authorization` metadata of each trace export request.
+    pub trace_authorizations: Vec<String>,
     /// `authorization` metadata of each log export request.
     pub log_authorizations: Vec<String>,
     pub service_names: Vec<String>,
@@ -216,6 +219,15 @@ impl TraceService for Collector {
             {
                 received.encodings.push(encoding.to_string());
             }
+            if let Some(authorization) = request
+                .metadata()
+                .get("authorization")
+                .and_then(|value| value.to_str().ok())
+            {
+                received
+                    .trace_authorizations
+                    .push(authorization.to_string());
+            }
             for resource_span in request.into_inner().resource_spans {
                 let attributes = received.record_resource(resource_span.resource);
                 for scope_span in resource_span.scope_spans {

crates/openshell-otel/Cargo.toml

@@ -37,6 +37,7 @@ openshell-otel-test-support = { path = "../openshell-otel-test-support" }
 opentelemetry-proto = { workspace = true, features = ["logs"] }
 opentelemetry_sdk = { workspace = true, features = ["testing"] }
 prost = { workspace = true }
+tempfile = "3"
 tokio = { workspace = true, features = ["test-util"] }
 
 [lints]

crates/openshell-otel/src/driver.rs

@@ -74,6 +74,7 @@ impl ComputeDriverTracing {
         self,
         endpoint: Option<&str>,
         protocol: crate::Protocol,
+        headers: &http::HeaderMap,
         service_version: &'static str,
         gateway_name: Option<&str>,
         compute_driver: Option<&str>,
@@ -89,6 +90,7 @@ impl ComputeDriverTracing {
                 ),
             })?
             .with_protocol(protocol)
+            .with_headers(headers.clone())
             .tracer_provider()
         });
         match provider {
@@ -225,6 +227,7 @@ pub fn install_driver_tracing(
     let (provider, setup_error) = descriptor.provider_for(
         config.endpoint,
         config.protocol,
+        &http::HeaderMap::new(),
         config.service_version,
         config.gateway_name,
         Some(descriptor.compute_driver()),
@@ -259,25 +262,35 @@ mod tests {
     }
 
     #[tokio::test(flavor = "multi_thread")]
-    async fn driver_provider_exports_over_the_configured_protocol() {
-        use opentelemetry::trace::{Tracer as _, TracerProvider as _};
-
-        const TRACING: ComputeDriverTracing = crate::compute_driver_tracing!();
-        let mut collector = openshell_otel_test_support::OtlpHttpTestServer::start().await;
-        let (provider, error) = TRACING.provider_for(
-            Some(collector.endpoint()),
-            crate::Protocol::HttpProtobuf,
-            "1.2.3",
-            Some("test-gateway"),
-            Some("docker"),
-        );
-        assert!(error.is_none(), "unexpected setup error: {error:?}");
-        let provider = provider.expect("provider");
-        provider.tracer("test").in_span("driver-span", |_| {});
-        let flush = tokio::task::spawn_blocking(move || provider.force_flush());
-
-        assert_eq!(collector.next_export().await.path, "/v1/traces");
-        flush.await.unwrap().expect("flush succeeds");
+    async fn driver_provider_exports_with_the_configured_protocol_and_headers() {
+        crate::test_environment::Environment::new()
+            .remove("OTEL_EXPORTER_OTLP_HEADERS")
+            .remove("OTEL_EXPORTER_OTLP_TRACES_HEADERS")
+            .remove("OTEL_EXPORTER_OTLP_LOGS_HEADERS")
+            .run_async(async {
+                use opentelemetry::trace::{Tracer as _, TracerProvider as _};
+
+                const TRACING: ComputeDriverTracing = crate::compute_driver_tracing!();
+                let mut collector = openshell_otel_test_support::OtlpHttpTestServer::start().await;
+                let (provider, error) = TRACING.provider_for(
+                    Some(collector.endpoint()),
+                    crate::Protocol::HttpProtobuf,
+                    &crate::parse_otlp_headers("authorization=driver"),
+                    "1.2.3",
+                    Some("test-gateway"),
+                    Some("docker"),
+                );
+                assert!(error.is_none(), "unexpected setup error: {error:?}");
+                let provider = provider.expect("provider");
+                provider.tracer("test").in_span("driver-span", |_| {});
+                let flush = tokio::task::spawn_blocking(move || provider.force_flush());
+
+                let export = collector.next_export().await;
+                assert_eq!(export.path, "/v1/traces");
+                assert_eq!(export.headers["authorization"], "driver");
+                flush.await.unwrap().expect("flush succeeds");
+            })
+            .await;
     }
 
     #[tokio::test]