feat(java): add setAttributes and custom span/log processors

e2e/java-plugin-spring-boot/README.md

@@ -45,6 +45,26 @@ Environment the app honours:
 | `SERVICE_VERSION` | Reported as `service.version`, default `dev`. |
 | `LD_BASE_URI`, `LD_STREAM_URI`, `LD_EVENTS_URI` | Flag SDK polling, streaming and events URLs for a non-default LaunchDarkly instance. Used only when all three are set. |
 
+### Global attributes repro
+
+`Application.main` calls `GlobalAttributesRepro.applyGlobalAttributes()`
+before Spring starts, so `LDObserve.setAttributes` runs before the
+`LDClient` and the plugin exist. It sets `utm_campaign=spring_promo` and
+`utm_source=email` on all telemetry. For every request, `RequestTelemetryFilter`
+prints what to look for in LaunchDarkly:
+
+```
+[#1] expect utm_campaign=spring_promo on this row (span GET /rolldice)
+[#1] expect utm_campaign=spring_promo on this row (metric http.server.requests, route /rolldice)
+[#1] expect utm_campaign=spring_promo on this row (log "GET /rolldice -> 200")
+[#1] expect utm_source=per_call (per-call wins over global) (log "GET /rolldice -> 200")
+```
+
+The request log line passes `utm_source=per_call` itself, so it must show
+`per_call` rather than the global `email`. Flag-evaluation spans and errors
+from `/error` carry the globals too. The tests boot the context without
+`main`, so they run without the globals.
+
 `make build` produces the executable jar in `build/libs/`. `make test` boots
 the application against a local OTLP collector and sampling endpoint
 (`ApplicationTelemetryTest`), drives every route, and checks the exported

e2e/java-plugin-spring-boot/src/main/java/com/launchdarkly/observability/example/springboot/Application.java

@@ -24,6 +24,7 @@
 @SpringBootApplication
 public class Application {
     public static void main(String[] args) {
+        GlobalAttributesRepro.applyGlobalAttributes();
         SpringApplication.run(Application.class, args);
     }
 }

e2e/java-plugin-spring-boot/src/main/java/com/launchdarkly/observability/example/springboot/GlobalAttributesRepro.java

@@ -0,0 +1,39 @@
+package com.launchdarkly.observability.example.springboot;
+
+import com.launchdarkly.observability.LDObserve;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+
+/**
+ * Reproduces "global attributes (e.g. UTM params) on all telemetry": the UTM
+ * attributes are set once at startup, before the plugin exists, and every
+ * request's span, log and metrics must carry them. The {@code [#1]} lines on
+ * stdout say what to look for on the matching row in LaunchDarkly.
+ */
+final class GlobalAttributesRepro {
+    static final AttributeKey<String> UTM_CAMPAIGN = AttributeKey.stringKey("utm_campaign");
+    static final AttributeKey<String> UTM_SOURCE = AttributeKey.stringKey("utm_source");
+
+    static final Attributes GLOBAL_ATTRIBUTES = Attributes.of(
+            UTM_CAMPAIGN, "spring_promo",
+            UTM_SOURCE, "email");
+
+    /** Passed on the request log line only; it must win over the global {@code utm_source}. */
+    static final Attributes PER_CALL_ATTRIBUTES = Attributes.of(UTM_SOURCE, "per_call");
+
+    private GlobalAttributesRepro() {}
+
+    /** Call before the Spring context (and so the LDClient and plugin) starts. */
+    static void applyGlobalAttributes() {
+        LDObserve.setAttributes(GLOBAL_ATTRIBUTES);
+        System.out.println("[#1] global attributes set before the LaunchDarkly client starts: " + GLOBAL_ATTRIBUTES);
+    }
+
+    static void expectGlobal(String row) {
+        System.out.println("[#1] expect utm_campaign=spring_promo on this row (" + row + ")");
+    }
+
+    static void expectPerCallWins(String row) {
+        System.out.println("[#1] expect utm_source=per_call (per-call wins over global) (" + row + ")");
+    }
+}

e2e/java-plugin-spring-boot/src/main/java/com/launchdarkly/observability/example/springboot/RequestTelemetryFilter.java

@@ -47,13 +47,18 @@ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse
             }
             span.setAttribute(STATUS, (long) status);
             span.end();
+            GlobalAttributesRepro.expectGlobal("span " + request.getMethod() + " " + route);
 
             double millis = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - started) / 1000.0;
             Attributes attrs = Attributes.of(ROUTE, route, STATUS, (long) status);
             LDObserve.recordHistogram("http.server.request.duration_ms", millis, attrs);
             LDObserve.recordIncr("http.server.requests", attrs);
-            LDObserve.recordLog(request.getMethod() + " " + route + " -> " + status,
-                    status >= 500 ? Severity.ERROR : Severity.INFO, attrs);
+            GlobalAttributesRepro.expectGlobal("metric http.server.requests, route " + route);
+            String message = request.getMethod() + " " + route + " -> " + status;
+            LDObserve.recordLog(message, status >= 500 ? Severity.ERROR : Severity.INFO,
+                    attrs.toBuilder().putAll(GlobalAttributesRepro.PER_CALL_ATTRIBUTES).build());
+            GlobalAttributesRepro.expectGlobal("log \"" + message + "\"");
+            GlobalAttributesRepro.expectPerCallWins("log \"" + message + "\"");
         }
     }
 }

sdk/@launchdarkly/observability-java/README.md

@@ -42,9 +42,9 @@ The plugin does not bring the LaunchDarkly SDK with it, so you choose its versio
 
 ### Dependencies you get
 
-- `io.opentelemetry:opentelemetry-api` is a compile-scoped (`api`) dependency, because `LDObserve` and `ObservabilityOptions` take OpenTelemetry API types such as `Attributes` and `Severity`. You do not need to declare it yourself.
+- `io.opentelemetry:opentelemetry-api` is a compile-scoped (`api`) dependency, because `LDObserve` and `ObservabilityOptions` take OpenTelemetry API types such as `Attributes` and `Severity`. You do not need to declare it yourself. `io.opentelemetry:opentelemetry-sdk` is compile-scoped too, because `ObservabilityOptions` accepts SDK `SpanProcessor` and `LogRecordProcessor` instances.
 - OTLP is sent over HTTP with OpenTelemetry's JDK sender, which uses `java.net.http.HttpClient`. The plugin does **not** pull OkHttp, Okio or the Kotlin standard library onto your classpath. The build fails if any of those leak in.
-- The OpenTelemetry SDK and exporters are runtime dependencies, and they are the only ones: JSON is handled with the LaunchDarkly SDK's own `LDValue`, so no Gson or other JSON library is added. The plugin records flag evaluations with its own hook and does not depend on `launchdarkly-java-server-sdk-otel` or on the LaunchDarkly SDK itself. The build fails if the POM names anything outside `io.opentelemetry`.
+- The OpenTelemetry exporters are runtime dependencies, and together with the OpenTelemetry API and SDK they are the only ones: JSON is handled with the LaunchDarkly SDK's own `LDValue`, so no Gson or other JSON library is added. The plugin records flag evaluations with its own hook and does not depend on `launchdarkly-java-server-sdk-otel` or on the LaunchDarkly SDK itself. The build fails if the POM names anything outside `io.opentelemetry`.
 
 ### OpenTelemetry versions
 
@@ -259,6 +259,73 @@ LDObserve.shutdown();
 
 Every `LDObserve` method is safe to call before the plugin is initialized; such calls are local no-ops and do not install the OpenTelemetry no-op global.
 
+### Global attributes
+
+`LDObserve.setAttributes` adds attributes to everything recorded after the call: every span and log record the plugin exports (flag-evaluation spans, error spans, and spans or records from instrumentation wired to `LDObserve.getOpenTelemetry()` included) and every metric recorded through `LDObserve`. Calling it again merges by key, and `LDObserve.removeAttributes` drops keys.
+
+```java
+import com.launchdarkly.observability.LDObserve;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+
+import java.util.List;
+
+LDObserve.setAttributes(Attributes.of(
+    AttributeKey.stringKey("tenant.id"), "acme",
+    AttributeKey.stringKey("deployment.region"), "us-east-1"));
+
+// Later: replace one value and drop another
+LDObserve.setAttributes(Attributes.of(AttributeKey.stringKey("deployment.region"), "eu-west-1"));
+LDObserve.removeAttributes(List.of("tenant.id"));
+```
+
+- Attributes passed to an `LDObserve` call, or set on a span builder, win over global attributes of the same key; global attributes only fill keys that are missing. Resource attributes are separate and unchanged.
+- A span gets the global attributes current when it starts.
+- Keys are matched by name, whatever the attribute type.
+- The attributes are held in memory for the life of the process (they survive `LDObserve.shutdown()`), are shared by every thread, and can be set before the plugin starts.
+- Metrics recorded directly on a meter from `LDObserve.getOpenTelemetry()` do not get them; record through `LDObserve` instead.
+
+### Custom span and log processors
+
+`spanProcessors` and `logProcessors` add your own OpenTelemetry `SpanProcessor` and `LogRecordProcessor` instances to the plugin's pipeline, for example to enrich or redact telemetry before it is exported. `spanProcessor` and `logProcessor` add one at a time.
+
+```java
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.logs.LogRecordProcessor;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+
+AttributeKey<String> email = AttributeKey.stringKey("user.email");
+
+SpanProcessor addPool = new SpanProcessor() {
+    @Override
+    public void onStart(Context parentContext, ReadWriteSpan span) {
+        span.setAttribute("host.pool", "blue");
+    }
+    @Override public boolean isStartRequired() { return true; }
+    @Override public void onEnd(ReadableSpan span) {}
+    @Override public boolean isEndRequired() { return false; }
+};
+LogRecordProcessor redactEmail = (context, logRecord) -> {
+    if (logRecord.toLogRecordData().getAttributes().get(email) != null) {
+        logRecord.setAttribute(email, "[redacted]");
+    }
+};
+
+ObservabilityOptions options = new ObservabilityOptions.Builder()
+    .serviceName("my-service")
+    .spanProcessors(List.of(addPool))
+    .logProcessors(List.of(redactEmail))
+    .build();
+```
+
+- Processors run in list order, after the plugin adds the global attributes and before its own processing: the `logsApiLevel` filter, the LaunchDarkly sampling rules and export. Changes a processor makes (in `onStart` for spans) are exported, and sampling still applies.
+- A processor that throws is logged and skipped for that call; the span or record is still exported. `forceFlush` and `shutdown` are forwarded when the plugin flushes or shuts down.
+- They are read when the providers start. A second `LDClient` with another SDK key does not add its processors (see [Lifecycle and multiple clients](#lifecycle-and-multiple-clients)).
+- The interfaces come from the OpenTelemetry SDK (`opentelemetry-sdk`), which the plugin exposes as a compile-scoped dependency for this reason.
+
 ## Configuration Options
 
 | Option | Default | Description |
@@ -275,6 +342,8 @@ Every `LDObserve` method is safe to call before the plugin is initialized; such
 | `logsApiLevel` | `INFO` | Minimum severity for exported logs. `NONE` disables log export. |
 | `debug` | `false` | Enable debug logging |
 | `manualStart` | `false` | Delay OTLP provider startup until `LDObserve.start(sdkKey, options)` |
+| `spanProcessors` / `spanProcessor` | none | Your own span processors, run in order before sampling and export. See [Custom span and log processors](#custom-span-and-log-processors) |
+| `logProcessors` / `logProcessor` | none | Your own log record processors, run in order before the level filter, sampling and export |
 | `shutdownHook` | `true` | Flush and stop the providers from a JVM shutdown hook. Set `false` under Spring Boot or another framework that owns shutdown, and call `LDObserve.shutdown()` from its close callback, so telemetry from requests drained during graceful shutdown is not dropped |
 
 ### Proxies and custom headers
@@ -307,7 +376,7 @@ Levels are `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `FATAL` and `NONE`, matchi
 ./gradlew :lib:publishToMavenLocal   # then see ../../e2e/java-plugin for consumer and server examples
 ```
 
-`:lib:build` fails if OkHttp, Okio or Kotlin appear on the runtime classpath, if the generated POM does not expose `opentelemetry-api` at compile scope, or if it names anything outside `io.opentelemetry`. `./gradlew :lib:test -PldServerSdkVersion=7.10.1` runs the tests against the oldest supported SDK.
+`:lib:build` fails if OkHttp, Okio or Kotlin appear on the runtime classpath, if the generated POM does not expose `opentelemetry-api` and `opentelemetry-sdk` at compile scope, or if it names anything outside `io.opentelemetry`. `./gradlew :lib:test -PldServerSdkVersion=7.10.1` runs the tests against the oldest supported SDK.
 
 The unit tests run the sampling scenarios shared with the Go, .NET, Python and Node SDKs (`lib/src/test/resources/sampling`, copied from `go/internal/otel`). `:lib:build` also fails if those copies drift from Go's; copy the files over when the scenarios change.
 

sdk/@launchdarkly/observability-java/lib/build.gradle

@@ -41,7 +41,9 @@ dependencies {
     // types such as Attributes and Severity, so consumers must compile against
     // it. 'api' publishes it with compile scope in the POM.
     api "io.opentelemetry:opentelemetry-api:${otelVersion}"
-    implementation "io.opentelemetry:opentelemetry-sdk:${otelVersion}"
+    // ObservabilityOptions takes SDK SpanProcessor and LogRecordProcessor
+    // instances, so the SDK is part of the public API too.
+    api "io.opentelemetry:opentelemetry-sdk:${otelVersion}"
     implementation "io.opentelemetry:opentelemetry-sdk-logs:${otelVersion}"
     implementation "io.opentelemetry:opentelemetry-sdk-metrics:${otelVersion}"
 
@@ -102,7 +104,7 @@ tasks.register('checkRuntimeClasspath') {
 
 tasks.register('checkPublishedPom') {
     group = 'verification'
-    description = 'Checks the generated POM exposes opentelemetry-api at compile scope and depends on nothing but OpenTelemetry.'
+    description = 'Checks the generated POM exposes opentelemetry-api and opentelemetry-sdk at compile scope and depends on nothing but OpenTelemetry.'
     dependsOn 'generatePomFileForReleasePublication'
     def pomFile = layout.buildDirectory.file('publications/release/pom-default.xml')
     doLast {
@@ -114,6 +116,11 @@ tasks.register('checkPublishedPom') {
             throw new GradleException("opentelemetry-api must be a compile-scoped dependency in the POM, found scope '"
                     + (otelApi.size() == 0 ? 'missing' : otelApi.scope.text()) + "'")
         }
+        def otelSdk = deps.find { it.artifactId.text() == 'opentelemetry-sdk' }
+        if (otelSdk.size() == 0 || otelSdk.scope.text() != 'compile') {
+            throw new GradleException("opentelemetry-sdk must be a compile-scoped dependency in the POM, found scope '"
+                    + (otelSdk.size() == 0 ? 'missing' : otelSdk.scope.text()) + "'")
+        }
         def okhttp = deps.find { it.artifactId.text() == 'opentelemetry-exporter-sender-okhttp' }
         if (okhttp.size() != 0) {
             throw new GradleException('The POM must not depend on opentelemetry-exporter-sender-okhttp')

sdk/@launchdarkly/observability-java/lib/src/main/java/com/launchdarkly/observability/LDObserve.java

@@ -1,6 +1,7 @@
 package com.launchdarkly.observability;
 
 import com.launchdarkly.observability.internal.Constants;
+import com.launchdarkly.observability.internal.GlobalAttributes;
 import com.launchdarkly.observability.internal.OtelManager;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.Attributes;
@@ -10,6 +11,8 @@
 import io.opentelemetry.api.trace.StatusCode;
 import io.opentelemetry.api.trace.Tracer;
 
+import java.util.Collection;
+
 /**
  * Static singleton public API for LaunchDarkly Observability.
  *
@@ -22,12 +25,44 @@
 public final class LDObserve {
     private LDObserve() {}
 
+    /**
+     * Merges attributes into the process-wide global attributes, which are
+     * added to every span and log record the plugin exports (flag
+     * evaluations, error spans and spans or records from instrumentation
+     * using {@link #getOpenTelemetry()} included) and to every metric
+     * recorded through {@code LDObserve}. A key already in the global
+     * attributes takes the new value.
+     *
+     * <p>Global attributes never replace an attribute the span, record or
+     * metric call already has: the attributes passed to an {@code LDObserve}
+     * method, or set on a span builder, win; resource attributes are
+     * separate and stay as they are. A span gets the values current when it
+     * starts; changing them later does not touch spans already started.</p>
+     *
+     * <p>Kept in memory for the life of the process, including across
+     * {@link #shutdown()}. Safe to call before the plugin starts, and from
+     * any thread. Keys are matched by name, whatever the attribute type.
+     * Null or empty attributes are ignored.</p>
+     */
+    public static void setAttributes(Attributes attributes) {
+        GlobalAttributes.set(attributes);
+    }
+
+    /**
+     * Removes the global attributes with these key names. See
+     * {@link #setAttributes(Attributes)}. Names that are not set are
+     * ignored, as is a null collection.
+     */
+    public static void removeAttributes(Collection<String> keys) {
+        GlobalAttributes.remove(keys);
+    }
+
     /**
      * Records a gauge metric.
      */
     public static void recordMetric(String name, double value, Attributes attributes) {
         try {
-            OtelManager.getGauge(name).set(value, orEmpty(attributes));
+            OtelManager.getGauge(name).set(value, GlobalAttributes.merge(attributes));
         } catch (LinkageError e) {
             // The synchronous gauge API arrived in OpenTelemetry 1.38. On an
             // older API (Spring Boot 3.2 pins 1.31) the plugin is already off
@@ -39,7 +74,7 @@ public static void recordMetric(String name, double value, Attributes attributes
      * Records a counter metric.
      */
     public static void recordCount(String name, long value, Attributes attributes) {
-        OtelManager.getCounter(name).add(value, orEmpty(attributes));
+        OtelManager.getCounter(name).add(value, GlobalAttributes.merge(attributes));
     }
 
     /**
@@ -53,14 +88,14 @@ public static void recordIncr(String name, Attributes attributes) {
      * Records a histogram metric.
      */
     public static void recordHistogram(String name, double value, Attributes attributes) {
-        OtelManager.getHistogram(name).record(value, orEmpty(attributes));
+        OtelManager.getHistogram(name).record(value, GlobalAttributes.merge(attributes));
     }
 
     /**
      * Records an up/down counter metric.
      */
     public static void recordUpDownCounter(String name, long value, Attributes attributes) {
-        OtelManager.getUpDownCounter(name).add(value, orEmpty(attributes));
+        OtelManager.getUpDownCounter(name).add(value, GlobalAttributes.merge(attributes));
     }
 
     /**