From 94564a811352be261f4a1bb5cfa3aac810d46b30 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 15:38:19 -0400 Subject: [PATCH 01/16] feat(client): pass the evaluated variation to handlers Keep the raw variation value that client.variation returns and hand it to handlers as variables["__ldValue"], next to __ld. It stays out of the __ld track data on purpose: that data rides on every $ld:ai:* track event, and the value includes the prompt. --- packages/client/src/launchdarkly_ai_server/lifecycle.py | 4 ++++ packages/client/src/launchdarkly_ai_server/tracking.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index e7bb8fc7..67118fca 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -627,6 +627,10 @@ async def extract_variation( # Pinned model-config identity; keys are omitted when absent so that # tracking payloads never carry ``None`` values. **model_stamps_from_meta(ld_meta), + # The variation exactly as the LD SDK returned it, ``_ldMeta`` included. + # Written to the root span's ``feature_flag`` event as + # ``feature_flag.result.value``; never copied into tracking payloads. + "evaluatedValue": raw, } # Strip _ldMeta for config parsing diff --git a/packages/client/src/launchdarkly_ai_server/tracking.py b/packages/client/src/launchdarkly_ai_server/tracking.py index acc4f8b5..104c6c57 100644 --- a/packages/client/src/launchdarkly_ai_server/tracking.py +++ b/packages/client/src/launchdarkly_ai_server/tracking.py @@ -212,6 +212,8 @@ async def execute_and_track( "ldContext": {**user_context}, "__ld": track_data, } + if isinstance(meta, dict) and "evaluatedValue" in meta: + merged_variables["__ldValue"] = meta["evaluatedValue"] start_time = time.monotonic() @@ -312,6 +314,8 @@ async def execute_and_stream( "ldContext": {**user_context}, "__ld": track_data, } + if isinstance(meta, dict) and "evaluatedValue" in meta: + merged_variables["__ldValue"] = meta["evaluatedValue"] start_time = time.monotonic() full_text = "" From 9db397d4331c3c6133b038a11b682b4b599af10b Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 15:38:19 -0400 Subject: [PATCH 02/16] feat(client): add result variant and value to the feature_flag span event AI Config monitoring only finds traces whose feature_flag event carries feature_flag.result.value, which the LaunchDarkly observability plugins always send. Write the variation key as feature_flag.result.variant (conditionally required by the OpenTelemetry feature-flag conventions) and the evaluated value as feature_flag.result.value, serialized with json.dumps exactly like the ldotel tracing hook. Both are omitted when there is nothing to write, for example on native-graph spans. --- TELEMETRY-CONTRACT.md | 2 + .../src/launchdarkly_ai_server/utils.py | 14 ++++++- .../client/tests/test_ld_span_attributes.py | 38 +++++++++++++++++++ packages/openai-agents/tests/test_handler.py | 7 +++- 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/TELEMETRY-CONTRACT.md b/TELEMETRY-CONTRACT.md index 1016d8f3..7fa5bacb 100644 --- a/TELEMETRY-CONTRACT.md +++ b/TELEMETRY-CONTRACT.md @@ -105,6 +105,8 @@ The root also carries one span event, `feature_flag`, with these event attribute |---|---| | `feature_flag.key` | config key | | `feature_flag.provider.name` | `LaunchDarkly` | +| `feature_flag.result.variant` | variation key, only when non-empty (graph spans may have none) | +| `feature_flag.result.value` | `json.dumps` of the raw variation as the LaunchDarkly SDK returned it (`_ldMeta` included), same serialization as the `ldotel` tracing hook with value inclusion on; not gated by content capture; absent when there is no evaluated value (native-graph spans) | | `feature_flag.set.id` | environment id, only when present | | `feature_flag.context.id` | canonical context key, only when `variables.ldContext` has a usable identity | | `feature_flag.contextKeys` | compact JSON of per-kind keys, only when identity is present | diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 5e1fabba..7a348345 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -790,7 +790,12 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: Span event (required for AI Config Monitoring Traces tab correlation): ``name='feature_flag'`` with ``feature_flag.key``, - ``feature_flag.provider.name``, ``feature_flag.set.id`` (when + ``feature_flag.provider.name``, ``feature_flag.result.variant`` (the + served variation key, when non-empty), ``feature_flag.result.value`` + (``json.dumps`` of the raw variation in ``variables['__ldValue']``, as the + ``ldotel`` tracing hook writes it with value inclusion on; omitted when + there is no evaluated value, e.g. native-graph spans), + ``feature_flag.set.id`` (when ``LD_ENVIRONMENT_ID`` is set or the TS SDK auto-resolved it), ``feature_flag.context.id``, and ``feature_flag.contextKeys`` (when ``ldContext`` has a usable identity). @@ -811,6 +816,13 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: "feature_flag.key": ld.get("configKey", ""), "feature_flag.provider.name": "LaunchDarkly", } + variation_key = ld.get("variationKey") + if isinstance(variation_key, str) and variation_key: + feature_flag_attrs["feature_flag.result.variant"] = variation_key + if "__ldValue" in variables: + feature_flag_attrs["feature_flag.result.value"] = json.dumps( + variables["__ldValue"] + ) if ld.get("environmentId"): feature_flag_attrs["feature_flag.set.id"] = ld["environmentId"] diff --git a/packages/client/tests/test_ld_span_attributes.py b/packages/client/tests/test_ld_span_attributes.py index bd0b470a..b7fc515c 100644 --- a/packages/client/tests/test_ld_span_attributes.py +++ b/packages/client/tests/test_ld_span_attributes.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from typing import Any import pytest @@ -194,3 +195,40 @@ def test_malformed_ld_context_emits_none_of_the_three_and_does_not_throw( span = FakeSpan() set_ld_span_attributes(span, variables) _assert_no_context_identity(span) + + +class TestResultVariant: + def test_variant_is_the_variation_key(self) -> None: + span = FakeSpan() + set_ld_span_attributes(span, _vars(None)) + assert _feature_flag(span)["feature_flag.result.variant"] == "variation-a" + + @pytest.mark.parametrize("variation_key", ["", None]) + def test_empty_or_missing_variation_key_omits_the_variant( + self, variation_key: Any + ) -> None: + ld = {**LD_FIXTURE, "variationKey": variation_key} + if variation_key is None: + del ld["variationKey"] + span = FakeSpan() + set_ld_span_attributes(span, {"__ld": ld}) + assert "feature_flag.result.variant" not in _feature_flag(span) + + def test_value_is_the_raw_variation_serialized_like_the_tracing_hook( + self, + ) -> None: + raw = { + "model": {"name": "gpt-x", "parameters": {"temperature": 0.5}}, + "provider": {"name": "openai"}, + "messages": [{"role": "system", "content": "Olá"}], + "_ldMeta": {"enabled": True, "variationKey": "variation-a"}, + } + span = FakeSpan() + set_ld_span_attributes(span, {**_vars(None), "__ldValue": raw}) + # ldotel's TracingHook writes json.dumps(detail.value) with defaults. + assert _feature_flag(span)["feature_flag.result.value"] == json.dumps(raw) + + def test_value_is_absent_without_an_evaluated_value(self) -> None: + span = FakeSpan() + set_ld_span_attributes(span, {"__ld": {**LD_FIXTURE, "variationKey": ""}}) + assert "feature_flag.result.value" not in _feature_flag(span) diff --git a/packages/openai-agents/tests/test_handler.py b/packages/openai-agents/tests/test_handler.py index db1737f5..fcdf01f0 100644 --- a/packages/openai-agents/tests/test_handler.py +++ b/packages/openai-agents/tests/test_handler.py @@ -834,7 +834,8 @@ async def test_feature_flag_event_on_root_only(self) -> None: "variationKey": "v1", "runId": "run-abc", "environmentId": "env-1", - } + }, + "__ldValue": {"model": {"name": "m"}, "_ldMeta": {"variationKey": "v1"}}, } with ctx, _patched_agents(agents_mod): await create_openai_agents_handler()(CONFIG, "q", {}, vs) @@ -844,6 +845,10 @@ async def test_feature_flag_event_on_root_only(self) -> None: { "feature_flag.key": "my-config", "feature_flag.provider.name": "LaunchDarkly", + "feature_flag.result.variant": "v1", + "feature_flag.result.value": ( + '{"model": {"name": "m"}, "_ldMeta": {"variationKey": "v1"}}' + ), "feature_flag.set.id": "env-1", }, ) From 56ac29663aff9f3ebff09767f0cce48bd103cd98 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 16:04:22 -0400 Subject: [PATCH 03/16] test: add the feature_flag result keys to the span vocabulary lock The parity suite fails on any span attribute key it does not know about, and CI caught feature_flag.result.variant. Both result keys are deliberate: they are the OpenTelemetry feature-flag convention's served variant and evaluated value. --- tests/test_cross_handler_parity.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index 60a3375f..5dc90ed5 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -376,6 +376,10 @@ def test_the_langchain_provider_name_is_the_configured_name( "feature_flag", "feature_flag.key", "feature_flag.provider.name", + # OTel feature-flag semconv: the served variation key, and the evaluated value (the raw + # variation with content capture on, else the JSON-encoded key). + "feature_flag.result.variant", + "feature_flag.result.value", "feature_flag.set.id", # AIC-3230: evaluation-context identity on the root feature_flag event / span. # `context.contextKeys` is the f-string prefix; the keys actually emitted are From 3cb53725072e3eead2c045eee58416fccd7d154b Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 16:04:22 -0400 Subject: [PATCH 04/16] feat(client): respect content capture for feature_flag.result.value The raw variation holds the prompt, so it must not leave the process unless the user opted in to content capture. set_ld_span_attributes takes the handler's resolved capture_content (the existing setting: explicit on the handler, else the configure() default, else off). With capture on and a raw variation present, result.value is json.dumps of that variation, as before. Otherwise it is json.dumps of the variation key when there is one, which also gives graph() roots a value. Otherwise it is omitted. Every handler package passes its own resolved setting to the root span, and the graph() roots pass the process default. Native-graph roots keep the default; they have no variation key and no raw variation, so nothing is written either way. The api report records the new keyword argument. --- TELEMETRY-CONTRACT.md | 2 +- .../launchdarkly_ai_claude_agents/handler.py | 4 +- .../launchdarkly_ai_claude_agents/spans.py | 6 ++- .../handler.py | 4 +- .../launchdarkly_ai_claude_messages/spans.py | 6 ++- .../api-report/launchdarkly_ai_server.txt | 2 +- .../src/launchdarkly_ai_server/graph.py | 14 ++++-- .../src/launchdarkly_ai_server/utils.py | 20 ++++++--- packages/client/tests/test_graph_stream.py | 41 +++++++++++++++++ .../client/tests/test_ld_span_attributes.py | 45 ++++++++++++++----- .../handler.py | 4 +- .../launchdarkly_ai_langchain_agents/spans.py | 6 ++- .../handler.py | 4 +- .../spans.py | 6 ++- .../launchdarkly_ai_litellm_agents/handler.py | 13 ++++-- .../handler.py | 13 ++++-- .../launchdarkly_ai_openai_agents/handler.py | 4 +- .../launchdarkly_ai_openai_agents/spans.py | 6 ++- packages/openai-agents/tests/test_handler.py | 5 +-- .../handler.py | 4 +- .../launchdarkly_ai_openai_messages/spans.py | 6 ++- .../handler.py | 2 +- .../launchdarkly_ai_opencode_agents/spans.py | 6 ++- .../src/launchdarkly_ai_typesafe/handler.py | 4 +- .../launchdarkly_ai_vercel_agents/handler.py | 9 +++- .../launchdarkly_ai_vercel_agents/spans.py | 6 ++- .../handler.py | 9 +++- .../launchdarkly_ai_vercel_messages/spans.py | 6 ++- 28 files changed, 186 insertions(+), 71 deletions(-) diff --git a/TELEMETRY-CONTRACT.md b/TELEMETRY-CONTRACT.md index 7fa5bacb..6d71b5c4 100644 --- a/TELEMETRY-CONTRACT.md +++ b/TELEMETRY-CONTRACT.md @@ -106,7 +106,7 @@ The root also carries one span event, `feature_flag`, with these event attribute | `feature_flag.key` | config key | | `feature_flag.provider.name` | `LaunchDarkly` | | `feature_flag.result.variant` | variation key, only when non-empty (graph spans may have none) | -| `feature_flag.result.value` | `json.dumps` of the raw variation as the LaunchDarkly SDK returned it (`_ldMeta` included), same serialization as the `ldotel` tracing hook with value inclusion on; not gated by content capture; absent when there is no evaluated value (native-graph spans) | +| `feature_flag.result.value` | with content capture on (the existing `capture_content` setting, including the `configure()` default) and a raw variation present: `json.dumps` of the variation as the LaunchDarkly SDK returned it (`_ldMeta` included), the same serialization as the `ldotel` tracing hook with value inclusion on. Otherwise, when the variation key is non-empty: `json.dumps` of the key (for example `"\"v1\""`), which also covers `graph()` roots. Otherwise absent (native-graph spans) | | `feature_flag.set.id` | environment id, only when present | | `feature_flag.context.id` | canonical context key, only when `variables.ldContext` has a usable identity | | `feature_flag.contextKeys` | compact JSON of per-kind keys, only when identity is present | diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index 664b1dbf..03e2de51 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -787,7 +787,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the finally below can tell an open root # from a closed one without asking the span. A mock span answers `is_recording()` @@ -1052,7 +1052,7 @@ async def _stream_gen( exit abandons it, and asyncio's finalizer later raises ``RuntimeError`` when it is suspended inside a real await in the SDK. """ - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) prompt, system_prompt = _build_prompt(config, user_input, variables, history) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/spans.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/spans.py index f25adc3e..2ef0ab54 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/spans.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/spans.py @@ -81,7 +81,9 @@ def tool_display_name(provider_name: str) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -92,7 +94,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, PROVIDER, model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index ebd17db4..0fd1be1e 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -570,7 +570,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the `finally` can tell an open root from a # closed one without asking the span. A mock span answers `is_recording()` truthily, and the @@ -712,7 +712,7 @@ async def _stream_gen( ``ended`` stops the success, failure and abandonment paths from ending the same span twice. """ - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) messages, system = _build_messages( diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/spans.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/spans.py index 3d55f7de..b87e882c 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/spans.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/spans.py @@ -46,7 +46,9 @@ def model_name(config: AiConfigRep) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -57,7 +59,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, PROVIDER, model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt index 4ed2c00e..51821839 100644 --- a/packages/client/api-report/launchdarkly_ai_server.txt +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -232,7 +232,7 @@ SDKOptions [class, @dataclass(frozen=True)] capture_content: bool = False set_conversation_id_if_absent(span: Any, conversation: str) -> None set_input_content_attributes(span: Any, capture: bool, *, system_instructions: str | None = None, messages: list[SpanMessage] | None = None, tool_definitions: list[ToolDefinitionInput] | None = None) -> None -set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None +set_ld_span_attributes(span: Any, variables: dict[str, Any] | None, *, capture_content: bool = False) -> None set_model_identity_attributes(span: Any, provider_name: str, request_model: str, legacy_system: str | None = None) -> None set_output_content_attributes(span: Any, capture: bool, messages: list[SpanMessage]) -> None set_skill_store(store: SkillStore | None) -> None diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index 4f8677b6..5f6ebdf3 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -895,7 +895,7 @@ async def _invoke( from opentelemetry import trace from .judges import run_judges - from .lifecycle import get_client + from .lifecycle import effective_capture_content, get_client ld_ctx = to_ld_context(get_client(), context) @@ -928,7 +928,9 @@ async def _invoke( tracer = trace.get_tracer("@launchdarkly/ai-server") with tracer.start_as_current_span("launchdarkly.graph") as span: set_ld_span_attributes( - span, {"__ld": graph_track_data, "ldContext": context} + span, + {"__ld": graph_track_data, "ldContext": context}, + capture_content=effective_capture_content(None), ) start_time = time.monotonic() @@ -1120,7 +1122,7 @@ async def _stream_events( from opentelemetry.trace import Status, StatusCode, set_span_in_context from .judges import run_judges - from .lifecycle import get_client + from .lifecycle import effective_capture_content, get_client resolved_input = user_input or "" ld_ctx = to_ld_context(get_client(), context) @@ -1153,7 +1155,11 @@ async def _stream_events( tracer = trace.get_tracer("@launchdarkly/ai-server") span = tracer.start_span("launchdarkly.graph", context=caller_context) - set_ld_span_attributes(span, {"__ld": graph_track_data, "ldContext": context}) + set_ld_span_attributes( + span, + {"__ld": graph_track_data, "ldContext": context}, + capture_content=effective_capture_content(None), + ) span_context = set_span_in_context(span, caller_context) ended: set[int] = set() start_time = time.monotonic() diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 7a348345..dd877c41 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -768,7 +768,9 @@ def _context_identity_from_ld_context( return None -def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: +def set_ld_span_attributes( + span: Any, variables: dict[str, Any] | None, *, capture_content: bool = False +) -> None: """ Sets LaunchDarkly config-identifying attributes on an OTel span and emits the ``feature_flag`` span event required by the AI Config Monitoring Traces @@ -792,13 +794,17 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: ``name='feature_flag'`` with ``feature_flag.key``, ``feature_flag.provider.name``, ``feature_flag.result.variant`` (the served variation key, when non-empty), ``feature_flag.result.value`` - (``json.dumps`` of the raw variation in ``variables['__ldValue']``, as the - ``ldotel`` tracing hook writes it with value inclusion on; omitted when - there is no evaluated value, e.g. native-graph spans), - ``feature_flag.set.id`` (when + (see below), ``feature_flag.set.id`` (when ``LD_ENVIRONMENT_ID`` is set or the TS SDK auto-resolved it), ``feature_flag.context.id``, and ``feature_flag.contextKeys`` (when ``ldContext`` has a usable identity). + + ``feature_flag.result.value`` follows content capture. With + *capture_content* on and the raw variation in ``variables['__ldValue']``, + it is ``json.dumps`` of that variation, as the ``ldotel`` tracing hook + writes it with value inclusion on. Otherwise, when the variation key is + non-empty, it is ``json.dumps`` of the key, so a prompt never leaves the + process without consent. Otherwise it is omitted (native-graph spans). """ span.set_attribute("launchdarkly.operation.type", "gen_ai") if not variables: @@ -819,10 +825,12 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: variation_key = ld.get("variationKey") if isinstance(variation_key, str) and variation_key: feature_flag_attrs["feature_flag.result.variant"] = variation_key - if "__ldValue" in variables: + if capture_content and "__ldValue" in variables: feature_flag_attrs["feature_flag.result.value"] = json.dumps( variables["__ldValue"] ) + elif isinstance(variation_key, str) and variation_key: + feature_flag_attrs["feature_flag.result.value"] = json.dumps(variation_key) if ld.get("environmentId"): feature_flag_attrs["feature_flag.set.id"] = ld["environmentId"] diff --git a/packages/client/tests/test_graph_stream.py b/packages/client/tests/test_graph_stream.py index f3f6d624..0119c36b 100644 --- a/packages/client/tests/test_graph_stream.py +++ b/packages/client/tests/test_graph_stream.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +import json from collections.abc import AsyncGenerator, Iterator from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -988,6 +989,46 @@ async def test_graph_span_carries_config_identity( assert len(flag_events) == 1 assert (flag_events[0].attributes or {})["feature_flag.key"] == "graph-key" + @pytest.mark.parametrize("mode", ["invoke", "stream"]) + async def test_graph_span_event_carries_the_graph_variation( + self, mode: str + ) -> None: + """A graph root has no raw variation, so its result value is the JSON-encoded key.""" + client = _make_client( + { + "root": "root-node", + "edges": {"root-node": [{"key": "leaf-node"}]}, + "_ldMeta": {"enabled": True, "variationKey": "graph-v1", "version": 3}, + } + ) + lifecycle_module._set_client_for_testing(client) + try: + g = graph("graph-key", handlers=[_make_streaming_handler(["ok"])]) + if mode == "invoke": + await g.invoke("hi", CONTEXT) + else: + await _collect(g.stream("hi", CONTEXT)) + finally: + lifecycle_module._reset_for_testing() + + graph_spans = [s for s in _finished() if s.name == "launchdarkly.graph"] + event = next(e for e in graph_spans[0].events if e.name == "feature_flag") + attrs = event.attributes or {} + assert attrs["feature_flag.result.variant"] == "graph-v1" + assert attrs["feature_flag.result.value"] == json.dumps("graph-v1") + + async def test_graph_span_without_a_variation_key_omits_the_result( + self, mock_ld_client: MagicMock + ) -> None: + await graph("graph-key", handlers=[_make_streaming_handler(["ok"])]).invoke( + "hi", CONTEXT + ) + graph_spans = [s for s in _finished() if s.name == "launchdarkly.graph"] + event = next(e for e in graph_spans[0].events if e.name == "feature_flag") + attrs = event.attributes or {} + assert "feature_flag.result.variant" not in attrs + assert "feature_flag.result.value" not in attrs + async def test_graph_parent_captured_at_stream_call_time( self, mock_ld_client: MagicMock ) -> None: diff --git a/packages/client/tests/test_ld_span_attributes.py b/packages/client/tests/test_ld_span_attributes.py index b7fc515c..f89b70d2 100644 --- a/packages/client/tests/test_ld_span_attributes.py +++ b/packages/client/tests/test_ld_span_attributes.py @@ -19,6 +19,14 @@ } +RAW_VARIATION: dict[str, Any] = { + "model": {"name": "gpt-x", "parameters": {"temperature": 0.5}}, + "provider": {"name": "openai"}, + "messages": [{"role": "system", "content": "Olá"}], + "_ldMeta": {"enabled": True, "variationKey": "variation-a"}, +} + + class FakeSpan: def __init__(self) -> None: self.attributes: dict[str, Any] = {} @@ -214,21 +222,34 @@ def test_empty_or_missing_variation_key_omits_the_variant( set_ld_span_attributes(span, {"__ld": ld}) assert "feature_flag.result.variant" not in _feature_flag(span) - def test_value_is_the_raw_variation_serialized_like_the_tracing_hook( + def test_capture_on_writes_the_raw_variation_like_the_tracing_hook( self, ) -> None: - raw = { - "model": {"name": "gpt-x", "parameters": {"temperature": 0.5}}, - "provider": {"name": "openai"}, - "messages": [{"role": "system", "content": "Olá"}], - "_ldMeta": {"enabled": True, "variationKey": "variation-a"}, - } - span = FakeSpan() - set_ld_span_attributes(span, {**_vars(None), "__ldValue": raw}) + span = FakeSpan() + set_ld_span_attributes( + span, {**_vars(None), "__ldValue": RAW_VARIATION}, capture_content=True + ) # ldotel's TracingHook writes json.dumps(detail.value) with defaults. - assert _feature_flag(span)["feature_flag.result.value"] == json.dumps(raw) + assert _feature_flag(span)["feature_flag.result.value"] == json.dumps( + RAW_VARIATION + ) - def test_value_is_absent_without_an_evaluated_value(self) -> None: + def test_capture_off_writes_the_variation_key_not_the_prompt(self) -> None: span = FakeSpan() - set_ld_span_attributes(span, {"__ld": {**LD_FIXTURE, "variationKey": ""}}) + set_ld_span_attributes(span, {**_vars(None), "__ldValue": RAW_VARIATION}) + assert _feature_flag(span)["feature_flag.result.value"] == '"variation-a"' + + def test_capture_on_without_a_raw_variation_falls_back_to_the_key(self) -> None: + span = FakeSpan() + set_ld_span_attributes(span, _vars(None), capture_content=True) + assert _feature_flag(span)["feature_flag.result.value"] == '"variation-a"' + + @pytest.mark.parametrize("capture", [True, False]) + def test_no_raw_variation_and_no_key_omits_the_value(self, capture: bool) -> None: + span = FakeSpan() + set_ld_span_attributes( + span, + {"__ld": {**LD_FIXTURE, "variationKey": ""}}, + capture_content=capture, + ) assert "feature_flag.result.value" not in _feature_flag(span) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index 5346a412..eaf57be3 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -671,7 +671,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the `finally` below can tell an open root # from a closed one without asking the span. A mock span answers `is_recording()` @@ -835,7 +835,7 @@ async def _stream_gen( """ import importlib - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) system_prompt = _extract_system_prompt(config, variables) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/spans.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/spans.py index 3621de06..c17f0a7a 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/spans.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/spans.py @@ -75,7 +75,9 @@ def serving_provider(config: AiConfigRep) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -88,7 +90,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: set_model_identity_attributes( span, serving_provider(config), model_name(config), legacy_system="langchain" ) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index 78f36622..2b09a310 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -802,7 +802,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the `finally` can tell an open root from a # closed one without asking the span. A mock span answers `is_recording()` truthily, and the @@ -1140,7 +1140,7 @@ async def _stream_gen( base_model = await _resolve_base_model(config, llm, importlib) - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) initial_messages = _build_messages(config, user_input, variables, history) diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/spans.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/spans.py index 1bbdc09b..9bef62e3 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/spans.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/spans.py @@ -63,7 +63,9 @@ def serving_provider(config: AiConfigRep) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -76,7 +78,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: set_model_identity_attributes( span, serving_provider(config), model_name(config), LEGACY_SYSTEM ) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py b/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py index 065a1e69..5278bef1 100644 --- a/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py +++ b/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py @@ -309,7 +309,12 @@ def _direct_usage(raw: Any) -> SpanUsage: ) -def _root_span(config_value: AiConfigRep, variables: dict[str, Any]) -> Any: +def _root_span( + config_value: AiConfigRep, + variables: dict[str, Any], + *, + capture_content: bool = False, +) -> Any: model_name = _model_config(config_value)[0] provider = str( (config_value.get("provider") or {}).get("name") or "litellm" @@ -319,7 +324,7 @@ def _root_span(config_value: AiConfigRep, variables: dict[str, Any]) -> Any: ) span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, provider, model_name, "litellm") - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span @@ -588,7 +593,7 @@ async def call( history, factory, ) - span = _root_span(config_value, values) + span = _root_span(config_value, values, capture_content=capture_content) parent = _parent_context(span) model_name = _model_config(config_value)[0] ended: set[int] = set() @@ -675,7 +680,7 @@ async def _stream( agent, prompt, instructions, max_turns = _agent_and_prompt( config_value, user_input, handlers, variables, history, model_factory ) - span = _root_span(config_value, variables) + span = _root_span(config_value, variables, capture_content=capture_content) parent = _parent_context(span) streamed: Any = None completed = False diff --git a/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py b/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py index 6ded1a1b..64407a12 100644 --- a/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py +++ b/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py @@ -277,7 +277,12 @@ def _request( return parameters -def _root_span(config_value: AiConfigRep, variables: dict[str, Any]) -> Any: +def _root_span( + config_value: AiConfigRep, + variables: dict[str, Any], + *, + capture_content: bool = False, +) -> Any: span = trace.get_tracer("@launchdarkly/ai-litellm-messages").start_span( "invoke_agent" ) @@ -285,7 +290,7 @@ def _root_span(config_value: AiConfigRep, variables: dict[str, Any]) -> Any: set_model_identity_attributes( span, _provider_name(config_value), _model_name(config_value), "litellm" ) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span @@ -464,7 +469,7 @@ async def call( values = variables or {} messages = _build_messages(config_value, user_input, values, history) tools = _build_tools(config_value, handlers) - root = _root_span(config_value, values) + root = _root_span(config_value, values, capture_content=capture_content) parent = _parent_context(root) total = SpanUsage() usage_reported = False @@ -605,7 +610,7 @@ async def _stream( ) -> AsyncGenerator[dict[str, Any], None]: messages = _build_messages(config_value, user_input, variables, history) tools = _build_tools(config_value, handlers) - root = _root_span(config_value, variables) + root = _root_span(config_value, variables, capture_content=capture_content) parent = _parent_context(root) total = SpanUsage() usage_reported = False diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index d68c60f2..bdf5ad17 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -684,7 +684,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared once this path (success or failure) has ended the root, so the `finally` below can # tell an open root from a closed one without asking the span. A mock span answers @@ -810,7 +810,7 @@ async def _stream_gen( agents_mod = importlib.import_module("agents") Runner = agents_mod.Runner - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) agent, prompt, instructions = _build_agent_and_prompt( diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/spans.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/spans.py index 128fe8d0..7a06aae5 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/spans.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/spans.py @@ -75,7 +75,9 @@ def model_name(config: AiConfigRep) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -86,7 +88,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, PROVIDER, model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/openai-agents/tests/test_handler.py b/packages/openai-agents/tests/test_handler.py index fcdf01f0..1948a7b1 100644 --- a/packages/openai-agents/tests/test_handler.py +++ b/packages/openai-agents/tests/test_handler.py @@ -846,9 +846,8 @@ async def test_feature_flag_event_on_root_only(self) -> None: "feature_flag.key": "my-config", "feature_flag.provider.name": "LaunchDarkly", "feature_flag.result.variant": "v1", - "feature_flag.result.value": ( - '{"model": {"name": "m"}, "_ldMeta": {"variationKey": "v1"}}' - ), + # Content capture is off, so the key stands in for the variation. + "feature_flag.result.value": '"v1"', "feature_flag.set.id": "env-1", }, ) diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index 5e804e41..4b77ac03 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -416,7 +416,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the `finally` can tell an open root from a # closed one without asking the span. A mock span answers `is_recording()` truthily, and @@ -626,7 +626,7 @@ async def _stream_gen( the root span is never ended, so it is never exported, and the whole run disappears from AI Config Monitoring along with the ``feature_flag`` event it carries. """ - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) ended: set[int] = set() diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/spans.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/spans.py index 3a78a2d6..dc4fc056 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/spans.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/spans.py @@ -81,7 +81,9 @@ def _attr(obj: Any, name: str) -> Any: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -92,7 +94,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, PROVIDER, model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py index 735598a9..9619a417 100644 --- a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py +++ b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py @@ -889,7 +889,7 @@ async def _stream_impl( history: list[dict[str, Any]] | None = None, ) -> AsyncGenerator[dict[str, Any], None]: capture = effective_capture_content(self._capture_content_override) - span = start_root_span(config, variables or {}) + span = start_root_span(config, variables or {}, capture_content=capture) parent = parent_context_of(span) ended: set[int] = set() open_tool_spans: dict[str, Any] = {} diff --git a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/spans.py b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/spans.py index 84ec5c46..7d3f9d52 100644 --- a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/spans.py +++ b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/spans.py @@ -51,14 +51,16 @@ def provider_name(config: AiConfigRep) -> str: return "opencode" -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent.""" if not _HAS_OTEL: return None span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, provider_name(config), model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py b/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py index 7509f9d7..ccf96730 100644 --- a/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py +++ b/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py @@ -115,7 +115,9 @@ async def _call( chat = _TRACER.start_span( f"chat {span_model}", context=trace.set_span_in_context(root) ) - set_ld_span_attributes(root, dict(variables) if variables else None) + set_ld_span_attributes( + root, dict(variables) if variables else None, capture_content=capture + ) set_model_identity_attributes(root, "typesafe", span_model) set_model_identity_attributes(chat, "typesafe", span_model) if capture: diff --git a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py index a4b91379..f66f529c 100644 --- a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py +++ b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py @@ -24,6 +24,7 @@ parse_template, report_usage, ) +from launchdarkly_ai_server.lifecycle import effective_capture_content from launchdarkly_ai_server.parameter_forwarding import select_forwarded_parameters from launchdarkly_ai_server.utils import model_parameters @@ -523,7 +524,9 @@ async def run( history: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: vs = variables or {} - span = start_root_span(cfg, vs) + span = start_root_span( + cfg, vs, capture_content=effective_capture_content(capture_content) + ) parent = parent_context_of(span) turns = _ChatTurns(cfg, parent) failed = False @@ -576,7 +579,9 @@ async def stream( history: list[dict[str, Any]] | None = None, ) -> AsyncGenerator[dict[str, Any], None]: vs = variables or {} - span = start_root_span(cfg, vs) + span = start_root_span( + cfg, vs, capture_content=effective_capture_content(capture_content) + ) parent = parent_context_of(span) turns = _ChatTurns(cfg, parent) completed = False diff --git a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/spans.py b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/spans.py index 0762842f..ae2e591d 100644 --- a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/spans.py +++ b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/spans.py @@ -24,11 +24,13 @@ def serving_provider(config: AiConfigRep) -> str: return str(config.get("provider", {}).get("name") or "vercel").lower() -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, serving_provider(config), model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py index 53e310c7..ed34fc23 100644 --- a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py +++ b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py @@ -24,6 +24,7 @@ parse_template, report_usage, ) +from launchdarkly_ai_server.lifecycle import effective_capture_content from launchdarkly_ai_server.parameter_forwarding import select_forwarded_parameters from launchdarkly_ai_server.utils import model_parameters @@ -546,7 +547,9 @@ async def invoke( history: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: vs = variables or {} - span = start_root_span(cfg, vs) + span = start_root_span( + cfg, vs, capture_content=effective_capture_content(capture_content) + ) parent = parent_context_of(span) failed = False cancelled = False @@ -599,7 +602,9 @@ async def stream( history: list[dict[str, Any]] | None = None, ) -> AsyncGenerator[dict[str, Any], None]: vs = variables or {} - span = start_root_span(cfg, vs) + span = start_root_span( + cfg, vs, capture_content=effective_capture_content(capture_content) + ) parent = parent_context_of(span) completed = False failed = False diff --git a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/spans.py b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/spans.py index a4a7cef4..1bf9e57d 100644 --- a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/spans.py +++ b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/spans.py @@ -24,11 +24,13 @@ def serving_provider(config: AiConfigRep) -> str: return str(config.get("provider", {}).get("name") or "vercel").lower() -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, serving_provider(config), model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span From 208c1e310b72f24865a160e8f2c26c25e1048406 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 18:00:14 -0400 Subject: [PATCH 05/16] refactor(client): drop the capture setting from graph root spans graph() roots never carry a raw variation, so the capture setting cannot change what their feature_flag event writes. Pass the variables alone, as before, and drop the now-unused imports. --- .../client/src/launchdarkly_ai_server/graph.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index 5f6ebdf3..4f8677b6 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -895,7 +895,7 @@ async def _invoke( from opentelemetry import trace from .judges import run_judges - from .lifecycle import effective_capture_content, get_client + from .lifecycle import get_client ld_ctx = to_ld_context(get_client(), context) @@ -928,9 +928,7 @@ async def _invoke( tracer = trace.get_tracer("@launchdarkly/ai-server") with tracer.start_as_current_span("launchdarkly.graph") as span: set_ld_span_attributes( - span, - {"__ld": graph_track_data, "ldContext": context}, - capture_content=effective_capture_content(None), + span, {"__ld": graph_track_data, "ldContext": context} ) start_time = time.monotonic() @@ -1122,7 +1120,7 @@ async def _stream_events( from opentelemetry.trace import Status, StatusCode, set_span_in_context from .judges import run_judges - from .lifecycle import effective_capture_content, get_client + from .lifecycle import get_client resolved_input = user_input or "" ld_ctx = to_ld_context(get_client(), context) @@ -1155,11 +1153,7 @@ async def _stream_events( tracer = trace.get_tracer("@launchdarkly/ai-server") span = tracer.start_span("launchdarkly.graph", context=caller_context) - set_ld_span_attributes( - span, - {"__ld": graph_track_data, "ldContext": context}, - capture_content=effective_capture_content(None), - ) + set_ld_span_attributes(span, {"__ld": graph_track_data, "ldContext": context}) span_context = set_span_in_context(span, caller_context) ended: set[int] = set() start_time = time.monotonic() From 8702b2cb8e05ecea81f8ee1a7720dd22b68dc8db Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 18:00:14 -0400 Subject: [PATCH 06/16] test: cover content capture on the root span's feature_flag value The parity suite now checks that every handler passes capture_content= to each root span call, since a missing keyword falls back to off and silently drops the value for capture-on runs. One openai-agents case runs the handler with capture on and asserts the event carries the raw variation. --- packages/openai-agents/tests/test_handler.py | 20 ++++++++ tests/test_cross_handler_parity.py | 51 ++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/packages/openai-agents/tests/test_handler.py b/packages/openai-agents/tests/test_handler.py index 1948a7b1..b0765ebf 100644 --- a/packages/openai-agents/tests/test_handler.py +++ b/packages/openai-agents/tests/test_handler.py @@ -823,6 +823,26 @@ async def test_run_totals_on_the_root(self) -> None: assert attrs["gen_ai.usage.output_tokens"] == 5 assert attrs["gen_ai.usage.total_tokens"] == 15 + async def test_capture_on_writes_the_raw_variation_as_the_result_value( + self, + ) -> None: + ctx, rec = _recording() + agents_mod = _fake_agents_module( + run=_make_run([{"output": _text_output("hi")}]) + ) + raw = {"model": {"name": "m"}, "_ldMeta": {"variationKey": "v1"}} + vs = { + "__ld": {"configKey": "my-config", "variationKey": "v1", "runId": "r"}, + "__ldValue": raw, + } + with ctx, _patched_agents(agents_mod): + await create_openai_agents_handler(capture_content=True)( + CONFIG, "q", {}, vs + ) + (name, attrs), *_ = rec.root.events + assert name == "feature_flag" + assert attrs["feature_flag.result.value"] == json.dumps(raw) + async def test_feature_flag_event_on_root_only(self) -> None: ctx, rec = _recording() agents_mod = _fake_agents_module( diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index 5dc90ed5..a53199df 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -681,3 +681,54 @@ def test_abandonment_is_distinct_from_failure(self, name: str) -> None: f"{name} has a close_open_spans that neither records the failure nor hands off to " "one that does, so it closes nothing." ) + + +# --------------------------------------------------------------------------- +# The root span's feature_flag value follows the handler's content capture +# --------------------------------------------------------------------------- + +#: Every handler module that opens a run's root span. The root's feature_flag event writes the raw +#: variation as `feature_flag.result.value` only when capture is on, so each must pass its resolved +#: setting. Left off, the keyword defaults to False and capture-on runs silently lose the value. +ROOT_SPAN_HANDLERS: dict[str, str] = { + "claude-agents": "launchdarkly_ai_claude_agents.handler", + "claude-messages": "launchdarkly_ai_claude_messages.handler", + "langchain-agents": "launchdarkly_ai_langchain_agents.handler", + "langchain-messages": "launchdarkly_ai_langchain_messages.handler", + "litellm-agents": "launchdarkly_ai_litellm_agents.handler", + "litellm-messages": "launchdarkly_ai_litellm_messages.handler", + "openai-agents": "launchdarkly_ai_openai_agents.handler", + "openai-messages": "launchdarkly_ai_openai_messages.handler", + "opencode-agents": "launchdarkly_ai_opencode_agents.handler", + "typesafe": "launchdarkly_ai_typesafe.handler", + "vercel-agents": "launchdarkly_ai_vercel_agents.handler", + "vercel-messages": "launchdarkly_ai_vercel_messages.handler", +} + +_ROOT_SPAN_CALLS = {"start_root_span", "_root_span", "set_ld_span_attributes"} + + +def _root_span_calls(source: str) -> list[ast.Call]: + calls: list[ast.Call] = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Call): + func = node.func + name = ( + func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + ) + if name in _ROOT_SPAN_CALLS: + calls.append(node) + return calls + + +class TestRootSpanCaptureContent: + @pytest.mark.parametrize("name", sorted(ROOT_SPAN_HANDLERS)) + def test_every_root_span_call_passes_capture_content(self, name: str) -> None: + source = _handler_source(ROOT_SPAN_HANDLERS[name]) + calls = _root_span_calls(source) + assert calls, f"{name} opens no root span this check can see." + for call in calls: + assert any(kw.arg == "capture_content" for kw in call.keywords), ( + f"{name} opens a root span without capture_content= at line " + f"{call.lineno}, so feature_flag.result.value ignores content capture." + ) From a9aa7a62594cccf8fe59cbe26cd9d94fba1cfeb4 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 18:32:34 -0400 Subject: [PATCH 07/16] test(openai-agents): pin capture off in the root event exact-attributes test Content capture now defaults to on, so the test's raw variation became the result value. It asserts the capture-off key fallback, so say so. --- packages/openai-agents/tests/test_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/openai-agents/tests/test_handler.py b/packages/openai-agents/tests/test_handler.py index 05f31a9e..05d6a896 100644 --- a/packages/openai-agents/tests/test_handler.py +++ b/packages/openai-agents/tests/test_handler.py @@ -860,7 +860,9 @@ async def test_feature_flag_event_on_root_only(self) -> None: "__ldValue": {"model": {"name": "m"}, "_ldMeta": {"variationKey": "v1"}}, } with ctx, _patched_agents(agents_mod): - await create_openai_agents_handler()(CONFIG, "q", {}, vs) + await create_openai_agents_handler(capture_content=False)( + CONFIG, "q", {}, vs + ) assert [event for event in rec.root.events if event[0] == "feature_flag"] == [ ( "feature_flag", From 296fc3cb2ec76238e7b2f74ebd3bc6819d0324b9 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 01:21:21 -0400 Subject: [PATCH 08/16] fix(client): keep the raw variation under a private meta key meta is public through GraphNode.meta and JudgeTask.judge_meta, so a public evaluatedValue key would expose the full prompt in user logging regardless of capture_content, and renaming it after 1.0 would break callers. Store it as _evaluatedValue, read only by the tracking helpers. --- .../src/launchdarkly_ai_server/lifecycle.py | 4 ++- .../src/launchdarkly_ai_server/tracking.py | 8 ++--- uv.lock | 30 +++++++++---------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index 675372d6..ba96a16a 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -631,7 +631,9 @@ async def extract_variation( # The variation exactly as the LD SDK returned it, ``_ldMeta`` included. # Written to the root span's ``feature_flag`` event as # ``feature_flag.result.value``; never copied into tracking payloads. - "evaluatedValue": raw, + # Private: ``meta`` surfaces publicly as ``GraphNode.meta`` and + # ``JudgeTask.judge_meta``. + "_evaluatedValue": raw, } # Strip _ldMeta for config parsing diff --git a/packages/client/src/launchdarkly_ai_server/tracking.py b/packages/client/src/launchdarkly_ai_server/tracking.py index 104c6c57..ea746bba 100644 --- a/packages/client/src/launchdarkly_ai_server/tracking.py +++ b/packages/client/src/launchdarkly_ai_server/tracking.py @@ -212,8 +212,8 @@ async def execute_and_track( "ldContext": {**user_context}, "__ld": track_data, } - if isinstance(meta, dict) and "evaluatedValue" in meta: - merged_variables["__ldValue"] = meta["evaluatedValue"] + if isinstance(meta, dict) and "_evaluatedValue" in meta: + merged_variables["__ldValue"] = meta["_evaluatedValue"] start_time = time.monotonic() @@ -314,8 +314,8 @@ async def execute_and_stream( "ldContext": {**user_context}, "__ld": track_data, } - if isinstance(meta, dict) and "evaluatedValue" in meta: - merged_variables["__ldValue"] = meta["evaluatedValue"] + if isinstance(meta, dict) and "_evaluatedValue" in meta: + merged_variables["__ldValue"] = meta["_evaluatedValue"] start_time = time.monotonic() full_text = "" diff --git a/uv.lock b/uv.lock index ec6a31c7..5c20048d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15'", @@ -1322,7 +1322,7 @@ wheels = [ [[package]] name = "launchdarkly-ai-claude-agents" -version = "0.2.6" +version = "0.3.0" source = { editable = "packages/claude-agents" } dependencies = [ { name = "anthropic" }, @@ -1341,7 +1341,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-claude-messages" -version = "0.2.5" +version = "0.3.0" source = { editable = "packages/claude-messages" } dependencies = [ { name = "anthropic" }, @@ -1358,7 +1358,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-langchain-agents" -version = "0.2.6" +version = "0.3.0" source = { editable = "packages/langchain-agents" } dependencies = [ { name = "langchain-core" }, @@ -1377,7 +1377,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-langchain-messages" -version = "0.2.5" +version = "0.3.0" source = { editable = "packages/langchain-messages" } dependencies = [ { name = "langchain-core" }, @@ -1394,7 +1394,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-litellm-agents" -version = "0.1.1" +version = "0.2.0" source = { editable = "packages/litellm-agents" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -1411,7 +1411,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-litellm-messages" -version = "0.1.1" +version = "0.2.0" source = { editable = "packages/litellm-messages" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -1428,7 +1428,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-openai-agents" -version = "0.2.6" +version = "0.3.0" source = { editable = "packages/openai-agents" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -1447,7 +1447,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-openai-messages" -version = "0.2.5" +version = "0.3.0" source = { editable = "packages/openai-messages" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -1464,7 +1464,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-opencode-agents" -version = "0.2.0" +version = "0.3.0" source = { editable = "packages/opencode-agents" } dependencies = [ { name = "httpx" }, @@ -1481,7 +1481,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-python" -version = "0.3.0" +version = "0.4.0" source = { editable = "packages/ai" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -1501,7 +1501,7 @@ provides-extras = ["otel"] [[package]] name = "launchdarkly-ai-server" -version = "0.4.0" +version = "0.5.0" source = { editable = "packages/client" } dependencies = [ { name = "opentelemetry-api" }, @@ -1523,7 +1523,7 @@ provides-extras = ["otel"] [[package]] name = "launchdarkly-ai-typesafe" -version = "0.1.1" +version = "0.2.0" source = { editable = "packages/typesafe" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -1540,7 +1540,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-vercel-agents" -version = "0.1.1" +version = "0.2.0" source = { editable = "packages/vercel-agents" } dependencies = [ { name = "ai" }, @@ -1557,7 +1557,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-vercel-messages" -version = "0.1.1" +version = "0.2.0" source = { editable = "packages/vercel-messages" } dependencies = [ { name = "ai" }, From a61c66411db2ebd2b2e6c5a1a46ca10901d9657e Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 11:05:20 -0400 Subject: [PATCH 09/16] fix(client): keep the raw variation off meta entirely A private _evaluatedValue key still rode on public objects: GraphNode.meta and JudgeTask.judge_meta carried the full prompt into any log or pickle of them, whatever capture_content was set to. extract_variation now returns the raw variation as a separate value, as JS's extractVariation does. execute_and_track and execute_and_stream take it as a keyword-only evaluated_value, config() and inline judges pass it, and the graph runner keeps node values in a private map beside its nodes. A JudgeTask run falls back to the key, as in JS. --- .../api-report/launchdarkly_ai_server.txt | 4 ++-- .../src/launchdarkly_ai_server/client.py | 2 ++ .../src/launchdarkly_ai_server/graph.py | 7 ++++++ .../src/launchdarkly_ai_server/judges.py | 1 + .../src/launchdarkly_ai_server/lifecycle.py | 13 ++++------- .../src/launchdarkly_ai_server/tracking.py | 14 +++++++---- packages/client/tests/test_lifecycle.py | 23 +++++++++++++++++++ 7 files changed, 49 insertions(+), 15 deletions(-) diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt index 82f1438d..0234f147 100644 --- a/packages/client/api-report/launchdarkly_ai_server.txt +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -61,8 +61,8 @@ EvaluationsModule [class] sdk_key: str | None [property] tools: ToolsClient [property] ui_base_uri: str [property] -execute_and_stream(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None) -> AsyncGenerator[ExecuteStreamEvent, None] [async] -execute_and_track(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None) -> dict[str, Any] [async] +execute_and_stream(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None, evaluated_value: Any = None) -> AsyncGenerator[ExecuteStreamEvent, None] [async] +execute_and_track(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None, evaluated_value: Any = None) -> dict[str, Any] [async] ExecuteStreamDoneEvent = dict[str, Any] ExecuteStreamEvent = dict[str, Any] extract_variation(config_key: str, context: dict[str, Any]) -> dict[str, Any] [async] diff --git a/packages/client/src/launchdarkly_ai_server/client.py b/packages/client/src/launchdarkly_ai_server/client.py index be124d87..ce9e2f09 100644 --- a/packages/client/src/launchdarkly_ai_server/client.py +++ b/packages/client/src/launchdarkly_ai_server/client.py @@ -109,6 +109,7 @@ async def _invoke( tool_handlers=resolved_tools, variables=variables, history=history, + evaluated_value=variation.get("value"), ) raw_response = result["response"] @@ -240,6 +241,7 @@ async def _execute_stream( tool_handlers=resolved_tools, variables=variables, history=history, + evaluated_value=variation.get("value"), ) ) as source: async for event in source: diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index 4f8677b6..e35b8458 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -171,11 +171,14 @@ def edges_from(node_key: str) -> list[GraphEdge]: return [e for e in edges if e.source_key == node_key] nodes: dict[str, GraphNode] = {} + # Raw variations, kept off the public GraphNode. + node_values: dict[str, Any] = {} try: for node_key in all_keys: variation = await extract_variation(node_key, context) node_config: AiConfigRep = variation["config"] node_meta: VariationMeta = variation["meta"] + node_values[node_key] = variation.get("value") node_edges = edges_from(node_key) nodes[node_key] = GraphNode( key=node_key, @@ -238,6 +241,7 @@ async def run_node( try: result = await execute_and_track( config_key=node.key, + evaluated_value=node_values.get(node.key), config=node.config, meta=node.meta, user_context=context, @@ -415,6 +419,7 @@ async def route( try: result = await execute_and_track( config_key=node.key, + evaluated_value=node_values.get(node.key), config=routed_config, meta=node.meta, user_context=context, @@ -530,6 +535,7 @@ async def stream_node( projector = StableProjector() async for event in execute_and_stream( config_key=node.key, + evaluated_value=node_values.get(node.key), config=node.config, meta=node.meta, user_context=context, @@ -670,6 +676,7 @@ async def stream_route( projector = StableProjector() async for event in execute_and_stream( config_key=node.key, + evaluated_value=node_values.get(node.key), config=routed_config, meta=node.meta, user_context=context, diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index d46b9c0a..60860a64 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -202,6 +202,7 @@ async def run_judges( "response_to_evaluate": llm_response, **({"input": user_input} if user_input else {}), }, + evaluated_value=variation.get("value"), ) usage = to_usage_dict(result["usage"]) diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index ba96a16a..79d4c3c1 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -588,7 +588,7 @@ async def extract_variation( ) -> dict[str, Any]: """ Fetches an AI config variation from the LD client and parses it. - Returns ``{"config": AiConfigRep, "meta": VariationMeta}``. + Returns ``{"config": AiConfigRep, "meta": VariationMeta, "value": Any}``. Raises on disabled or invalid variations. Lazily initializes the LD client when ``LD_SDK_KEY`` is set, matching @@ -628,12 +628,6 @@ async def extract_variation( # Pinned model-config identity; keys are omitted when absent so that # tracking payloads never carry ``None`` values. **model_stamps_from_meta(ld_meta), - # The variation exactly as the LD SDK returned it, ``_ldMeta`` included. - # Written to the root span's ``feature_flag`` event as - # ``feature_flag.result.value``; never copied into tracking payloads. - # Private: ``meta`` surfaces publicly as ``GraphNode.meta`` and - # ``JudgeTask.judge_meta``. - "_evaluatedValue": raw, } # Strip _ldMeta for config parsing @@ -649,4 +643,7 @@ async def extract_variation( f"Invalid AI config variation for '{config_key}': {result.error['message']}" ) - return {"config": result.data, "meta": meta} + # ``value`` is the variation exactly as the LD SDK returned it, ``_ldMeta`` + # included. It stays off ``meta``, which surfaces publicly as + # ``GraphNode.meta`` and ``JudgeTask.judge_meta``. + return {"config": result.data, "meta": meta, "value": raw} diff --git a/packages/client/src/launchdarkly_ai_server/tracking.py b/packages/client/src/launchdarkly_ai_server/tracking.py index ea746bba..268a5bb7 100644 --- a/packages/client/src/launchdarkly_ai_server/tracking.py +++ b/packages/client/src/launchdarkly_ai_server/tracking.py @@ -167,10 +167,13 @@ async def execute_and_track( variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None, + evaluated_value: Any = None, ) -> dict[str, Any]: """ Calls *handler*, emits LD telemetry events, and returns - ``{usage, response, track_data}``. + ``{usage, response, track_data}``. *evaluated_value* is the variation as + the LD SDK returned it; with content capture on it becomes the root + span's ``feature_flag.result.value``. """ from .lifecycle import get_client @@ -212,8 +215,8 @@ async def execute_and_track( "ldContext": {**user_context}, "__ld": track_data, } - if isinstance(meta, dict) and "_evaluatedValue" in meta: - merged_variables["__ldValue"] = meta["_evaluatedValue"] + if evaluated_value is not None: + merged_variables["__ldValue"] = evaluated_value start_time = time.monotonic() @@ -268,6 +271,7 @@ async def execute_and_stream( variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None, + evaluated_value: Any = None, ) -> AsyncGenerator[ExecuteStreamEvent, None]: """ Streaming counterpart to ``execute_and_track``. Yields ``chunk`` events, @@ -314,8 +318,8 @@ async def execute_and_stream( "ldContext": {**user_context}, "__ld": track_data, } - if isinstance(meta, dict) and "_evaluatedValue" in meta: - merged_variables["__ldValue"] = meta["_evaluatedValue"] + if evaluated_value is not None: + merged_variables["__ldValue"] = evaluated_value start_time = time.monotonic() full_text = "" diff --git a/packages/client/tests/test_lifecycle.py b/packages/client/tests/test_lifecycle.py index fe67a597..ac70c7e2 100644 --- a/packages/client/tests/test_lifecycle.py +++ b/packages/client/tests/test_lifecycle.py @@ -6,6 +6,7 @@ import ast import asyncio import inspect +import json import os import subprocess import sys @@ -1031,6 +1032,28 @@ async def test_preserves_model_key_and_version_on_meta(self) -> None: assert extracted["meta"]["modelKey"] == "my-model" assert extracted["meta"]["modelVersion"] == 3 + async def test_returns_the_raw_variation_beside_meta_not_on_it(self) -> None: + raw = { + "_ldMeta": {"enabled": True, "variationKey": "v1", "version": 1}, + "model": {"name": "claude-3-5"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", + } + stub = _make_stub_client() + stub.variation = AsyncMock(return_value=raw) + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=stub) + ctx = {"kind": "user", "key": "user-1"} + with patch( + "launchdarkly_ai_server.utils.to_ld_context", + side_effect=lambda _c, ctx: ctx, + ): + extracted = await lifecycle_module.extract_variation("my-flag", ctx) + + assert extracted["value"] == raw + assert raw not in extracted["meta"].values() + assert "instructions" not in json.dumps(extracted["meta"]) + async def test_returns_enabled_false_and_null_config_when_disabled(self) -> None: stub = _make_stub_client() stub.variation = AsyncMock(return_value={"_ldMeta": {"enabled": False}}) From 36e2e932e2ec5f224d4585e5b2ab37c1e69fc83f Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 11:45:36 -0400 Subject: [PATCH 10/16] feat(client): carry the judge variation on JudgeTask for feature_flag.result.value A deferred judge (skip_judges=True, then run_judge) wrote the variation key even with content capture on, while an inline judge wrote the raw variation. JudgeTask now carries it as judge_value, and run_judge passes it on, so both paths write the same value. The task already holds the judge config and the response, so this adds no new kind of content. --- .../api-report/launchdarkly_ai_server.txt | 3 +- .../src/launchdarkly_ai_server/judges.py | 2 + .../src/launchdarkly_ai_server/types.py | 6 +++ packages/client/tests/test_judges.py | 50 +++++++++++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt index 0234f147..0d15a8d3 100644 --- a/packages/client/api-report/launchdarkly_ai_server.txt +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -173,7 +173,7 @@ JudgeRunResult [class, @dataclass] track_data: TrackData usage: UsageDict JudgeTask [class, @dataclass] - __init__(config_key: str, judge_config: AiConfigRep, judge_meta: VariationMeta, actual_output: str, user_context: LDContext, judge_provider: str | None, judge_mode: str, collapse_messages: bool, parent_track_data: TrackData, variables: dict[str, Any] | None = None, evaluation_metric_key: str | None = None, user_input: str | None = None, trajectory: str = '') + __init__(config_key: str, judge_config: AiConfigRep, judge_meta: VariationMeta, actual_output: str, user_context: LDContext, judge_provider: str | None, judge_mode: str, collapse_messages: bool, parent_track_data: TrackData, variables: dict[str, Any] | None = None, evaluation_metric_key: str | None = None, user_input: str | None = None, trajectory: str = '', judge_value: Any = None) actual_output: str collapse_messages: bool config_key: str @@ -182,6 +182,7 @@ JudgeTask [class, @dataclass] judge_meta: VariationMeta judge_mode: str judge_provider: str | None + judge_value: Any = None parent_track_data: TrackData trajectory: str = '' user_context: LDContext diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 60860a64..01bf3cf1 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -387,6 +387,7 @@ async def _build_judge_tasks( collapse_messages=collapse_messages, parent_track_data=base_track_data, evaluation_metric_key=evaluation_metric_key, + judge_value=variation.get("value"), ) ) except Exception as exc: @@ -460,6 +461,7 @@ async def run_judge( "response_to_evaluate": task.actual_output, **({"input": task.user_input} if task.user_input else {}), }, + evaluated_value=task.judge_value, ) usage = to_usage_dict(result["usage"]) diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 37c8db16..ea0de20a 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -347,6 +347,12 @@ class JudgeTask: must stay picklable. """ """LD metric key to track the score against.""" + judge_value: Any = None + """The judge variation exactly as the LD SDK returned it. + + Written as the root span's ``feature_flag.result.value`` when the judge + handler captures content; otherwise the variation key is written. + """ @dataclass diff --git a/packages/client/tests/test_judges.py b/packages/client/tests/test_judges.py index 6e53b4ee..7a6caba3 100644 --- a/packages/client/tests/test_judges.py +++ b/packages/client/tests/test_judges.py @@ -1247,6 +1247,56 @@ async def test_judge_task_carries_no_output_format( roundtripped = json.loads(json.dumps(asdict(task))) assert "outputFormat" not in roundtripped["judge_config"] + async def test_judge_task_carries_the_raw_judge_variation( + self, mock_ld_client: MagicMock + ) -> None: + """A deferred judge writes the same feature_flag.result.value as an inline one.""" + judge_variation = _judge_variation_with_output_format() + mock_ld_client.variation = AsyncMock(return_value=judge_variation) + config = { + "model": {"name": "gpt-4"}, + "provider": {"name": "TestProvider"}, + "instructions": "hi", + "judgeConfiguration": {"judges": [{"key": "judge-1", "samplingRate": 1.0}]}, + } + + import random + + with patch.object(random, "random", return_value=0.0): + tasks = await build_judge_tasks( + config=config, + user_context=CONTEXT, + handler=_make_handler(), + llm_response="response", + base_track_data={"runId": "x"}, + ) + + assert tasks[0].judge_value == judge_variation + assert ( + json.loads(json.dumps(asdict(tasks[0])))["judge_value"] == judge_variation + ) + + seen: list[dict[str, Any]] = [] + + async def recording_fn( + config, user_input, tool_handlers, variables, history=None + ) -> dict: # type: ignore[override] + seen.append(dict(variables or {})) + return { + "output": '{"score": 0.6, "reasoning": "ok"}', + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + from launchdarkly_ai_server.types import ProviderHandler + + judge_handler = ProviderHandler( + fn=recording_fn, # type: ignore[arg-type] + provides_for=("TestProvider", tasks[0].judge_mode), + ) + await run_judge(tasks[0], [judge_handler]) + + assert seen[0]["__ldValue"] == judge_variation + async def test_run_judge_strips_defensively( self, mock_ld_client: MagicMock ) -> None: From 404fcc176cf23bd205624774c2c8e1da98a7df3b Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 11:45:36 -0400 Subject: [PATCH 11/16] feat(client): write the graph variation as feature_flag.result.value on the graph root The graph root span always wrote the key. It now writes the graph variation as returned by the LD SDK when content capture is on, and the key otherwise. The root has no single handler, so it follows the process default. The helper also skips a null value instead of writing "null". --- .../src/launchdarkly_ai_server/graph.py | 37 +++++++++++++------ .../src/launchdarkly_ai_server/utils.py | 2 +- packages/client/tests/test_graph_stream.py | 26 +++++++------ 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index e35b8458..963fb3a8 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -107,16 +107,16 @@ async def _fetch_graph_variation( topology = variation if isinstance(variation, dict) else None if not topology or not topology.get("root"): - return {"enabled": False, "meta": meta} + return {"enabled": False, "meta": meta, "value": variation} - return {"enabled": True, "topology": topology, "meta": meta} + return {"enabled": True, "topology": topology, "meta": meta, "value": variation} async def _build_graph( key: str, context: LDContext, options: dict[str, Any], -) -> tuple[GraphDefinition, TrackData, Callable[..., Any]]: +) -> tuple[GraphDefinition, TrackData, Callable[..., Any], Any]: from .judges import run_judges from .lifecycle import extract_variation, get_client from .tracking import _try_get_environment_id, execute_and_track @@ -127,6 +127,9 @@ async def _build_graph( enabled: bool = result["enabled"] topology: dict[str, Any] | None = result.get("topology") meta = result.get("meta", {}) + # The graph variation as the LD SDK returned it, for the root span's + # feature_flag.result.value. Kept off GraphDefinition, which is public. + graph_value = result.get("value") graph_track_data: TrackData = { "runId": str(uuid.uuid4()), @@ -147,6 +150,7 @@ async def _build_graph( _disabled_definition(key), graph_track_data, _disabled_stream_route(key), + graph_value, ) raw_edges_map: dict[str, list[dict[str, Any]]] = topology.get("edges") or {} @@ -193,6 +197,7 @@ def edges_from(node_key: str) -> list[GraphEdge]: _disabled_definition(key), graph_track_data, _disabled_stream_route(key), + graph_value, ) root_node = nodes.get(topology["root"]) @@ -836,7 +841,7 @@ async def visit(node_key: str) -> None: reverse_traverse=reverse_traverse, ) - return graph_def, graph_track_data, stream_route + return graph_def, graph_track_data, stream_route, graph_value async def resolve_graph( @@ -864,7 +869,7 @@ async def resolve_graph( "tool_handlers": resolved_tools, "registry": registry, } - graph_def, _, _ = await _build_graph(key, context, options) + graph_def, _, _, _ = await _build_graph(key, context, options) return graph_def @@ -902,7 +907,7 @@ async def _invoke( from opentelemetry import trace from .judges import run_judges - from .lifecycle import get_client + from .lifecycle import effective_capture_content, get_client ld_ctx = to_ld_context(get_client(), context) @@ -925,7 +930,7 @@ async def _invoke( ) built = await _build_graph(self._key, context, resolved_options) - graph_def, built_track_data, _ = built + graph_def, built_track_data, _, graph_value = built if not graph_def.enabled: raise ValueError(f'Agent graph "{self._key}" is disabled') @@ -935,7 +940,13 @@ async def _invoke( tracer = trace.get_tracer("@launchdarkly/ai-server") with tracer.start_as_current_span("launchdarkly.graph") as span: set_ld_span_attributes( - span, {"__ld": graph_track_data, "ldContext": context} + span, + { + "__ld": graph_track_data, + "ldContext": context, + "__ldValue": graph_value, + }, + capture_content=effective_capture_content(None), ) start_time = time.monotonic() @@ -1127,7 +1138,7 @@ async def _stream_events( from opentelemetry.trace import Status, StatusCode, set_span_in_context from .judges import run_judges - from .lifecycle import get_client + from .lifecycle import effective_capture_content, get_client resolved_input = user_input or "" ld_ctx = to_ld_context(get_client(), context) @@ -1151,7 +1162,7 @@ async def _stream_events( ) built = await _build_graph(self._key, context, resolved_options) - graph_def, built_track_data, stream_route = built + graph_def, built_track_data, stream_route, graph_value = built if not graph_def.enabled: raise ValueError(f'Agent graph "{self._key}" is disabled') @@ -1160,7 +1171,11 @@ async def _stream_events( tracer = trace.get_tracer("@launchdarkly/ai-server") span = tracer.start_span("launchdarkly.graph", context=caller_context) - set_ld_span_attributes(span, {"__ld": graph_track_data, "ldContext": context}) + set_ld_span_attributes( + span, + {"__ld": graph_track_data, "ldContext": context, "__ldValue": graph_value}, + capture_content=effective_capture_content(None), + ) span_context = set_span_in_context(span, caller_context) ended: set[int] = set() start_time = time.monotonic() diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index dd877c41..e6f43744 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -825,7 +825,7 @@ def set_ld_span_attributes( variation_key = ld.get("variationKey") if isinstance(variation_key, str) and variation_key: feature_flag_attrs["feature_flag.result.variant"] = variation_key - if capture_content and "__ldValue" in variables: + if capture_content and variables.get("__ldValue") is not None: feature_flag_attrs["feature_flag.result.value"] = json.dumps( variables["__ldValue"] ) diff --git a/packages/client/tests/test_graph_stream.py b/packages/client/tests/test_graph_stream.py index 0119c36b..0b43178f 100644 --- a/packages/client/tests/test_graph_stream.py +++ b/packages/client/tests/test_graph_stream.py @@ -19,7 +19,7 @@ from opentelemetry.trace import StatusCode import launchdarkly_ai_server.lifecycle as lifecycle_module -from launchdarkly_ai_server import ProviderHandler, graph +from launchdarkly_ai_server import ProviderHandler, SDKOptions, configure, graph from launchdarkly_ai_server.conversation import ( GEN_AI_CONVERSATION_ID, ConversationIdSpanProcessor, @@ -990,18 +990,19 @@ async def test_graph_span_carries_config_identity( assert (flag_events[0].attributes or {})["feature_flag.key"] == "graph-key" @pytest.mark.parametrize("mode", ["invoke", "stream"]) + @pytest.mark.parametrize("capture", [True, False]) async def test_graph_span_event_carries_the_graph_variation( - self, mode: str + self, mode: str, capture: bool ) -> None: - """A graph root has no raw variation, so its result value is the JSON-encoded key.""" - client = _make_client( - { - "root": "root-node", - "edges": {"root-node": [{"key": "leaf-node"}]}, - "_ldMeta": {"enabled": True, "variationKey": "graph-v1", "version": 3}, - } - ) + """With capture on the result value is the raw graph variation; off, the key.""" + raw = { + "root": "root-node", + "edges": {"root-node": [{"key": "leaf-node"}]}, + "_ldMeta": {"enabled": True, "variationKey": "graph-v1", "version": 3}, + } + client = _make_client(raw) lifecycle_module._set_client_for_testing(client) + configure(options=SDKOptions(capture_content=capture)) try: g = graph("graph-key", handlers=[_make_streaming_handler(["ok"])]) if mode == "invoke": @@ -1015,11 +1016,14 @@ async def test_graph_span_event_carries_the_graph_variation( event = next(e for e in graph_spans[0].events if e.name == "feature_flag") attrs = event.attributes or {} assert attrs["feature_flag.result.variant"] == "graph-v1" - assert attrs["feature_flag.result.value"] == json.dumps("graph-v1") + expected = json.dumps(raw) if capture else json.dumps("graph-v1") + assert attrs["feature_flag.result.value"] == expected async def test_graph_span_without_a_variation_key_omits_the_result( self, mock_ld_client: MagicMock ) -> None: + """With capture off and no variation key there is nothing to write.""" + configure(options=SDKOptions(capture_content=False)) await graph("graph-key", handlers=[_make_streaming_handler(["ok"])]).invoke( "hi", CONTEXT ) From 558cc6f7d2ae7f2581e7455cdcaa81bdbfaa0953 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 15:27:29 -0400 Subject: [PATCH 12/16] fix(client): never pass a caller-supplied __ldValue to a handler execute_and_track and execute_and_stream merged the caller's variables and only set __ldValue when there was an evaluated value, so a caller-supplied __ldValue could reach the handler and be written as feature_flag.result.value. Both now drop it first, the same as the JS SDK does. --- .../src/launchdarkly_ai_server/tracking.py | 6 +++ packages/client/tests/test_tracking.py | 44 ++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/tracking.py b/packages/client/src/launchdarkly_ai_server/tracking.py index 9b6926ea..2649b519 100644 --- a/packages/client/src/launchdarkly_ai_server/tracking.py +++ b/packages/client/src/launchdarkly_ai_server/tracking.py @@ -215,6 +215,9 @@ async def execute_and_track( "ldContext": {**user_context}, "__ld": track_data, } + # ``__ldValue`` is SDK-owned like ``__ld``: a caller-supplied one never + # reaches the handler. + merged_variables.pop("__ldValue", None) if evaluated_value is not None: merged_variables["__ldValue"] = evaluated_value @@ -318,6 +321,9 @@ async def execute_and_stream( "ldContext": {**user_context}, "__ld": track_data, } + # ``__ldValue`` is SDK-owned like ``__ld``: a caller-supplied one never + # reaches the handler. + merged_variables.pop("__ldValue", None) if evaluated_value is not None: merged_variables["__ldValue"] = evaluated_value diff --git a/packages/client/tests/test_tracking.py b/packages/client/tests/test_tracking.py index eb9db7dc..6053b3bc 100644 --- a/packages/client/tests/test_tracking.py +++ b/packages/client/tests/test_tracking.py @@ -1,10 +1,15 @@ """ -Tests for wrap_tool_handlers. +Tests for wrap_tool_handlers and the handler variables execute_and_track builds. """ +from typing import Any from unittest.mock import MagicMock, patch +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool, wrap_tool_handlers +from launchdarkly_ai_server.tracking import execute_and_track def _make_mock_client() -> MagicMock: @@ -85,3 +90,40 @@ def test_handoff_prefix_skips_tracking(self) -> None: def test_undefined_tool_handlers(self) -> None: result = wrap_tool_handlers(None, CONTEXT, TRACK_DATA) assert result == {} + + +class TestHandlerVariables: + @pytest.mark.parametrize( + ("evaluated", "expected"), + [(None, None), ({"raw": True}, {"raw": True})], + ) + async def test_a_caller_supplied_ld_value_never_reaches_the_handler( + self, evaluated: Any, expected: Any + ) -> None: + seen: list[dict[str, Any]] = [] + + async def handler( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + seen.append(dict(variables or {})) + return {"output": "ok", "usage": {}} + + lifecycle_module._set_client_for_testing(_make_mock_client()) + try: + await execute_and_track( + config_key="c", + config={"model": {"name": "m"}, "provider": {"name": "P"}}, + meta={"variationKey": "v", "version": 1}, + user_context={"kind": "user", "key": "u1"}, + handler=handler, # type: ignore[arg-type] + variables={"__ldValue": "from-caller"}, + evaluated_value=evaluated, + ) + finally: + lifecycle_module._reset_for_testing() + + assert seen[0].get("__ldValue") == expected From 248ad6d7bd20369fc6389737b0f3ee2406b868d2 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 15:31:36 -0400 Subject: [PATCH 13/16] feat(client): let set_ld_span_attributes follow the configured capture default capture_content defaulted to False, so a custom handler that called the helper without the flag sent only the variation key, while the JS SDK writes whatever the core passes, which follows configure(). The argument is now bool | None = None, and None reads the process default at call time. Built-in handlers already pass their resolved setting, so they are unchanged. --- .../api-report/launchdarkly_ai_server.txt | 2 +- .../src/launchdarkly_ai_server/utils.py | 9 +++++++- .../client/tests/test_ld_span_attributes.py | 21 ++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt index 0006d09c..607611f2 100644 --- a/packages/client/api-report/launchdarkly_ai_server.txt +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -318,7 +318,7 @@ SDKOptions [class, @dataclass(frozen=True)] capture_content: bool = True set_conversation_id_if_absent(span: Any, conversation: str) -> None set_input_content_attributes(span: Any, capture: bool, *, system_instructions: str | None = None, messages: list[SpanMessage] | None = None, tool_definitions: list[ToolDefinitionInput] | None = None) -> None -set_ld_span_attributes(span: Any, variables: dict[str, Any] | None, *, capture_content: bool = False) -> None +set_ld_span_attributes(span: Any, variables: dict[str, Any] | None, *, capture_content: bool | None = None) -> None set_model_identity_attributes(span: Any, provider_name: str, request_model: str, legacy_system: str | None = None) -> None set_output_content_attributes(span: Any, capture: bool, messages: list[SpanMessage]) -> None set_skill_store(store: SkillStore | None) -> None diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index e6f43744..d73c6a84 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -769,7 +769,7 @@ def _context_identity_from_ld_context( def set_ld_span_attributes( - span: Any, variables: dict[str, Any] | None, *, capture_content: bool = False + span: Any, variables: dict[str, Any] | None, *, capture_content: bool | None = None ) -> None: """ Sets LaunchDarkly config-identifying attributes on an OTel span and emits @@ -805,6 +805,9 @@ def set_ld_span_attributes( writes it with value inclusion on. Otherwise, when the variation key is non-empty, it is ``json.dumps`` of the key, so a prompt never leaves the process without consent. Otherwise it is omitted (native-graph spans). + *capture_content* left as ``None`` follows the process default from + ``configure()``, so a custom handler gets the same value a built-in one + does. """ span.set_attribute("launchdarkly.operation.type", "gen_ai") if not variables: @@ -825,6 +828,10 @@ def set_ld_span_attributes( variation_key = ld.get("variationKey") if isinstance(variation_key, str) and variation_key: feature_flag_attrs["feature_flag.result.variant"] = variation_key + if capture_content is None: + from .lifecycle import effective_capture_content + + capture_content = effective_capture_content(None) if capture_content and variables.get("__ldValue") is not None: feature_flag_attrs["feature_flag.result.value"] = json.dumps( variables["__ldValue"] diff --git a/packages/client/tests/test_ld_span_attributes.py b/packages/client/tests/test_ld_span_attributes.py index 2854f95a..6b2c2819 100644 --- a/packages/client/tests/test_ld_span_attributes.py +++ b/packages/client/tests/test_ld_span_attributes.py @@ -7,6 +7,8 @@ import pytest +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_server import SDKOptions, configure from launchdarkly_ai_server.utils import set_ld_span_attributes LD_FIXTURE = { @@ -236,9 +238,26 @@ def test_capture_on_writes_the_raw_variation_like_the_tracing_hook( def test_capture_off_writes_the_variation_key_not_the_prompt(self) -> None: span = FakeSpan() - set_ld_span_attributes(span, {**_vars(None), "__ldValue": RAW_VARIATION}) + set_ld_span_attributes( + span, {**_vars(None), "__ldValue": RAW_VARIATION}, capture_content=False + ) assert _feature_flag(span)["feature_flag.result.value"] == '"variation-a"' + @pytest.mark.parametrize(("default", "raw"), [(True, True), (False, False)]) + def test_an_unset_capture_follows_the_process_default( + self, default: bool, raw: bool + ) -> None: + """A custom handler that passes no capture_content gets what configure() says.""" + lifecycle_module._reset_for_testing() + configure(options=SDKOptions(capture_content=default)) + try: + span = FakeSpan() + set_ld_span_attributes(span, {**_vars(None), "__ldValue": RAW_VARIATION}) + finally: + lifecycle_module._reset_for_testing() + expected = json.dumps(RAW_VARIATION) if raw else '"variation-a"' + assert _feature_flag(span)["feature_flag.result.value"] == expected + def test_capture_on_without_a_raw_variation_falls_back_to_the_key(self) -> None: span = FakeSpan() set_ld_span_attributes(span, _vars(None), capture_content=True) From 4173b65001479b24a556720a29db78f87cbec058 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 15:39:18 -0400 Subject: [PATCH 14/16] feat(client): keep a handler's stream and explicit capture override when it is wrapped again create_handler(q, create_handler(p, fn, None, False)).capture_content was True, because the new ProviderHandler started unset. Wrapping a ProviderHandler again now keeps its streaming implementation and its explicit capture_content override unless new ones are passed, as the JS SDK does. An unset inner handler stays unset and still follows configure(). --- .../src/launchdarkly_ai_server/utils.py | 13 ++++++++- packages/client/tests/test_configure.py | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index d73c6a84..17ed7962 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -43,7 +43,18 @@ def _create_handler( stream_fn: _StreamFn | None = None, capture_content: bool | None = None, ) -> ProviderHandler: - """Non-reporting :func:`create_handler`, used by the package factories.""" + """Non-reporting :func:`create_handler`, used by the package factories. + + Wrapping a :class:`ProviderHandler` again keeps its streaming + implementation and its explicit ``capture_content`` override unless new + ones are passed, as of the moment it is wrapped. An unset inner handler + stays unset. + """ + if isinstance(fn, ProviderHandler): + if stream_fn is None: + stream_fn = fn._stream_fn + if capture_content is None: + capture_content = fn._capture_content_override return ProviderHandler( fn=fn, provides_for=provides_for, diff --git a/packages/client/tests/test_configure.py b/packages/client/tests/test_configure.py index 4f0ab359..7fc17c8c 100644 --- a/packages/client/tests/test_configure.py +++ b/packages/client/tests/test_configure.py @@ -246,3 +246,30 @@ async def test_init_client_does_not_carry_behavior_defaults( await init_client({"capture_content": False}, stub) assert _handler().capture_content is True assert any("capture_content" in record.getMessage() for record in caplog.records) + + +def test_rewrapping_keeps_an_explicit_override() -> None: + from launchdarkly_ai_server import create_handler + + inner = create_handler(("A", "messages"), _fn, None, False) + outer = create_handler(("B", "messages"), inner) + assert outer.capture_content is False + assert create_handler(("B", "messages"), inner, None, True).capture_content is True + + +def test_rewrapping_an_unset_handler_still_follows_configure() -> None: + from launchdarkly_ai_server import create_handler + + outer = create_handler(("B", "messages"), create_handler(("A", "messages"), _fn)) + configure(options=SDKOptions(capture_content=False)) + assert outer.capture_content is False + + +def test_rewrapping_keeps_the_stream() -> None: + from launchdarkly_ai_server import create_handler + + async def stream(*_args: Any, **_kwargs: Any) -> Any: + yield {"type": "done", "output": "", "usage": {}} + + inner = create_handler(("A", "messages"), _fn, stream) + assert create_handler(("B", "messages"), inner)._stream_fn is stream From f1681767e85aa1e7882452ed8f732266b98e747e Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 16:21:39 -0400 Subject: [PATCH 15/16] fix(client): pass the raw variation only to handlers that capture content execute_and_track and execute_and_stream handed __ldValue to every handler, and since set_ld_span_attributes now follows the process default when no flag is passed, a custom handler built with capture_content=False whose function called the helper without a flag exported the whole variation. The core now sets __ldValue only when the handler's effective setting is on (a plain function follows the process default), as the JS SDK does. The helper's default stays as a second gate. --- .../src/launchdarkly_ai_server/tracking.py | 19 +++++++- packages/client/tests/test_tracking.py | 44 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/tracking.py b/packages/client/src/launchdarkly_ai_server/tracking.py index 2649b519..274c73a6 100644 --- a/packages/client/src/launchdarkly_ai_server/tracking.py +++ b/packages/client/src/launchdarkly_ai_server/tracking.py @@ -155,6 +155,21 @@ def _exposed_tool_keys( ] +def _handler_captures_content(handler: Any) -> bool: + """Effective content capture for a call made through *handler*. + + A :class:`ProviderHandler` answers with its own override, else the process + default. A plain function has no setting, so it follows the process + default. The raw variation only reaches a handler that captures content. + """ + capture = getattr(handler, "capture_content", None) + if isinstance(capture, bool): + return capture + from .lifecycle import effective_capture_content + + return effective_capture_content(None) + + async def execute_and_track( *, config_key: str, @@ -218,7 +233,7 @@ async def execute_and_track( # ``__ldValue`` is SDK-owned like ``__ld``: a caller-supplied one never # reaches the handler. merged_variables.pop("__ldValue", None) - if evaluated_value is not None: + if evaluated_value is not None and _handler_captures_content(handler): merged_variables["__ldValue"] = evaluated_value start_time = time.monotonic() @@ -324,7 +339,7 @@ async def execute_and_stream( # ``__ldValue`` is SDK-owned like ``__ld``: a caller-supplied one never # reaches the handler. merged_variables.pop("__ldValue", None) - if evaluated_value is not None: + if evaluated_value is not None and _handler_captures_content(handler): merged_variables["__ldValue"] = evaluated_value start_time = time.monotonic() diff --git a/packages/client/tests/test_tracking.py b/packages/client/tests/test_tracking.py index 6053b3bc..2b75a3da 100644 --- a/packages/client/tests/test_tracking.py +++ b/packages/client/tests/test_tracking.py @@ -127,3 +127,47 @@ async def handler( lifecycle_module._reset_for_testing() assert seen[0].get("__ldValue") == expected + + +class TestCustomHandlerCapture: + async def test_an_explicit_false_custom_handler_never_gets_the_raw_variation( + self, + ) -> None: + """A custom handler's function calls the helper without a flag; its own False wins.""" + from launchdarkly_ai_server import create_handler + from launchdarkly_ai_server.utils import set_ld_span_attributes + + events: list[dict[str, Any]] = [] + + class Span: + def set_attribute(self, *_a: Any) -> None: + pass + + def add_event(self, name: str, attributes: dict[str, Any]) -> None: + events.append(attributes) + + async def fn( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + set_ld_span_attributes(Span(), variables) + return {"output": "ok", "usage": {}} + + handler = create_handler(("Custom", "messages"), fn, None, False) + lifecycle_module._set_client_for_testing(_make_mock_client()) + try: + await execute_and_track( + config_key="c", + config={"model": {"name": "m"}, "provider": {"name": "Custom"}}, + meta={"variationKey": "v1", "version": 1}, + user_context={"kind": "user", "key": "u1"}, + handler=handler, + evaluated_value={"instructions": "secret prompt"}, + ) + finally: + lifecycle_module._reset_for_testing() + + assert events[0]["feature_flag.result.value"] == '"v1"' From 0ccd265309eec34ac24674ae524a100dcbdcfdd8 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 16:37:33 -0400 Subject: [PATCH 16/16] fix(client): let set_ld_span_attributes trust the core when capture_content is None With configure(capture_content=False), a custom handler built with an explicit capture_content=True got only the key: the core passed __ldValue because the handler's effective setting was on, but the helper resolved its own None to the process default again and dropped it. The core already passes __ldValue only to a capture-on handler, so None now writes it whenever it is present, as the JS helper does. An explicit False still suppresses it. --- .../src/launchdarkly_ai_server/utils.py | 13 +++++------- .../client/tests/test_ld_span_attributes.py | 13 ++++++------ packages/client/tests/test_tracking.py | 20 ++++++++++++++----- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 17ed7962..e1c283a8 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -816,9 +816,10 @@ def set_ld_span_attributes( writes it with value inclusion on. Otherwise, when the variation key is non-empty, it is ``json.dumps`` of the key, so a prompt never leaves the process without consent. Otherwise it is omitted (native-graph spans). - *capture_content* left as ``None`` follows the process default from - ``configure()``, so a custom handler gets the same value a built-in one - does. + *capture_content* left as ``None`` trusts the core: ``execute_and_track`` + and ``execute_and_stream`` only pass ``__ldValue`` to a handler whose + effective setting is on, so the value is written whenever it is present. + Only an explicit ``False`` suppresses it. """ span.set_attribute("launchdarkly.operation.type", "gen_ai") if not variables: @@ -839,11 +840,7 @@ def set_ld_span_attributes( variation_key = ld.get("variationKey") if isinstance(variation_key, str) and variation_key: feature_flag_attrs["feature_flag.result.variant"] = variation_key - if capture_content is None: - from .lifecycle import effective_capture_content - - capture_content = effective_capture_content(None) - if capture_content and variables.get("__ldValue") is not None: + if capture_content is not False and variables.get("__ldValue") is not None: feature_flag_attrs["feature_flag.result.value"] = json.dumps( variables["__ldValue"] ) diff --git a/packages/client/tests/test_ld_span_attributes.py b/packages/client/tests/test_ld_span_attributes.py index 6b2c2819..29c7904f 100644 --- a/packages/client/tests/test_ld_span_attributes.py +++ b/packages/client/tests/test_ld_span_attributes.py @@ -243,11 +243,9 @@ def test_capture_off_writes_the_variation_key_not_the_prompt(self) -> None: ) assert _feature_flag(span)["feature_flag.result.value"] == '"variation-a"' - @pytest.mark.parametrize(("default", "raw"), [(True, True), (False, False)]) - def test_an_unset_capture_follows_the_process_default( - self, default: bool, raw: bool - ) -> None: - """A custom handler that passes no capture_content gets what configure() says.""" + @pytest.mark.parametrize("default", [True, False]) + def test_an_unset_capture_trusts_the_core(self, default: bool) -> None: + """The core only passes __ldValue to a capture-on handler, so None writes it.""" lifecycle_module._reset_for_testing() configure(options=SDKOptions(capture_content=default)) try: @@ -255,8 +253,9 @@ def test_an_unset_capture_follows_the_process_default( set_ld_span_attributes(span, {**_vars(None), "__ldValue": RAW_VARIATION}) finally: lifecycle_module._reset_for_testing() - expected = json.dumps(RAW_VARIATION) if raw else '"variation-a"' - assert _feature_flag(span)["feature_flag.result.value"] == expected + assert _feature_flag(span)["feature_flag.result.value"] == json.dumps( + RAW_VARIATION + ) def test_capture_on_without_a_raw_variation_falls_back_to_the_key(self) -> None: span = FakeSpan() diff --git a/packages/client/tests/test_tracking.py b/packages/client/tests/test_tracking.py index 2b75a3da..dbe53bd3 100644 --- a/packages/client/tests/test_tracking.py +++ b/packages/client/tests/test_tracking.py @@ -130,10 +130,17 @@ async def handler( class TestCustomHandlerCapture: - async def test_an_explicit_false_custom_handler_never_gets_the_raw_variation( - self, + @pytest.mark.parametrize( + ("explicit", "default", "expected"), + [ + (False, True, '"v1"'), + (True, False, '{"instructions": "secret prompt"}'), + ], + ) + async def test_a_custom_handler_s_explicit_setting_beats_the_default( + self, explicit: bool, default: bool, expected: str ) -> None: - """A custom handler's function calls the helper without a flag; its own False wins.""" + """A custom handler's function calls the helper without a flag; its own setting beats the process default.""" from launchdarkly_ai_server import create_handler from launchdarkly_ai_server.utils import set_ld_span_attributes @@ -156,8 +163,11 @@ async def fn( set_ld_span_attributes(Span(), variables) return {"output": "ok", "usage": {}} - handler = create_handler(("Custom", "messages"), fn, None, False) + from launchdarkly_ai_server import SDKOptions, configure + + handler = create_handler(("Custom", "messages"), fn, None, explicit) lifecycle_module._set_client_for_testing(_make_mock_client()) + configure(options=SDKOptions(capture_content=default)) try: await execute_and_track( config_key="c", @@ -170,4 +180,4 @@ async def fn( finally: lifecycle_module._reset_for_testing() - assert events[0]["feature_flag.result.value"] == '"v1"' + assert events[0]["feature_flag.result.value"] == expected