diff --git a/TELEMETRY-CONTRACT.md b/TELEMETRY-CONTRACT.md index f4f7db93..212610b8 100644 --- a/TELEMETRY-CONTRACT.md +++ b/TELEMETRY-CONTRACT.md @@ -106,6 +106,8 @@ The root also carries one span event, `feature_flag`, with these event attribute |---|---| | `feature_flag.key` | config key | | `feature_flag.provider.name` | `LaunchDarkly` | +| `feature_flag.result.variant` | variation key, only when non-empty (graph spans may have none) | +| `feature_flag.result.value` | with content capture on (the existing `capture_content` setting, including the `configure()` default) and a raw variation present: `json.dumps` of the variation as the LaunchDarkly SDK returned it (`_ldMeta` included), the same serialization as the `ldotel` tracing hook with value inclusion on. Otherwise, when the variation key is non-empty: `json.dumps` of the key (for example `"\"v1\""`), which also covers `graph()` roots. Otherwise absent (native-graph spans) | | `feature_flag.set.id` | environment id, only when present | | `feature_flag.context.id` | canonical context key, only when `variables.ldContext` has a usable identity | | `feature_flag.contextKeys` | compact JSON of per-kind keys, only when identity is present | diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index 8cc2c3ee..53bf03d2 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -821,7 +821,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the finally below can tell an open root # from a closed one without asking the span. A mock span answers `is_recording()` @@ -1086,7 +1086,7 @@ async def _stream_gen( exit abandons it, and asyncio's finalizer later raises ``RuntimeError`` when it is suspended inside a real await in the SDK. """ - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) prompt, system_prompt = _build_prompt(config, user_input, variables, history) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/spans.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/spans.py index f25adc3e..2ef0ab54 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/spans.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/spans.py @@ -81,7 +81,9 @@ def tool_display_name(provider_name: str) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -92,7 +94,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, PROVIDER, model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 5222f989..6adc035c 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -571,7 +571,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the `finally` can tell an open root from a # closed one without asking the span. A mock span answers `is_recording()` truthily, and the @@ -713,7 +713,7 @@ async def _stream_gen( ``ended`` stops the success, failure and abandonment paths from ending the same span twice. """ - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) messages, system = _build_messages( diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/spans.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/spans.py index 3d55f7de..b87e882c 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/spans.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/spans.py @@ -46,7 +46,9 @@ def model_name(config: AiConfigRep) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -57,7 +59,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, PROVIDER, model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt index 4ac0e30a..607611f2 100644 --- a/packages/client/api-report/launchdarkly_ai_server.txt +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -61,8 +61,8 @@ EvaluationsModule [class] sdk_key: str | None [property] tools: ToolsClient [property] ui_base_uri: str [property] -execute_and_stream(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: Mapping[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None) -> AsyncGenerator[ExecuteStreamEvent, None] [async] -execute_and_track(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: Mapping[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None) -> dict[str, Any] [async] +execute_and_stream(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: Mapping[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None, evaluated_value: Any = None) -> AsyncGenerator[ExecuteStreamEvent, None] [async] +execute_and_track(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: Mapping[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None, evaluated_value: Any = None) -> dict[str, Any] [async] ExecuteStreamDoneEvent = dict[str, Any] ExecuteStreamEvent = dict[str, Any] extract_variation(config_key: str, context: dict[str, Any]) -> dict[str, Any] [async] @@ -173,7 +173,7 @@ JudgeRunResult [class, @dataclass] track_data: TrackData usage: UsageDict JudgeTask [class, @dataclass] - __init__(config_key: str, judge_config: AiConfigRep, judge_meta: VariationMeta, actual_output: str, user_context: LDContext, judge_provider: str | None, judge_mode: str, collapse_messages: bool, parent_track_data: TrackData, variables: dict[str, Any] | None = None, evaluation_metric_key: str | None = None, user_input: str | None = None, trajectory: str = '') + __init__(config_key: str, judge_config: AiConfigRep, judge_meta: VariationMeta, actual_output: str, user_context: LDContext, judge_provider: str | None, judge_mode: str, collapse_messages: bool, parent_track_data: TrackData, variables: dict[str, Any] | None = None, evaluation_metric_key: str | None = None, user_input: str | None = None, trajectory: str = '', judge_value: Any = None) actual_output: str collapse_messages: bool config_key: str @@ -182,6 +182,7 @@ JudgeTask [class, @dataclass] judge_meta: VariationMeta judge_mode: str judge_provider: str | None + judge_value: Any = None parent_track_data: TrackData trajectory: str = '' user_context: LDContext @@ -317,7 +318,7 @@ SDKOptions [class, @dataclass(frozen=True)] capture_content: bool = True set_conversation_id_if_absent(span: Any, conversation: str) -> None set_input_content_attributes(span: Any, capture: bool, *, system_instructions: str | None = None, messages: list[SpanMessage] | None = None, tool_definitions: list[ToolDefinitionInput] | None = None) -> None -set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None +set_ld_span_attributes(span: Any, variables: dict[str, Any] | None, *, capture_content: bool | None = None) -> None set_model_identity_attributes(span: Any, provider_name: str, request_model: str, legacy_system: str | None = None) -> None set_output_content_attributes(span: Any, capture: bool, messages: list[SpanMessage]) -> None set_skill_store(store: SkillStore | None) -> None diff --git a/packages/client/src/launchdarkly_ai_server/client.py b/packages/client/src/launchdarkly_ai_server/client.py index 48d1109c..6ca377e8 100644 --- a/packages/client/src/launchdarkly_ai_server/client.py +++ b/packages/client/src/launchdarkly_ai_server/client.py @@ -109,6 +109,7 @@ async def _invoke( tool_handlers=resolved_tools, variables=variables, history=history, + evaluated_value=variation.get("value"), ) raw_response = result["response"] @@ -240,6 +241,7 @@ async def _execute_stream( tool_handlers=resolved_tools, variables=variables, history=history, + evaluated_value=variation.get("value"), ) ) as source: async for event in source: diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index 069bb8e8..ca4db8e9 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -107,16 +107,16 @@ async def _fetch_graph_variation( topology = variation if isinstance(variation, dict) else None if not topology or not topology.get("root"): - return {"enabled": False, "meta": meta} + return {"enabled": False, "meta": meta, "value": variation} - return {"enabled": True, "topology": topology, "meta": meta} + return {"enabled": True, "topology": topology, "meta": meta, "value": variation} async def _build_graph( key: str, context: LDContext, options: dict[str, Any], -) -> tuple[GraphDefinition, TrackData, Callable[..., Any]]: +) -> tuple[GraphDefinition, TrackData, Callable[..., Any], Any]: from .judges import run_judges from .lifecycle import extract_variation, get_client from .tracking import _try_get_environment_id, execute_and_track @@ -127,6 +127,9 @@ async def _build_graph( enabled: bool = result["enabled"] topology: dict[str, Any] | None = result.get("topology") meta = result.get("meta", {}) + # The graph variation as the LD SDK returned it, for the root span's + # feature_flag.result.value. Kept off GraphDefinition, which is public. + graph_value = result.get("value") graph_track_data: TrackData = { "runId": str(uuid.uuid4()), @@ -147,6 +150,7 @@ async def _build_graph( _disabled_definition(key), graph_track_data, _disabled_stream_route(key), + graph_value, ) raw_edges_map: dict[str, list[dict[str, Any]]] = topology.get("edges") or {} @@ -171,11 +175,14 @@ def edges_from(node_key: str) -> list[GraphEdge]: return [e for e in edges if e.source_key == node_key] nodes: dict[str, GraphNode] = {} + # Raw variations, kept off the public GraphNode. + node_values: dict[str, Any] = {} try: for node_key in all_keys: variation = await extract_variation(node_key, context) node_config: AiConfigRep = variation["config"] node_meta: VariationMeta = variation["meta"] + node_values[node_key] = variation.get("value") node_edges = edges_from(node_key) nodes[node_key] = GraphNode( key=node_key, @@ -190,6 +197,7 @@ def edges_from(node_key: str) -> list[GraphEdge]: _disabled_definition(key), graph_track_data, _disabled_stream_route(key), + graph_value, ) root_node = nodes.get(topology["root"]) @@ -238,6 +246,7 @@ async def run_node( try: result = await execute_and_track( config_key=node.key, + evaluated_value=node_values.get(node.key), config=node.config, meta=node.meta, user_context=context, @@ -415,6 +424,7 @@ async def route( try: result = await execute_and_track( config_key=node.key, + evaluated_value=node_values.get(node.key), config=routed_config, meta=node.meta, user_context=context, @@ -530,6 +540,7 @@ async def stream_node( projector = StableProjector() async for event in execute_and_stream( config_key=node.key, + evaluated_value=node_values.get(node.key), config=node.config, meta=node.meta, user_context=context, @@ -670,6 +681,7 @@ async def stream_route( projector = StableProjector() async for event in execute_and_stream( config_key=node.key, + evaluated_value=node_values.get(node.key), config=routed_config, meta=node.meta, user_context=context, @@ -829,7 +841,7 @@ async def visit(node_key: str) -> None: reverse_traverse=reverse_traverse, ) - return graph_def, graph_track_data, stream_route + return graph_def, graph_track_data, stream_route, graph_value async def resolve_graph( @@ -857,7 +869,7 @@ async def resolve_graph( "tool_handlers": resolved_tools, "registry": registry, } - graph_def, _, _ = await _build_graph(key, context, options) + graph_def, _, _, _ = await _build_graph(key, context, options) return graph_def @@ -895,7 +907,7 @@ async def _invoke( from opentelemetry import trace from .judges import run_judges - from .lifecycle import get_client + from .lifecycle import effective_capture_content, get_client ld_ctx = to_ld_context(get_client(), context) @@ -918,7 +930,7 @@ async def _invoke( ) built = await _build_graph(self._key, context, resolved_options) - graph_def, built_track_data, _ = built + graph_def, built_track_data, _, graph_value = built if not graph_def.enabled: raise ValueError(f'Agent graph "{self._key}" is disabled') @@ -928,7 +940,13 @@ async def _invoke( tracer = trace.get_tracer("@launchdarkly/ai-server") with tracer.start_as_current_span("launchdarkly.graph") as span: set_ld_span_attributes( - span, {"__ld": graph_track_data, "ldContext": context} + span, + { + "__ld": graph_track_data, + "ldContext": context, + "__ldValue": graph_value, + }, + capture_content=effective_capture_content(None), ) start_time = time.monotonic() @@ -1120,7 +1138,7 @@ async def _stream_events( from opentelemetry.trace import Status, StatusCode, set_span_in_context from .judges import run_judges - from .lifecycle import get_client + from .lifecycle import effective_capture_content, get_client resolved_input = user_input or "" ld_ctx = to_ld_context(get_client(), context) @@ -1144,7 +1162,7 @@ async def _stream_events( ) built = await _build_graph(self._key, context, resolved_options) - graph_def, built_track_data, stream_route = built + graph_def, built_track_data, stream_route, graph_value = built if not graph_def.enabled: raise ValueError(f'Agent graph "{self._key}" is disabled') @@ -1153,7 +1171,11 @@ async def _stream_events( tracer = trace.get_tracer("@launchdarkly/ai-server") span = tracer.start_span("launchdarkly.graph", context=caller_context) - set_ld_span_attributes(span, {"__ld": graph_track_data, "ldContext": context}) + set_ld_span_attributes( + span, + {"__ld": graph_track_data, "ldContext": context, "__ldValue": graph_value}, + capture_content=effective_capture_content(None), + ) span_context = set_span_in_context(span, caller_context) ended: set[int] = set() start_time = time.monotonic() diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 93c9cec3..cf9ec3b8 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -202,6 +202,7 @@ async def run_judges( "response_to_evaluate": llm_response, **({"input": user_input} if user_input else {}), }, + evaluated_value=variation.get("value"), ) usage = to_usage_dict(result["usage"]) @@ -386,6 +387,7 @@ async def _build_judge_tasks( collapse_messages=collapse_messages, parent_track_data=base_track_data, evaluation_metric_key=evaluation_metric_key, + judge_value=variation.get("value"), ) ) except Exception as exc: @@ -459,6 +461,7 @@ async def run_judge( "response_to_evaluate": task.actual_output, **({"input": task.user_input} if task.user_input else {}), }, + evaluated_value=task.judge_value, ) usage = to_usage_dict(result["usage"]) diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index 335d9917..b53153f5 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -589,7 +589,7 @@ async def extract_variation( ) -> dict[str, Any]: """ Fetches an AI config variation from the LD client and parses it. - Returns ``{"config": AiConfigRep, "meta": VariationMeta}``. + Returns ``{"config": AiConfigRep, "meta": VariationMeta, "value": Any}``. Raises on disabled or invalid variations. Lazily initializes the LD client when ``LD_SDK_KEY`` is set, matching @@ -644,4 +644,7 @@ async def extract_variation( f"Invalid AI config variation for '{config_key}': {result.error['message']}" ) - return {"config": result.data, "meta": meta} + # ``value`` is the variation exactly as the LD SDK returned it, ``_ldMeta`` + # included. It stays off ``meta``, which surfaces publicly as + # ``GraphNode.meta`` and ``JudgeTask.judge_meta``. + return {"config": result.data, "meta": meta, "value": raw} diff --git a/packages/client/src/launchdarkly_ai_server/tracking.py b/packages/client/src/launchdarkly_ai_server/tracking.py index e93f6918..274c73a6 100644 --- a/packages/client/src/launchdarkly_ai_server/tracking.py +++ b/packages/client/src/launchdarkly_ai_server/tracking.py @@ -155,6 +155,21 @@ def _exposed_tool_keys( ] +def _handler_captures_content(handler: Any) -> bool: + """Effective content capture for a call made through *handler*. + + A :class:`ProviderHandler` answers with its own override, else the process + default. A plain function has no setting, so it follows the process + default. The raw variation only reaches a handler that captures content. + """ + capture = getattr(handler, "capture_content", None) + if isinstance(capture, bool): + return capture + from .lifecycle import effective_capture_content + + return effective_capture_content(None) + + async def execute_and_track( *, config_key: str, @@ -167,10 +182,13 @@ async def execute_and_track( variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None, + evaluated_value: Any = None, ) -> dict[str, Any]: """ Calls *handler*, emits LD telemetry events, and returns - ``{usage, response, track_data}``. + ``{usage, response, track_data}``. *evaluated_value* is the variation as + the LD SDK returned it; with content capture on it becomes the root + span's ``feature_flag.result.value``. """ from .lifecycle import get_client @@ -212,6 +230,11 @@ async def execute_and_track( "ldContext": {**user_context}, "__ld": track_data, } + # ``__ldValue`` is SDK-owned like ``__ld``: a caller-supplied one never + # reaches the handler. + merged_variables.pop("__ldValue", None) + if evaluated_value is not None and _handler_captures_content(handler): + merged_variables["__ldValue"] = evaluated_value start_time = time.monotonic() @@ -266,6 +289,7 @@ async def execute_and_stream( variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None, + evaluated_value: Any = None, ) -> AsyncGenerator[ExecuteStreamEvent, None]: """ Streaming counterpart to ``execute_and_track``. Yields ``chunk`` events, @@ -312,6 +336,11 @@ async def execute_and_stream( "ldContext": {**user_context}, "__ld": track_data, } + # ``__ldValue`` is SDK-owned like ``__ld``: a caller-supplied one never + # reaches the handler. + merged_variables.pop("__ldValue", None) + if evaluated_value is not None and _handler_captures_content(handler): + merged_variables["__ldValue"] = evaluated_value start_time = time.monotonic() full_text = "" diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 079f9d2e..7d4482ac 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -355,6 +355,12 @@ class JudgeTask: must stay picklable. """ """LD metric key to track the score against.""" + judge_value: Any = None + """The judge variation exactly as the LD SDK returned it. + + Written as the root span's ``feature_flag.result.value`` when the judge + handler captures content; otherwise the variation key is written. + """ @dataclass diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 5e1fabba..e1c283a8 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -43,7 +43,18 @@ def _create_handler( stream_fn: _StreamFn | None = None, capture_content: bool | None = None, ) -> ProviderHandler: - """Non-reporting :func:`create_handler`, used by the package factories.""" + """Non-reporting :func:`create_handler`, used by the package factories. + + Wrapping a :class:`ProviderHandler` again keeps its streaming + implementation and its explicit ``capture_content`` override unless new + ones are passed, as of the moment it is wrapped. An unset inner handler + stays unset. + """ + if isinstance(fn, ProviderHandler): + if stream_fn is None: + stream_fn = fn._stream_fn + if capture_content is None: + capture_content = fn._capture_content_override return ProviderHandler( fn=fn, provides_for=provides_for, @@ -768,7 +779,9 @@ def _context_identity_from_ld_context( return None -def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: +def set_ld_span_attributes( + span: Any, variables: dict[str, Any] | None, *, capture_content: bool | None = None +) -> None: """ Sets LaunchDarkly config-identifying attributes on an OTel span and emits the ``feature_flag`` span event required by the AI Config Monitoring Traces @@ -790,10 +803,23 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: Span event (required for AI Config Monitoring Traces tab correlation): ``name='feature_flag'`` with ``feature_flag.key``, - ``feature_flag.provider.name``, ``feature_flag.set.id`` (when + ``feature_flag.provider.name``, ``feature_flag.result.variant`` (the + served variation key, when non-empty), ``feature_flag.result.value`` + (see below), ``feature_flag.set.id`` (when ``LD_ENVIRONMENT_ID`` is set or the TS SDK auto-resolved it), ``feature_flag.context.id``, and ``feature_flag.contextKeys`` (when ``ldContext`` has a usable identity). + + ``feature_flag.result.value`` follows content capture. With + *capture_content* on and the raw variation in ``variables['__ldValue']``, + it is ``json.dumps`` of that variation, as the ``ldotel`` tracing hook + writes it with value inclusion on. Otherwise, when the variation key is + non-empty, it is ``json.dumps`` of the key, so a prompt never leaves the + process without consent. Otherwise it is omitted (native-graph spans). + *capture_content* left as ``None`` trusts the core: ``execute_and_track`` + and ``execute_and_stream`` only pass ``__ldValue`` to a handler whose + effective setting is on, so the value is written whenever it is present. + Only an explicit ``False`` suppresses it. """ span.set_attribute("launchdarkly.operation.type", "gen_ai") if not variables: @@ -811,6 +837,15 @@ def set_ld_span_attributes(span: Any, variables: dict[str, Any] | None) -> None: "feature_flag.key": ld.get("configKey", ""), "feature_flag.provider.name": "LaunchDarkly", } + variation_key = ld.get("variationKey") + if isinstance(variation_key, str) and variation_key: + feature_flag_attrs["feature_flag.result.variant"] = variation_key + if capture_content is not False and variables.get("__ldValue") is not None: + feature_flag_attrs["feature_flag.result.value"] = json.dumps( + variables["__ldValue"] + ) + elif isinstance(variation_key, str) and variation_key: + feature_flag_attrs["feature_flag.result.value"] = json.dumps(variation_key) if ld.get("environmentId"): feature_flag_attrs["feature_flag.set.id"] = ld["environmentId"] diff --git a/packages/client/tests/test_configure.py b/packages/client/tests/test_configure.py index 4f0ab359..7fc17c8c 100644 --- a/packages/client/tests/test_configure.py +++ b/packages/client/tests/test_configure.py @@ -246,3 +246,30 @@ async def test_init_client_does_not_carry_behavior_defaults( await init_client({"capture_content": False}, stub) assert _handler().capture_content is True assert any("capture_content" in record.getMessage() for record in caplog.records) + + +def test_rewrapping_keeps_an_explicit_override() -> None: + from launchdarkly_ai_server import create_handler + + inner = create_handler(("A", "messages"), _fn, None, False) + outer = create_handler(("B", "messages"), inner) + assert outer.capture_content is False + assert create_handler(("B", "messages"), inner, None, True).capture_content is True + + +def test_rewrapping_an_unset_handler_still_follows_configure() -> None: + from launchdarkly_ai_server import create_handler + + outer = create_handler(("B", "messages"), create_handler(("A", "messages"), _fn)) + configure(options=SDKOptions(capture_content=False)) + assert outer.capture_content is False + + +def test_rewrapping_keeps_the_stream() -> None: + from launchdarkly_ai_server import create_handler + + async def stream(*_args: Any, **_kwargs: Any) -> Any: + yield {"type": "done", "output": "", "usage": {}} + + inner = create_handler(("A", "messages"), _fn, stream) + assert create_handler(("B", "messages"), inner)._stream_fn is stream diff --git a/packages/client/tests/test_graph_stream.py b/packages/client/tests/test_graph_stream.py index f2e79596..69866a16 100644 --- a/packages/client/tests/test_graph_stream.py +++ b/packages/client/tests/test_graph_stream.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio +import json from collections.abc import AsyncGenerator, Iterator from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -17,7 +18,7 @@ from opentelemetry.trace import StatusCode import launchdarkly_ai_server.lifecycle as lifecycle_module -from launchdarkly_ai_server import ProviderHandler, graph +from launchdarkly_ai_server import ProviderHandler, SDKOptions, configure, graph from launchdarkly_ai_server.conversation import ( GEN_AI_CONVERSATION_ID, ConversationIdSpanProcessor, @@ -987,6 +988,50 @@ async def test_graph_span_carries_config_identity( assert len(flag_events) == 1 assert (flag_events[0].attributes or {})["feature_flag.key"] == "graph-key" + @pytest.mark.parametrize("mode", ["invoke", "stream"]) + @pytest.mark.parametrize("capture", [True, False]) + async def test_graph_span_event_carries_the_graph_variation( + self, mode: str, capture: bool + ) -> None: + """With capture on the result value is the raw graph variation; off, the key.""" + raw = { + "root": "root-node", + "edges": {"root-node": [{"key": "leaf-node"}]}, + "_ldMeta": {"enabled": True, "variationKey": "graph-v1", "version": 3}, + } + client = _make_client(raw) + lifecycle_module._set_client_for_testing(client) + configure(options=SDKOptions(capture_content=capture)) + try: + g = graph("graph-key", handlers=[_make_streaming_handler(["ok"])]) + if mode == "invoke": + await g.invoke("hi", CONTEXT) + else: + await _collect(g.stream("hi", CONTEXT)) + finally: + lifecycle_module._reset_for_testing() + + graph_spans = [s for s in _finished() if s.name == "launchdarkly.graph"] + event = next(e for e in graph_spans[0].events if e.name == "feature_flag") + attrs = event.attributes or {} + assert attrs["feature_flag.result.variant"] == "graph-v1" + expected = json.dumps(raw) if capture else json.dumps("graph-v1") + assert attrs["feature_flag.result.value"] == expected + + async def test_graph_span_without_a_variation_key_omits_the_result( + self, mock_ld_client: MagicMock + ) -> None: + """With capture off and no variation key there is nothing to write.""" + configure(options=SDKOptions(capture_content=False)) + await graph("graph-key", handlers=[_make_streaming_handler(["ok"])]).invoke( + "hi", CONTEXT + ) + graph_spans = [s for s in _finished() if s.name == "launchdarkly.graph"] + event = next(e for e in graph_spans[0].events if e.name == "feature_flag") + attrs = event.attributes or {} + assert "feature_flag.result.variant" not in attrs + assert "feature_flag.result.value" not in attrs + async def test_graph_parent_captured_at_stream_call_time( self, mock_ld_client: MagicMock ) -> None: diff --git a/packages/client/tests/test_judges.py b/packages/client/tests/test_judges.py index ee5fd7c7..c91d1ea0 100644 --- a/packages/client/tests/test_judges.py +++ b/packages/client/tests/test_judges.py @@ -1246,6 +1246,56 @@ async def test_judge_task_carries_no_output_format( roundtripped = json.loads(json.dumps(asdict(task))) assert "outputFormat" not in roundtripped["judge_config"] + async def test_judge_task_carries_the_raw_judge_variation( + self, mock_ld_client: MagicMock + ) -> None: + """A deferred judge writes the same feature_flag.result.value as an inline one.""" + judge_variation = _judge_variation_with_output_format() + mock_ld_client.variation = AsyncMock(return_value=judge_variation) + config = { + "model": {"name": "gpt-4"}, + "provider": {"name": "TestProvider"}, + "instructions": "hi", + "judgeConfiguration": {"judges": [{"key": "judge-1", "samplingRate": 1.0}]}, + } + + import random + + with patch.object(random, "random", return_value=0.0): + tasks = await build_judge_tasks( + config=config, + user_context=CONTEXT, + handler=_make_handler(), + llm_response="response", + base_track_data={"runId": "x"}, + ) + + assert tasks[0].judge_value == judge_variation + assert ( + json.loads(json.dumps(asdict(tasks[0])))["judge_value"] == judge_variation + ) + + seen: list[dict[str, Any]] = [] + + async def recording_fn( + config, user_input, tool_handlers, variables, history=None + ) -> dict: # type: ignore[override] + seen.append(dict(variables or {})) + return { + "output": '{"score": 0.6, "reasoning": "ok"}', + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + from launchdarkly_ai_server.types import ProviderHandler + + judge_handler = ProviderHandler( + fn=recording_fn, # type: ignore[arg-type] + provides_for=("TestProvider", tasks[0].judge_mode), + ) + await run_judge(tasks[0], [judge_handler]) + + assert seen[0]["__ldValue"] == judge_variation + async def test_run_judge_strips_defensively( self, mock_ld_client: MagicMock ) -> None: diff --git a/packages/client/tests/test_ld_span_attributes.py b/packages/client/tests/test_ld_span_attributes.py index edbda39d..29c7904f 100644 --- a/packages/client/tests/test_ld_span_attributes.py +++ b/packages/client/tests/test_ld_span_attributes.py @@ -2,10 +2,13 @@ from __future__ import annotations +import json from typing import Any import pytest +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_server import SDKOptions, configure from launchdarkly_ai_server.utils import set_ld_span_attributes LD_FIXTURE = { @@ -18,6 +21,14 @@ } +RAW_VARIATION: dict[str, Any] = { + "model": {"name": "gpt-x", "parameters": {"temperature": 0.5}}, + "provider": {"name": "openai"}, + "messages": [{"role": "system", "content": "Olá"}], + "_ldMeta": {"enabled": True, "variationKey": "variation-a"}, +} + + class FakeSpan: def __init__(self) -> None: self.attributes: dict[str, Any] = {} @@ -194,3 +205,69 @@ def test_malformed_ld_context_emits_none_of_the_three_and_does_not_throw( span = FakeSpan() set_ld_span_attributes(span, variables) _assert_no_context_identity(span) + + +class TestResultVariant: + def test_variant_is_the_variation_key(self) -> None: + span = FakeSpan() + set_ld_span_attributes(span, _vars(None)) + assert _feature_flag(span)["feature_flag.result.variant"] == "variation-a" + + @pytest.mark.parametrize("variation_key", ["", None]) + def test_empty_or_missing_variation_key_omits_the_variant( + self, variation_key: Any + ) -> None: + ld = {**LD_FIXTURE, "variationKey": variation_key} + if variation_key is None: + del ld["variationKey"] + span = FakeSpan() + set_ld_span_attributes(span, {"__ld": ld}) + assert "feature_flag.result.variant" not in _feature_flag(span) + + def test_capture_on_writes_the_raw_variation_like_the_tracing_hook( + self, + ) -> None: + span = FakeSpan() + set_ld_span_attributes( + span, {**_vars(None), "__ldValue": RAW_VARIATION}, capture_content=True + ) + # ldotel's TracingHook writes json.dumps(detail.value) with defaults. + assert _feature_flag(span)["feature_flag.result.value"] == json.dumps( + RAW_VARIATION + ) + + def test_capture_off_writes_the_variation_key_not_the_prompt(self) -> None: + span = FakeSpan() + set_ld_span_attributes( + span, {**_vars(None), "__ldValue": RAW_VARIATION}, capture_content=False + ) + assert _feature_flag(span)["feature_flag.result.value"] == '"variation-a"' + + @pytest.mark.parametrize("default", [True, False]) + def test_an_unset_capture_trusts_the_core(self, default: bool) -> None: + """The core only passes __ldValue to a capture-on handler, so None writes it.""" + lifecycle_module._reset_for_testing() + configure(options=SDKOptions(capture_content=default)) + try: + span = FakeSpan() + set_ld_span_attributes(span, {**_vars(None), "__ldValue": RAW_VARIATION}) + finally: + lifecycle_module._reset_for_testing() + assert _feature_flag(span)["feature_flag.result.value"] == json.dumps( + RAW_VARIATION + ) + + def test_capture_on_without_a_raw_variation_falls_back_to_the_key(self) -> None: + span = FakeSpan() + set_ld_span_attributes(span, _vars(None), capture_content=True) + assert _feature_flag(span)["feature_flag.result.value"] == '"variation-a"' + + @pytest.mark.parametrize("capture", [True, False]) + def test_no_raw_variation_and_no_key_omits_the_value(self, capture: bool) -> None: + span = FakeSpan() + set_ld_span_attributes( + span, + {"__ld": {**LD_FIXTURE, "variationKey": ""}}, + capture_content=capture, + ) + assert "feature_flag.result.value" not in _feature_flag(span) diff --git a/packages/client/tests/test_lifecycle.py b/packages/client/tests/test_lifecycle.py index 0987bc43..c3f12a43 100644 --- a/packages/client/tests/test_lifecycle.py +++ b/packages/client/tests/test_lifecycle.py @@ -5,6 +5,7 @@ import ast import asyncio import inspect +import json import os import subprocess import sys @@ -1030,6 +1031,28 @@ async def test_preserves_model_key_and_version_on_meta(self) -> None: assert extracted["meta"]["modelKey"] == "my-model" assert extracted["meta"]["modelVersion"] == 3 + async def test_returns_the_raw_variation_beside_meta_not_on_it(self) -> None: + raw = { + "_ldMeta": {"enabled": True, "variationKey": "v1", "version": 1}, + "model": {"name": "claude-3-5"}, + "provider": {"name": "Anthropic"}, + "instructions": "You are helpful.", + } + stub = _make_stub_client() + stub.variation = AsyncMock(return_value=raw) + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=stub) + ctx = {"kind": "user", "key": "user-1"} + with patch( + "launchdarkly_ai_server.utils.to_ld_context", + side_effect=lambda _c, ctx: ctx, + ): + extracted = await lifecycle_module.extract_variation("my-flag", ctx) + + assert extracted["value"] == raw + assert raw not in extracted["meta"].values() + assert "instructions" not in json.dumps(extracted["meta"]) + async def test_returns_enabled_false_and_null_config_when_disabled(self) -> None: stub = _make_stub_client() stub.variation = AsyncMock(return_value={"_ldMeta": {"enabled": False}}) diff --git a/packages/client/tests/test_tracking.py b/packages/client/tests/test_tracking.py index eb9db7dc..dbe53bd3 100644 --- a/packages/client/tests/test_tracking.py +++ b/packages/client/tests/test_tracking.py @@ -1,10 +1,15 @@ """ -Tests for wrap_tool_handlers. +Tests for wrap_tool_handlers and the handler variables execute_and_track builds. """ +from typing import Any from unittest.mock import MagicMock, patch +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module from launchdarkly_ai_server import NATIVE_TOOL_KEY, NativeTool, wrap_tool_handlers +from launchdarkly_ai_server.tracking import execute_and_track def _make_mock_client() -> MagicMock: @@ -85,3 +90,94 @@ def test_handoff_prefix_skips_tracking(self) -> None: def test_undefined_tool_handlers(self) -> None: result = wrap_tool_handlers(None, CONTEXT, TRACK_DATA) assert result == {} + + +class TestHandlerVariables: + @pytest.mark.parametrize( + ("evaluated", "expected"), + [(None, None), ({"raw": True}, {"raw": True})], + ) + async def test_a_caller_supplied_ld_value_never_reaches_the_handler( + self, evaluated: Any, expected: Any + ) -> None: + seen: list[dict[str, Any]] = [] + + async def handler( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + seen.append(dict(variables or {})) + return {"output": "ok", "usage": {}} + + lifecycle_module._set_client_for_testing(_make_mock_client()) + try: + await execute_and_track( + config_key="c", + config={"model": {"name": "m"}, "provider": {"name": "P"}}, + meta={"variationKey": "v", "version": 1}, + user_context={"kind": "user", "key": "u1"}, + handler=handler, # type: ignore[arg-type] + variables={"__ldValue": "from-caller"}, + evaluated_value=evaluated, + ) + finally: + lifecycle_module._reset_for_testing() + + assert seen[0].get("__ldValue") == expected + + +class TestCustomHandlerCapture: + @pytest.mark.parametrize( + ("explicit", "default", "expected"), + [ + (False, True, '"v1"'), + (True, False, '{"instructions": "secret prompt"}'), + ], + ) + async def test_a_custom_handler_s_explicit_setting_beats_the_default( + self, explicit: bool, default: bool, expected: str + ) -> None: + """A custom handler's function calls the helper without a flag; its own setting beats the process default.""" + from launchdarkly_ai_server import create_handler + from launchdarkly_ai_server.utils import set_ld_span_attributes + + events: list[dict[str, Any]] = [] + + class Span: + def set_attribute(self, *_a: Any) -> None: + pass + + def add_event(self, name: str, attributes: dict[str, Any]) -> None: + events.append(attributes) + + async def fn( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict[str, Any]: + set_ld_span_attributes(Span(), variables) + return {"output": "ok", "usage": {}} + + from launchdarkly_ai_server import SDKOptions, configure + + handler = create_handler(("Custom", "messages"), fn, None, explicit) + lifecycle_module._set_client_for_testing(_make_mock_client()) + configure(options=SDKOptions(capture_content=default)) + try: + await execute_and_track( + config_key="c", + config={"model": {"name": "m"}, "provider": {"name": "Custom"}}, + meta={"variationKey": "v1", "version": 1}, + user_context={"kind": "user", "key": "u1"}, + handler=handler, + evaluated_value={"instructions": "secret prompt"}, + ) + finally: + lifecycle_module._reset_for_testing() + + assert events[0]["feature_flag.result.value"] == expected diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index f13f7dbb..c64fb960 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -679,7 +679,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the `finally` below can tell an open root # from a closed one without asking the span. A mock span answers `is_recording()` @@ -843,7 +843,7 @@ async def _stream_gen( """ import importlib - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) system_prompt = _extract_system_prompt(config, variables) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/spans.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/spans.py index 3621de06..c17f0a7a 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/spans.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/spans.py @@ -75,7 +75,9 @@ def serving_provider(config: AiConfigRep) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -88,7 +90,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: set_model_identity_attributes( span, serving_provider(config), model_name(config), legacy_system="langchain" ) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index 0ecaca1b..cf6eb6a8 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -809,7 +809,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the `finally` can tell an open root from a # closed one without asking the span. A mock span answers `is_recording()` truthily, and the @@ -1147,7 +1147,7 @@ async def _stream_gen( base_model = await _resolve_base_model(config, llm, importlib) - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) initial_messages = _build_messages(config, user_input, variables, history) diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/spans.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/spans.py index 1bbdc09b..9bef62e3 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/spans.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/spans.py @@ -63,7 +63,9 @@ def serving_provider(config: AiConfigRep) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -76,7 +78,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: set_model_identity_attributes( span, serving_provider(config), model_name(config), LEGACY_SYSTEM ) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py b/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py index 6531fa3f..99de5d9c 100644 --- a/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py +++ b/packages/litellm-agents/src/launchdarkly_ai_litellm_agents/handler.py @@ -310,7 +310,12 @@ def _direct_usage(raw: Any) -> SpanUsage: ) -def _root_span(config_value: AiConfigRep, variables: dict[str, Any]) -> Any: +def _root_span( + config_value: AiConfigRep, + variables: dict[str, Any], + *, + capture_content: bool = False, +) -> Any: model_name = _model_config(config_value)[0] provider = str( (config_value.get("provider") or {}).get("name") or "litellm" @@ -320,7 +325,7 @@ def _root_span(config_value: AiConfigRep, variables: dict[str, Any]) -> Any: ) span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, provider, model_name, "litellm") - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span @@ -589,7 +594,7 @@ async def call( history, factory, ) - span = _root_span(config_value, values) + span = _root_span(config_value, values, capture_content=capture_content) parent = _parent_context(span) model_name = _model_config(config_value)[0] ended: set[int] = set() @@ -676,7 +681,7 @@ async def _stream( agent, prompt, instructions, max_turns = _agent_and_prompt( config_value, user_input, handlers, variables, history, model_factory ) - span = _root_span(config_value, variables) + span = _root_span(config_value, variables, capture_content=capture_content) parent = _parent_context(span) streamed: Any = None completed = False diff --git a/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py b/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py index 00ca89f2..7bdcc58e 100644 --- a/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py +++ b/packages/litellm-messages/src/launchdarkly_ai_litellm_messages/handler.py @@ -279,7 +279,12 @@ def _request( return parameters -def _root_span(config_value: AiConfigRep, variables: dict[str, Any]) -> Any: +def _root_span( + config_value: AiConfigRep, + variables: dict[str, Any], + *, + capture_content: bool = False, +) -> Any: span = trace.get_tracer("@launchdarkly/ai-litellm-messages").start_span( "invoke_agent" ) @@ -287,7 +292,7 @@ def _root_span(config_value: AiConfigRep, variables: dict[str, Any]) -> Any: set_model_identity_attributes( span, _provider_name(config_value), _model_name(config_value), "litellm" ) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span @@ -466,7 +471,7 @@ async def call( values = variables or {} messages = _build_messages(config_value, user_input, values, history) tools = _build_tools(config_value, handlers) - root = _root_span(config_value, values) + root = _root_span(config_value, values, capture_content=capture_content) parent = _parent_context(root) total = SpanUsage() usage_reported = False @@ -607,7 +612,7 @@ async def _stream( ) -> AsyncGenerator[dict[str, Any], None]: messages = _build_messages(config_value, user_input, variables, history) tools = _build_tools(config_value, handlers) - root = _root_span(config_value, variables) + root = _root_span(config_value, variables, capture_content=capture_content) parent = _parent_context(root) total = SpanUsage() usage_reported = False diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index 58647da6..a57e071b 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -685,7 +685,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared once this path (success or failure) has ended the root, so the `finally` below can # tell an open root from a closed one without asking the span. A mock span answers @@ -811,7 +811,7 @@ async def _stream_gen( agents_mod = importlib.import_module("agents") Runner = agents_mod.Runner - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) agent, prompt, instructions = _build_agent_and_prompt( diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/spans.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/spans.py index 128fe8d0..7a06aae5 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/spans.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/spans.py @@ -75,7 +75,9 @@ def model_name(config: AiConfigRep) -> str: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -86,7 +88,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, PROVIDER, model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/openai-agents/tests/test_handler.py b/packages/openai-agents/tests/test_handler.py index e15fb693..6283d046 100644 --- a/packages/openai-agents/tests/test_handler.py +++ b/packages/openai-agents/tests/test_handler.py @@ -825,6 +825,26 @@ async def test_run_totals_on_the_root(self) -> None: assert attrs["gen_ai.usage.output_tokens"] == 5 assert attrs["gen_ai.usage.total_tokens"] == 15 + async def test_capture_on_writes_the_raw_variation_as_the_result_value( + self, + ) -> None: + ctx, rec = _recording() + agents_mod = _fake_agents_module( + run=_make_run([{"output": _text_output("hi")}]) + ) + raw = {"model": {"name": "m"}, "_ldMeta": {"variationKey": "v1"}} + vs = { + "__ld": {"configKey": "my-config", "variationKey": "v1", "runId": "r"}, + "__ldValue": raw, + } + with ctx, _patched_agents(agents_mod): + await create_openai_agents_handler(capture_content=True)( + CONFIG, "q", {}, vs + ) + (name, attrs), *_ = rec.root.events + assert name == "feature_flag" + assert attrs["feature_flag.result.value"] == json.dumps(raw) + async def test_feature_flag_event_on_root_only(self) -> None: ctx, rec = _recording() agents_mod = _fake_agents_module( @@ -836,16 +856,22 @@ async def test_feature_flag_event_on_root_only(self) -> None: "variationKey": "v1", "runId": "run-abc", "environmentId": "env-1", - } + }, + "__ldValue": {"model": {"name": "m"}, "_ldMeta": {"variationKey": "v1"}}, } with ctx, _patched_agents(agents_mod): - await create_openai_agents_handler()(CONFIG, "q", {}, vs) + await create_openai_agents_handler(capture_content=False)( + CONFIG, "q", {}, vs + ) assert [event for event in rec.root.events if event[0] == "feature_flag"] == [ ( "feature_flag", { "feature_flag.key": "my-config", "feature_flag.provider.name": "LaunchDarkly", + "feature_flag.result.variant": "v1", + # Content capture is off, so the key stands in for the variation. + "feature_flag.result.value": '"v1"', "feature_flag.set.id": "env-1", }, ) diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index 0b0d63f9..3e616c17 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -417,7 +417,7 @@ async def _call_impl( th = tool_handlers or {} vs = variables or {} - span = start_root_span(config, vs) + span = start_root_span(config, vs, capture_content=capture_content) parent = parent_context_of(span) # Cleared by whichever path ends the root, so the `finally` can tell an open root from a # closed one without asking the span. A mock span answers `is_recording()` truthily, and @@ -627,7 +627,7 @@ async def _stream_gen( the root span is never ended, so it is never exported, and the whole run disappears from AI Config Monitoring along with the ``feature_flag`` event it carries. """ - span = start_root_span(config, variables) + span = start_root_span(config, variables, capture_content=capture_content) parent = parent_context_of(span) ended: set[int] = set() diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/spans.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/spans.py index 3a78a2d6..dc4fc056 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/spans.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/spans.py @@ -81,7 +81,9 @@ def _attr(obj: Any, name: str) -> Any: # ─── Span starts ───────────────────────────────────────────────────────────── -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent. The root is the only span carrying ``launchdarkly.*`` and the ``feature_flag`` event, so it is @@ -92,7 +94,7 @@ def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, PROVIDER, model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py index 9ca641d0..d18e0325 100644 --- a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py +++ b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/handler.py @@ -892,7 +892,7 @@ async def _stream_impl( history: list[dict[str, Any]] | None = None, ) -> AsyncGenerator[dict[str, Any], None]: capture = effective_capture_content(self._capture_content_override) - span = start_root_span(config, variables or {}) + span = start_root_span(config, variables or {}, capture_content=capture) parent = parent_context_of(span) ended: set[int] = set() open_tool_spans: dict[str, Any] = {} diff --git a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/spans.py b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/spans.py index 84ec5c46..7d3f9d52 100644 --- a/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/spans.py +++ b/packages/opencode-agents/src/launchdarkly_ai_opencode_agents/spans.py @@ -51,14 +51,16 @@ def provider_name(config: AiConfigRep) -> str: return "opencode" -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: """Opens the ``invoke_agent`` root and returns it, or ``None`` when OTel is absent.""" if not _HAS_OTEL: return None span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, provider_name(config), model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py b/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py index b7c0dcee..8bcd1c4f 100644 --- a/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py +++ b/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py @@ -116,7 +116,9 @@ async def _call( chat = _TRACER.start_span( f"chat {span_model}", context=trace.set_span_in_context(root) ) - set_ld_span_attributes(root, dict(variables) if variables else None) + set_ld_span_attributes( + root, dict(variables) if variables else None, capture_content=capture + ) set_model_identity_attributes(root, "typesafe", span_model) set_model_identity_attributes(chat, "typesafe", span_model) if capture: diff --git a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py index 051f6b65..03bc1f48 100644 --- a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py +++ b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/handler.py @@ -24,6 +24,7 @@ parse_template, report_usage, ) +from launchdarkly_ai_server.lifecycle import effective_capture_content from launchdarkly_ai_server.parameter_forwarding import select_forwarded_parameters from launchdarkly_ai_server.utils import model_parameters @@ -523,7 +524,9 @@ async def run( history: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: vs = variables or {} - span = start_root_span(cfg, vs) + span = start_root_span( + cfg, vs, capture_content=effective_capture_content(capture_content) + ) parent = parent_context_of(span) turns = _ChatTurns(cfg, parent) failed = False @@ -576,7 +579,9 @@ async def stream( history: list[dict[str, Any]] | None = None, ) -> AsyncGenerator[dict[str, Any], None]: vs = variables or {} - span = start_root_span(cfg, vs) + span = start_root_span( + cfg, vs, capture_content=effective_capture_content(capture_content) + ) parent = parent_context_of(span) turns = _ChatTurns(cfg, parent) completed = False diff --git a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/spans.py b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/spans.py index 0762842f..ae2e591d 100644 --- a/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/spans.py +++ b/packages/vercel-agents/src/launchdarkly_ai_vercel_agents/spans.py @@ -24,11 +24,13 @@ def serving_provider(config: AiConfigRep) -> str: return str(config.get("provider", {}).get("name") or "vercel").lower() -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, serving_provider(config), model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py index 051ebf6a..f18a0ffd 100644 --- a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py +++ b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/handler.py @@ -24,6 +24,7 @@ parse_template, report_usage, ) +from launchdarkly_ai_server.lifecycle import effective_capture_content from launchdarkly_ai_server.parameter_forwarding import select_forwarded_parameters from launchdarkly_ai_server.utils import model_parameters @@ -546,7 +547,9 @@ async def invoke( history: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: vs = variables or {} - span = start_root_span(cfg, vs) + span = start_root_span( + cfg, vs, capture_content=effective_capture_content(capture_content) + ) parent = parent_context_of(span) failed = False cancelled = False @@ -599,7 +602,9 @@ async def stream( history: list[dict[str, Any]] | None = None, ) -> AsyncGenerator[dict[str, Any], None]: vs = variables or {} - span = start_root_span(cfg, vs) + span = start_root_span( + cfg, vs, capture_content=effective_capture_content(capture_content) + ) parent = parent_context_of(span) completed = False failed = False diff --git a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/spans.py b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/spans.py index a4a7cef4..1bf9e57d 100644 --- a/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/spans.py +++ b/packages/vercel-messages/src/launchdarkly_ai_vercel_messages/spans.py @@ -24,11 +24,13 @@ def serving_provider(config: AiConfigRep) -> str: return str(config.get("provider", {}).get("name") or "vercel").lower() -def start_root_span(config: AiConfigRep, variables: dict[str, Any]) -> Any: +def start_root_span( + config: AiConfigRep, variables: dict[str, Any], *, capture_content: bool = False +) -> Any: span = trace.get_tracer(TRACER_NAME).start_span("invoke_agent") span.set_attribute("gen_ai.operation.name", "invoke_agent") set_model_identity_attributes(span, serving_provider(config), model_name(config)) - set_ld_span_attributes(span, variables) + set_ld_span_attributes(span, variables, capture_content=capture_content) return span diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index 60a3375f..a53199df 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -376,6 +376,10 @@ def test_the_langchain_provider_name_is_the_configured_name( "feature_flag", "feature_flag.key", "feature_flag.provider.name", + # OTel feature-flag semconv: the served variation key, and the evaluated value (the raw + # variation with content capture on, else the JSON-encoded key). + "feature_flag.result.variant", + "feature_flag.result.value", "feature_flag.set.id", # AIC-3230: evaluation-context identity on the root feature_flag event / span. # `context.contextKeys` is the f-string prefix; the keys actually emitted are @@ -677,3 +681,54 @@ def test_abandonment_is_distinct_from_failure(self, name: str) -> None: f"{name} has a close_open_spans that neither records the failure nor hands off to " "one that does, so it closes nothing." ) + + +# --------------------------------------------------------------------------- +# The root span's feature_flag value follows the handler's content capture +# --------------------------------------------------------------------------- + +#: Every handler module that opens a run's root span. The root's feature_flag event writes the raw +#: variation as `feature_flag.result.value` only when capture is on, so each must pass its resolved +#: setting. Left off, the keyword defaults to False and capture-on runs silently lose the value. +ROOT_SPAN_HANDLERS: dict[str, str] = { + "claude-agents": "launchdarkly_ai_claude_agents.handler", + "claude-messages": "launchdarkly_ai_claude_messages.handler", + "langchain-agents": "launchdarkly_ai_langchain_agents.handler", + "langchain-messages": "launchdarkly_ai_langchain_messages.handler", + "litellm-agents": "launchdarkly_ai_litellm_agents.handler", + "litellm-messages": "launchdarkly_ai_litellm_messages.handler", + "openai-agents": "launchdarkly_ai_openai_agents.handler", + "openai-messages": "launchdarkly_ai_openai_messages.handler", + "opencode-agents": "launchdarkly_ai_opencode_agents.handler", + "typesafe": "launchdarkly_ai_typesafe.handler", + "vercel-agents": "launchdarkly_ai_vercel_agents.handler", + "vercel-messages": "launchdarkly_ai_vercel_messages.handler", +} + +_ROOT_SPAN_CALLS = {"start_root_span", "_root_span", "set_ld_span_attributes"} + + +def _root_span_calls(source: str) -> list[ast.Call]: + calls: list[ast.Call] = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Call): + func = node.func + name = ( + func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + ) + if name in _ROOT_SPAN_CALLS: + calls.append(node) + return calls + + +class TestRootSpanCaptureContent: + @pytest.mark.parametrize("name", sorted(ROOT_SPAN_HANDLERS)) + def test_every_root_span_call_passes_capture_content(self, name: str) -> None: + source = _handler_source(ROOT_SPAN_HANDLERS[name]) + calls = _root_span_calls(source) + assert calls, f"{name} opens no root span this check can see." + for call in calls: + assert any(kw.arg == "capture_content" for kw in call.keywords), ( + f"{name} opens a root span without capture_content= at line " + f"{call.lineno}, so feature_flag.result.value ignores content capture." + ) diff --git a/uv.lock b/uv.lock index f3053e7f..7ff3f4e4 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.15'",