From 43ecea876b676b9b7184fd2e85b4e96aba3efb64 Mon Sep 17 00:00:00 2001 From: Dylan O'Neill Date: Wed, 28 Oct 2026 09:59:52 -0700 Subject: [PATCH] feat(evaluations)!: promote Evaluations from Code to core Export the Evaluations from Code names from launchdarkly_ai_server. Remove launchdarkly_ai_server.experimental.evaluations. Report the usage helper as client.initEvaluations. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 +- packages/ai/README.md | 6 +- packages/ai/tests/test_reexports.py | 6 +- packages/client/README.md | 12 +-- packages/client/agents.md | 25 ++++-- ...kly_ai_server.experimental.evaluations.txt | 83 ------------------- .../api-report/launchdarkly_ai_server.txt | 82 ++++++++++++++++++ .../src/launchdarkly_ai_server/__init__.py | 34 ++++++++ .../evaluations/module.py | 2 +- .../experimental/__init__.py | 2 +- .../experimental/evaluations.py | 43 ---------- packages/client/tests/test_evaluations.py | 50 +++++------ packages/client/tests/test_evaluations_run.py | 7 +- .../client/tests/test_evaluations_spans.py | 14 ++-- packages/client/tests/test_lifecycle.py | 26 ++++-- packages/client/tests/test_public_surface.py | 15 ++++ packages/client/tests/test_sdk_usage.py | 8 +- release-please-config.json | 3 - 18 files changed, 220 insertions(+), 204 deletions(-) delete mode 100644 packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt delete mode 100644 packages/client/src/launchdarkly_ai_server/experimental/evaluations.py diff --git a/AGENTS.md b/AGENTS.md index b4eb3bb3..e7fa7b82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,10 +86,10 @@ Every public module has a committed API report at `packages//api-report PRs are squash-merged with the PR title as the commit message, and release-please builds each package's changelog from those messages. -- **A change to an experimental feature uses that feature's scope:** `evaluations` (Evaluations from Code). `feat`, `fix` and `perf` commits with that scope are listed under **Experimental** (`changelog-sections` in `release-please-config.json`), for example `feat(evaluations): score rows in parallel`. Any other scope, such as `client` or `skills`, lists the change as a core feature or fix. Agent Skills is core, so `feat(skills):` lands under Features. -- **Use exactly one scope.** `feat(evaluations,client):` matches no Experimental row, so it lands under Features. +- **A change to an experimental feature uses that feature's scope.** `feat`, `fix` and `perf` commits with that scope are listed under **Experimental** (`changelog-sections` in `release-please-config.json`). No feature is experimental now. Any other scope, such as `client`, `skills` or `evaluations`, lists the change as a core feature or fix. Agent Skills and Evaluations from Code are core, so `feat(skills):` and `feat(evaluations):` land under Features. +- **Use exactly one scope.** `feat(beta,client):` matches no Experimental row, so it lands under Features. - **A PR that changes both core and experimental behaviour** should be split. If it can't be, add a `BEGIN_COMMIT_OVERRIDE` block to its description with one conventional-commit line per change. An override also corrects a merged PR's title, as long as the release PR hasn't merged yet. -- **A breaking experimental change** uses `!` while the package is 0.x: `feat(evaluations)!:` bumps the minor version, which is what an experimental break is allowed. After 1.0, `!` would force a major release. Use `feat(evaluations):` without `!` instead, and say the change is breaking in the title. +- **A breaking experimental change** uses `!` while the package is 0.x: `feat(beta)!:` bumps the minor version, which is what an experimental break is allowed. After 1.0, `!` would force a major release. Use `feat(beta):` without `!` instead, and say the change is breaking in the title. - **When a feature is promoted to core,** remove its rows from `changelog-sections`. The rule is specified in `TESTING.md` ยง0.3 in `launchdarkly/ai-sdks-monorepo`. diff --git a/packages/ai/README.md b/packages/ai/README.md index 391ce7b8..edb6413a 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -70,10 +70,10 @@ Experimental features are not re-exported from this package. They live in `launchdarkly-ai-server`, the package this one depends on, and need a `launchdarkly-ai-server` release that includes them. Import them from `launchdarkly_ai_server.experimental`; if that import fails, upgrade `launchdarkly-ai-server` itself, since upgrading this package alone can -leave an older `launchdarkly-ai-server` in place. The experimental feature is: +leave an older `launchdarkly-ai-server` in place. No feature is experimental now. -- **Evaluations from Code**, in `launchdarkly_ai_server.experimental.evaluations`; see the - [evaluations guide](https://github.com/launchdarkly/python-ai-sdk/blob/main/packages/client/README.md#run-an-evaluation-from-code-experimental). +Evaluations from Code is core, so this package re-exports it. See the +[evaluations guide](https://github.com/launchdarkly/python-ai-sdk/blob/main/packages/client/README.md#run-an-evaluation-from-code). --- diff --git a/packages/ai/tests/test_reexports.py b/packages/ai/tests/test_reexports.py index 18a7fbe6..aa52cc4d 100644 --- a/packages/ai/tests/test_reexports.py +++ b/packages/ai/tests/test_reexports.py @@ -27,9 +27,9 @@ def test_reexported_values_are_identical_references(self) -> None: f"launchdarkly_ai_python.{name} is not the same object as launchdarkly_ai_server.{name}" ) - def test_experimental_evaluations_are_not_reexported(self) -> None: - importlib.import_module("launchdarkly_ai_server.experimental.evaluations") + def test_evaluations_are_reexported(self) -> None: + server = importlib.import_module("launchdarkly_ai_server") barrel = importlib.import_module("launchdarkly_ai_python") for name in ("init_evaluations", "EvalTool", "Judge", "Scorer", "DatasetRow"): - assert not hasattr(barrel, name), name + assert getattr(barrel, name) is getattr(server, name), name diff --git a/packages/client/README.md b/packages/client/README.md index a7c10e15..a78e98bf 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -44,11 +44,7 @@ No code changes are required โ€” `init_client()` detects the packages at runtime | `LD_API_BASE_URI` | No | Evaluations management API host override; intentionally separate from `LD_BASE_URI` | | `LD_UI_BASE_URI` | No | LaunchDarkly application host for evaluation-run links (default: `https://app.launchdarkly.com`). Set it for a non-production project, or its runs still link to the production app | -### Run an evaluation from code (experimental) - -> **Experimental.** Import Evaluations from Code from `launchdarkly_ai_server.experimental.evaluations`; -> none of these names is exported from the package root. They may change in a minor release, -> so review the changelog when you upgrade. +### Run an evaluation from code The evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. Rows can then be scored by LaunchDarkly judges and local scorer functions; see [Score rows with judges and scorers](#score-rows-with-judges-and-scorers). Each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event is logged at `INFO` on the `launchdarkly_ai_server.evaluations.runner` logger with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time once that logger is enabled. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. @@ -61,7 +57,7 @@ import asyncio import sys from launchdarkly_ai_openai_messages import create_openai_messages_handler -from launchdarkly_ai_server.experimental.evaluations import init_evaluations +from launchdarkly_ai_server import init_evaluations async def main() -> int: @@ -122,7 +118,7 @@ Pass `criteria` to `run()` to score every generated row. A `Judge` references an ```python from launchdarkly_ai_claude_messages import create_claude_messages_handler from launchdarkly_ai_openai_messages import create_openai_messages_handler -from launchdarkly_ai_server.experimental.evaluations import ( +from launchdarkly_ai_server import ( DatasetRow, Judge, Scorer, init_evaluations, ) @@ -269,7 +265,7 @@ async def main(): Pass `tools` to `run()` as a list of `EvalTool`. Construct one to define a tool in code. Await `evals.tools.get()` to use a tool that already exists in your project's AI library, which reads the tool and pins its version at that point. One list can hold both kinds. ```python -from launchdarkly_ai_server.experimental.evaluations import EvalTool, init_evaluations +from launchdarkly_ai_server import EvalTool, init_evaluations def lookup_order(order_id: str) -> str: diff --git a/packages/client/agents.md b/packages/client/agents.md index 3eea9a15..03ec106c 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -39,7 +39,7 @@ No other `launchdarkly-ai-*` package may define or duplicate these. They import | `src/launchdarkly_ai_server/utils.py` | `parse_template`, `parse_json_with_possible_fences`, `create_handler`, `parse_usage`, `make_track_data`, `to_ld_context` | | `src/launchdarkly_ai_server/registry.py` | `Registry`, `global_registry`, `compose`, `resolve_handlers`, `resolve_tools` | | `src/launchdarkly_ai_server/judges.py` | `run_judges`, `build_judge_tasks`, `run_judge` | -| `src/launchdarkly_ai_server/evaluations/` | `init_evaluations`, the private management API operations, and `EvaluationsModule.run()` orchestration. Experimental: public names are exported only from `experimental/evaluations.py`, never from the package root | +| `src/launchdarkly_ai_server/evaluations/` | `init_evaluations`, the private management API operations, and `EvaluationsModule.run()` orchestration. Public names are exported from the package root | | `src/launchdarkly_ai_server/__init__.py` | Public barrel โ€” the only surface handler packages import from | --- @@ -78,7 +78,13 @@ from launchdarkly_ai_server import execute_and_track, execute_and_stream, wrap_t # Entry points from launchdarkly_ai_server import config, graph, resolve_graph -from launchdarkly_ai_server.experimental.evaluations import init_evaluations + +# Evaluations from Code +from launchdarkly_ai_server import ( + init_evaluations, EvaluationsModule, AIConfig, GenerationConfig, GenerationMode, + GenerationOverrides, DatasetRow, EvalTool, Criterion, Judge, Scorer, + EvalRunResult, RunSummary, EvaluationsError, +) # Agent Skills from launchdarkly_ai_server import ( @@ -91,12 +97,13 @@ from launchdarkly_ai_server import ( ) ``` -An experimental feature (today only Evaluations from Code) is exported only from its module -under `launchdarkly_ai_server.experimental`, and its names may change in a minor release. No -core public type, function or `init_client` option may name it. Agent Skills is core and is -exported from the package root; `shutdown` still reaches its state only through an internal -hook (it clears the skill store), and an error there is logged, never raised into -`shutdown`. +An experimental feature is exported only from its module under +`launchdarkly_ai_server.experimental`, and its names may change in a minor release. No feature +is experimental now. No core public type, function or `init_client` option may name an +experimental feature. Agent Skills and Evaluations from Code are core and are exported from +the package root. `lifecycle` imports no Evaluations from Code module. For Agent Skills, +`shutdown` still reaches its state only through an internal hook (it clears the skill store), +and an error there is logged, never raised into `shutdown`. `MAX_SKILL_CONTENT_BYTES` and `SKILL_OBJECT_KIND` are deliberately **not** exported; both stay internal to `skills_core`: @@ -163,7 +170,7 @@ Handlers may return any of these โ€” the client normalizes them before emitting ## SDK-run evaluations -Evaluations from Code is experimental. Its public names are exported from `launchdarkly_ai_server.experimental.evaluations` and pinned by `TestPackageExports` in `tests/test_evaluations.py`; none may be added to the package root. `init_evaluations` reports the helper `experimental.initEvaluations`. +Evaluations from Code is core. Its public names are exported from the package root and pinned by `TestPackageExports` in `tests/test_evaluations.py` and by `tests/test_public_surface.py`. `init_evaluations` reports the helper `client.initEvaluations`. `init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host (`api_base_uri`, then `LD_API_BASE_URI`; a blank value at either level counts as unset). Do not reuse `LD_BASE_URI`, or name the option `base_uri`: that host configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. An event transport is resolved in `init_evaluations()`, which raises before any network I/O when it finds neither an SDK key (`sdk_key` or `LD_SDK_KEY`) nor an already-initialized event-capable client: generation events are the only ingest path for row results, so a run without a transport could never complete. The lifecycle module's bring-your-own-client path (`init_client(client=...)`) therefore satisfies the check on its own, and `run()` reuses that singleton through `_resolve_client`; `run()` raises if the client disappears before it emits. Both polling arguments reject NaN, which would otherwise never compare past a deadline and hang the run. The harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. diff --git a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt deleted file mode 100644 index abc72168..00000000 --- a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt +++ /dev/null @@ -1,83 +0,0 @@ -# Public API of launchdarkly_ai_server.experimental.evaluations. Generated by `make api-report`; do not edit. -AIConfig [class, @dataclass(frozen=True)] - __init__(key: str, variation: str) - key: str - variation: str -Criterion = Judge | Scorer [type] -DatasetRow [class, @dataclass] - __init__(row_index: int, input: str | None = None, expected_output: str | None = None, variables: dict[str, Any] = dict(), metadata: dict[str, Any] | None = None) - expected_output: str | None = None - input: str | None = None - metadata: dict[str, Any] | None = None - row_index: int - variables: dict[str, Any] = field(default_factory=dict) -EvalRunResult [class, @dataclass] - __init__(passed: bool, url: str, run_id: str, summary: RunSummary) - passed: bool - run_id: str - summary: RunSummary - url: str -EvalTool [class, @dataclass(frozen=True)] - __init__(key: str, implementation: ToolImplementation, schema: dict[str, Any], description: str = '') - description: str = '' - implementation: ToolImplementation - key: str - project_key: str | None = field(default=None, init=False) - schema: dict[str, Any] - source: Literal['library', 'inline'] = field(default='inline', init=False) - to_create_wire() -> dict[str, Any] - version: int | None = field(default=None, init=False) -EvaluationsError(Exception) [class] -EvaluationsModule [class] - __init__(api_client: LDApiClient, project_key: str, sdk_key: str | None, ui_base_uri: str = DEFAULT_UI_BASE_URI) - project_key: str [property] - run(*, key: str, dataset: str | Sequence[InlineDatasetRow], handler: EvalHandler | None = None, handlers: Sequence[EvalHandler] | None = None, generation: GenerationConfig | GenerationOverrides | None = None, ai_config: AIConfig | None = None, tools: Sequence[EvalTool] | None = None, criteria: list[Criterion] | None = None, concurrency: int = 10, poll_interval_seconds: float | None = None, poll_timeout_seconds: float | None = None) -> EvalRunResult [async] - sdk_key: str | None [property] - tools: ToolsClient [property] - ui_base_uri: str [property] -GenerationConfig(TypedDict, total=False) [class] - instructions: str - messages: list[dict[str, Any]] - mode: Required[GenerationMode] - model: Required[str] - output_format: dict[str, Any] - parameters: dict[str, Any] - prompt_snippets: dict[str, str] - provider: Required[str] -GenerationMode = Literal['completion', 'agent'] -GenerationOverrides(TypedDict, total=False) [class] - instructions: str - messages: list[dict[str, Any]] - model: str - output_format: dict[str, Any] - parameters: dict[str, Any] - prompt_snippets: dict[str, str] - provider: str -init_evaluations(project_key: str | None = None, api_key: str | None = None, sdk_key: str | None = None, api_base_uri: str | None = None, ui_base_uri: str | None = None, transport: Transport = urllib_transport) -> EvaluationsModule -Judge [class, @dataclass(frozen=True)] - __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD, ground_truth_context: str | None = None) - __post_init__() -> None - criterion_type: str [property] - ground_truth_context: str | None = None - key: str - pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD - threshold: float | None = DEFAULT_JUDGE_THRESHOLD - to_criteria_wire() -> dict[str, Any] -RunSummary [class, @dataclass] - __init__(total_rows: int = 0, passed_rows: int = 0, failed_rows: int = 0, error_rows: int = 0, pending_rows: int = 0) - error_rows: int = 0 - failed_rows: int = 0 - from_wire(data: Mapping[str, Any] | None) -> RunSummary [classmethod] - passed_rows: int = 0 - pending_rows: int = 0 - total_rows: int = 0 -Scorer [class, @dataclass(frozen=True)] - __init__(name: str, fn: ScorerFn, threshold: float | None = 1.0, pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD, success_direction: SuccessDirection = 'higher_is_better') - __post_init__() -> None - criterion_type: str [property] - fn: ScorerFn - name: str - pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD - success_direction: SuccessDirection = 'higher_is_better' - threshold: float | None = 1.0 - to_criteria_wire() -> dict[str, Any] diff --git a/packages/client/api-report/launchdarkly_ai_server.txt b/packages/client/api-report/launchdarkly_ai_server.txt index 4ed2c00e..51183d9f 100644 --- a/packages/client/api-report/launchdarkly_ai_server.txt +++ b/packages/client/api-report/launchdarkly_ai_server.txt @@ -1,5 +1,9 @@ # Public API of launchdarkly_ai_server. Generated by `make api-report`; do not edit. add_cached_tokens_to_input(raw_usage: dict[str, Any]) -> SpanUsage +AIConfig [class, @dataclass(frozen=True)] + __init__(key: str, variation: str) + key: str + variation: str AiConfigRep = dict[str, Any] all_skills() -> list[Skill] [async] any_multimodal(turns: list[dict[str, Any]]) -> bool @@ -22,14 +26,64 @@ ConversationIdSpanProcessor(_SpanProcessorBase) [class] shutdown() -> None create_handler(provides_for: tuple[str, Literal['agent', 'messages']], fn: _HandlerFn, stream_fn: _StreamFn | None = None, capture_content: bool | None = None) -> ProviderHandler create_run_usage() -> RunUsage +Criterion = Judge | Scorer [type] +DatasetRow [class, @dataclass] + __init__(row_index: int, input: str | None = None, expected_output: str | None = None, variables: dict[str, Any] = dict(), metadata: dict[str, Any] | None = None) + expected_output: str | None = None + input: str | None = None + metadata: dict[str, Any] | None = None + row_index: int + variables: dict[str, Any] = field(default_factory=dict) end_span_once(span: Any, tracker: set[int], abandoned: bool = False, cancelled: bool = False) -> None end_unfinished_spans(*spans: Any) -> None +EvalRunResult [class, @dataclass] + __init__(passed: bool, url: str, run_id: str, summary: RunSummary) + passed: bool + run_id: str + summary: RunSummary + url: str +EvalTool [class, @dataclass(frozen=True)] + __init__(key: str, implementation: ToolImplementation, schema: dict[str, Any], description: str = '') + description: str = '' + implementation: ToolImplementation + key: str + project_key: str | None = field(default=None, init=False) + schema: dict[str, Any] + source: Literal['library', 'inline'] = field(default='inline', init=False) + to_create_wire() -> dict[str, Any] + version: int | None = field(default=None, init=False) +EvaluationsError(Exception) [class] +EvaluationsModule [class] + __init__(api_client: LDApiClient, project_key: str, sdk_key: str | None, ui_base_uri: str = DEFAULT_UI_BASE_URI) + project_key: str [property] + run(*, key: str, dataset: str | Sequence[InlineDatasetRow], handler: EvalHandler | None = None, handlers: Sequence[EvalHandler] | None = None, generation: GenerationConfig | GenerationOverrides | None = None, ai_config: AIConfig | None = None, tools: Sequence[EvalTool] | None = None, criteria: list[Criterion] | None = None, concurrency: int = 10, poll_interval_seconds: float | None = None, poll_timeout_seconds: float | None = None) -> EvalRunResult [async] + sdk_key: str | None [property] + tools: ToolsClient [property] + ui_base_uri: str [property] execute_and_stream(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None) -> AsyncGenerator[ExecuteStreamEvent, None] [async] execute_and_track(*, config_key: str, config: AiConfigRep, meta: VariationMeta, user_context: LDContext, handler: ProviderHandler, user_input: str | None = None, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, variables: dict[str, Any] | None = None, graph_key: str | None = None, history: list[dict[str, Any]] | None = None) -> dict[str, Any] [async] ExecuteStreamDoneEvent = dict[str, Any] ExecuteStreamEvent = dict[str, Any] extract_variation(config_key: str, context: dict[str, Any]) -> dict[str, Any] [async] FDv2SkillStore = LaunchDarklySkillStore +GenerationConfig(TypedDict, total=False) [class] + instructions: str + messages: list[dict[str, Any]] + mode: Required[GenerationMode] + model: Required[str] + output_format: dict[str, Any] + parameters: dict[str, Any] + prompt_snippets: dict[str, str] + provider: Required[str] +GenerationMode = Literal['completion', 'agent'] +GenerationOverrides(TypedDict, total=False) [class] + instructions: str + messages: list[dict[str, Any]] + model: str + output_format: dict[str, Any] + parameters: dict[str, Any] + prompt_snippets: dict[str, str] + provider: str get_client() -> Any get_skill(key: str, *, version: int | None = None) -> Skill | None [async] get_skill_result(key: str, *, version: int | None = None) -> SkillOutcome [async] @@ -77,6 +131,7 @@ HandlerStreamEvent [class] has_multimodal_content(content: MessageContent) -> bool image_block_to_url(block: dict[str, Any]) -> str init_client(options: InitClientOptions | None = None, client: Any = None) -> Any [async] +init_evaluations(project_key: str | None = None, api_key: str | None = None, sdk_key: str | None = None, api_base_uri: str | None = None, ui_base_uri: str | None = None, transport: Transport = urllib_transport) -> EvaluationsModule InitClientOptions = dict[str, Any] InMemorySkillStore [class] __init__(objects: dict[str, dict[str, Any]] | None = None) @@ -92,6 +147,15 @@ InputTokenDetails [class, @dataclass] uncached: int = 0 inspect_config(config_key: str, context: dict[str, Any]) -> dict[str, Any] [async] is_content_blocks(content: MessageContent) -> bool +Judge [class, @dataclass(frozen=True)] + __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD, ground_truth_context: str | None = None) + __post_init__() -> None + criterion_type: str [property] + ground_truth_context: str | None = None + key: str + pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD + threshold: float | None = DEFAULT_JUDGE_THRESHOLD + to_criteria_wire() -> dict[str, Any] JudgeResult [class, @dataclass] __init__(usage: UsageDict, response: str, score: float, event_key: str | None = None) event_key: str | None = None @@ -219,11 +283,29 @@ resolve_handlers(registry: Registry | None, local_handlers: list[ProviderHandler resolve_tools(registry: Registry | None, local_tools: dict[str, Callable[..., Any] | NativeTool] | None) -> dict[str, Callable[..., Any] | NativeTool] | None run_judge(task: JudgeTask, handlers: list[ProviderHandler]) -> JudgeRunResult | None [async] run_judges(*, config: AiConfigRep, user_context: LDContext, handler: ProviderHandler, handlers: list[ProviderHandler] | None = None, user_input: str | None, llm_response: str, base_track_data: TrackData, tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, graph_key: str | None = None, trajectory: str = '') -> dict[str, JudgeResult] [async] +RunSummary [class, @dataclass] + __init__(total_rows: int = 0, passed_rows: int = 0, failed_rows: int = 0, error_rows: int = 0, pending_rows: int = 0) + error_rows: int = 0 + failed_rows: int = 0 + from_wire(data: Mapping[str, Any] | None) -> RunSummary [classmethod] + passed_rows: int = 0 + pending_rows: int = 0 + total_rows: int = 0 RunUsage [class, @dataclass] __init__(total: SpanUsage = SpanUsage(), _turns: int = 0) add(turn: SpanUsage | None) -> None reported: bool [property] total: SpanUsage = field(default_factory=SpanUsage) +Scorer [class, @dataclass(frozen=True)] + __init__(name: str, fn: ScorerFn, threshold: float | None = 1.0, pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD, success_direction: SuccessDirection = 'higher_is_better') + __post_init__() -> None + criterion_type: str [property] + fn: ScorerFn + name: str + pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD + success_direction: SuccessDirection = 'higher_is_better' + threshold: float | None = 1.0 + to_criteria_wire() -> dict[str, Any] SDK_INFO_CONTEXT: LDContext = {'kind': 'ld_ai', 'key': 'ld-internal-tracking', 'anonymous': True} SDK_INFO_EVENT = '$ld:ai:sdk:info' SDKOptions [class, @dataclass(frozen=True)] diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index b170c1cc..18e86e9e 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -27,6 +27,22 @@ conversation_id, set_conversation_id_if_absent, ) +from .evaluations import ( + AIConfig, + Criterion, + DatasetRow, + EvalRunResult, + EvalTool, + EvaluationsError, + EvaluationsModule, + GenerationConfig, + GenerationMode, + GenerationOverrides, + Judge, + RunSummary, + Scorer, + init_evaluations, +) from .graph import GraphInstance, graph, resolve_graph from .graph import _graph as _graph from .history import ( @@ -303,6 +319,24 @@ "SKILL_FILENAME", "MANIFEST_FILENAME", "MANIFEST_VERSION", + # Evaluations from Code: entry point + "init_evaluations", + "EvaluationsModule", + # Evaluations from Code: run() arguments + "AIConfig", + "GenerationConfig", + "GenerationMode", + "GenerationOverrides", + "DatasetRow", + "EvalTool", + # Evaluations from Code: criteria + "Criterion", + "Judge", + "Scorer", + # Evaluations from Code: results and errors + "EvalRunResult", + "RunSummary", + "EvaluationsError", ] register_ai_sdk_package("launchdarkly-ai-server", __version__) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index ec6e88a5..3afbc446 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -892,7 +892,7 @@ def init_evaluations( """ from ..sdk_usage import report_usage - report_usage("experimental.initEvaluations") + report_usage("client.initEvaluations") resolved_project_key = (project_key or "").strip() or _env("LD_PROJECT_KEY") if not resolved_project_key: raise EvaluationsError( diff --git a/packages/client/src/launchdarkly_ai_server/experimental/__init__.py b/packages/client/src/launchdarkly_ai_server/experimental/__init__.py index 1d2e1e28..5fdaa0fb 100644 --- a/packages/client/src/launchdarkly_ai_server/experimental/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/experimental/__init__.py @@ -1,7 +1,7 @@ """ Experimental features of the LaunchDarkly AI SDK. -Each feature is a submodule, for example ``launchdarkly_ai_server.experimental.evaluations``. +Each feature is a submodule of this package. No feature is experimental now. Names imported from here may change in a minor release, so review the changelog when you upgrade. Names from the package root change only in a major release. """ diff --git a/packages/client/src/launchdarkly_ai_server/experimental/evaluations.py b/packages/client/src/launchdarkly_ai_server/experimental/evaluations.py deleted file mode 100644 index 593f90e8..00000000 --- a/packages/client/src/launchdarkly_ai_server/experimental/evaluations.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Evaluations from Code (experimental). - -Create a harness with ``init_evaluations``, then call ``run()`` on it to generate an -output for every dataset row and score it against LaunchDarkly judges or local -scorers. These names may change in a minor release, so review the changelog when -you upgrade. -""" - -from ..evaluations.api import EvaluationsError -from ..evaluations.criteria import Criterion, Judge, Scorer -from ..evaluations.module import EvaluationsModule, init_evaluations -from ..evaluations.tools import EvalTool -from ..evaluations.types import ( - AIConfig, - DatasetRow, - EvalRunResult, - GenerationConfig, - GenerationMode, - GenerationOverrides, - RunSummary, -) - -__all__ = [ # noqa: RUF022 - # entry point - "init_evaluations", - "EvaluationsModule", - # run() arguments - "AIConfig", - "GenerationConfig", - "GenerationMode", - "GenerationOverrides", - "DatasetRow", - "EvalTool", - # criteria - "Criterion", - "Judge", - "Scorer", - # results and errors - "EvalRunResult", - "RunSummary", - "EvaluationsError", -] diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index 069f1fe9..c08eaa7a 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib import json from collections.abc import Iterator from typing import Any @@ -8,18 +9,18 @@ import pytest import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_server import ( + EvalRunResult, + EvaluationsError, + RunSummary, + init_evaluations, +) from launchdarkly_ai_server.evaluations import ( DEFAULT_BASE_URI, LDApiError, Usage, ) from launchdarkly_ai_server.evaluations.api import HttpResponse, LDApiClient -from launchdarkly_ai_server.experimental.evaluations import ( - EvalRunResult, - EvaluationsError, - RunSummary, - init_evaluations, -) @pytest.fixture(autouse=True) @@ -595,7 +596,7 @@ def test_missing_project_key_raises_before_network_io(monkeypatch) -> None: init_evaluations(transport=failing_transport) -EXPERIMENTAL_EVALUATIONS_SURFACE = frozenset( +EVALUATIONS_SURFACE = frozenset( { "init_evaluations", "EvaluationsModule", @@ -617,29 +618,22 @@ def test_missing_project_key_raises_before_network_io(monkeypatch) -> None: class TestPackageExports: """ - Evaluations from Code is experimental: every public name is - exported from ``launchdarkly_ai_server.experimental.evaluations`` and none - from the package root. + Evaluations from Code is core: every public name is exported from the + package root, and ``launchdarkly_ai_server.experimental.evaluations`` no + longer exists. """ - def test_experimental_surface_is_exactly_the_documented_set(self) -> None: - from launchdarkly_ai_server.experimental import evaluations as experimental - - assert set(experimental.__all__) == EXPERIMENTAL_EVALUATIONS_SURFACE - assert len(experimental.__all__) == len(EXPERIMENTAL_EVALUATIONS_SURFACE) - for name in EXPERIMENTAL_EVALUATIONS_SURFACE: - assert hasattr(experimental, name), name - - def test_experimental_module_defines_no_other_public_name(self) -> None: - from launchdarkly_ai_server.experimental import evaluations as experimental + def test_every_evaluations_name_is_exported_from_the_package_root(self) -> None: + import launchdarkly_ai_server as package - public = {name for name in dir(experimental) if not name.startswith("_")} - assert public == EXPERIMENTAL_EVALUATIONS_SURFACE + assert EVALUATIONS_SURFACE <= set(package.__all__) + for name in EVALUATIONS_SURFACE: + assert hasattr(package, name), name - def test_no_evaluations_name_is_exported_from_the_package_root(self) -> None: - import launchdarkly_ai_server as package - import launchdarkly_ai_server.experimental.evaluations + def test_the_experimental_evaluations_module_is_gone(self) -> None: + import launchdarkly_ai_server.experimental as experimental - for name in EXPERIMENTAL_EVALUATIONS_SURFACE: - assert name not in package.__all__, name - assert not hasattr(package, name), name + with pytest.raises(ImportError): + importlib.import_module("launchdarkly_ai_server.experimental.evaluations") + for name in EVALUATIONS_SURFACE: + assert not hasattr(experimental, name), name diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 1a82c850..91cd8a07 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -11,17 +11,18 @@ import pytest -from launchdarkly_ai_server import NativeTool, create_handler -from launchdarkly_ai_server.evaluations.api import HttpResponse -from launchdarkly_ai_server.experimental.evaluations import ( +from launchdarkly_ai_server import ( AIConfig, DatasetRow, EvalTool, EvaluationsError, Judge, + NativeTool, Scorer, + create_handler, init_evaluations, ) +from launchdarkly_ai_server.evaluations.api import HttpResponse @pytest.fixture(autouse=True) diff --git a/packages/client/tests/test_evaluations_spans.py b/packages/client/tests/test_evaluations_spans.py index cefd9bba..a1a873b9 100644 --- a/packages/client/tests/test_evaluations_spans.py +++ b/packages/client/tests/test_evaluations_spans.py @@ -20,7 +20,14 @@ from opentelemetry.trace import StatusCode from opentelemetry.util._once import Once -from launchdarkly_ai_server import SDKOptions, configure, create_handler +from launchdarkly_ai_server import ( + EvalTool, + Judge, + SDKOptions, + configure, + create_handler, + init_evaluations, +) from launchdarkly_ai_server.conversation import ( GEN_AI_CONVERSATION_ID, ConversationIdSpanProcessor, @@ -34,11 +41,6 @@ record_tool_calls, start_row_span, ) -from launchdarkly_ai_server.experimental.evaluations import ( - EvalTool, - Judge, - init_evaluations, -) from launchdarkly_ai_server.lifecycle import ( _reset_for_testing, effective_capture_content, diff --git a/packages/client/tests/test_lifecycle.py b/packages/client/tests/test_lifecycle.py index 187ca9c5..fe67a597 100644 --- a/packages/client/tests/test_lifecycle.py +++ b/packages/client/tests/test_lifecycle.py @@ -680,8 +680,8 @@ def failing() -> None: assert "Could not clear the Failing Feature state" in caplog.messages later.assert_called_once() # one failing hook does not skip the rest - def test_lifecycle_imports_no_skills_module(self) -> None: - """agents.md: lifecycle reaches Agent Skills only through the shutdown hooks.""" + @staticmethod + def _lifecycle_imports() -> set[str]: tree = ast.parse(inspect.getsource(lifecycle_module)) imported: set[str] = set() for node in ast.walk(tree): @@ -691,20 +691,34 @@ def test_lifecycle_imports_no_skills_module(self) -> None: imported.add(node.module or "") # `from . import skills_core` names the module in the alias, not in `module`. imported.update(alias.name for alias in node.names) + return imported + + def test_lifecycle_imports_no_skills_module(self) -> None: + """agents.md: lifecycle reaches Agent Skills only through the shutdown hooks.""" skills = sorted( - name for name in imported if name.rsplit(".", 1)[-1].startswith("skills") + name + for name in self._lifecycle_imports() + if name.rsplit(".", 1)[-1].startswith("skills") ) assert skills == [] + def test_lifecycle_imports_no_evaluations_module(self) -> None: + """Lifecycle imports no Evaluations from Code module.""" + evaluations = sorted( + name + for name in self._lifecycle_imports() + if "evaluations" in name.split(".") + ) + assert evaluations == [] + def test_core_does_not_import_experimental_code(self) -> None: """Importing and shutting down core loads no experimental module.""" script = ( "import asyncio, sys\n" "import launchdarkly_ai_server\n" "asyncio.run(launchdarkly_ai_server.shutdown())\n" - "loaded = sorted(m for m in sys.modules if m.startswith(" - "('launchdarkly_ai_server.evaluations'," - " 'launchdarkly_ai_server.experimental')))\n" + "loaded = sorted(m for m in sys.modules" + " if m.startswith('launchdarkly_ai_server.experimental'))\n" "print(','.join(loaded))\n" ) result = subprocess.run( diff --git a/packages/client/tests/test_public_surface.py b/packages/client/tests/test_public_surface.py index bf106ef8..1c6c6607 100644 --- a/packages/client/tests/test_public_surface.py +++ b/packages/client/tests/test_public_surface.py @@ -144,6 +144,21 @@ "SKILL_FILENAME", "MANIFEST_FILENAME", "MANIFEST_VERSION", + # Evaluations from Code + "init_evaluations", + "EvaluationsModule", + "AIConfig", + "GenerationConfig", + "GenerationMode", + "GenerationOverrides", + "DatasetRow", + "EvalTool", + "Criterion", + "Judge", + "Scorer", + "EvalRunResult", + "RunSummary", + "EvaluationsError", } ) diff --git a/packages/client/tests/test_sdk_usage.py b/packages/client/tests/test_sdk_usage.py index 73f0e33a..092d7047 100644 --- a/packages/client/tests/test_sdk_usage.py +++ b/packages/client/tests/test_sdk_usage.py @@ -26,6 +26,7 @@ get_skills, graph, init_client, + init_evaluations, inspect_config, resolve_graph, run_judge, @@ -35,7 +36,6 @@ watch_skills, write_skills, ) -from launchdarkly_ai_server.experimental.evaluations import init_evaluations CONTEXT = {"kind": "user", "key": "user-1"} HELPER = "client.buildJudgeTasks" @@ -106,7 +106,7 @@ def _expect_usage(client: MagicMock, helper: str) -> None: "helperPackageName": "launchdarkly-ai-server", "helperPackageVersion": __version__, } - # A client.* or experimental.* helper belongs to the core package. + # A client.* helper belongs to the core package. payload = calls[0].args[2] assert payload["helperPackageName"] == payload["aiSdkName"] assert payload["helperPackageVersion"] == payload["aiSdkVersion"] @@ -166,7 +166,7 @@ async def _fn(*_args: Any, **_kwargs: Any) -> dict[str, Any]: create_handler(("OpenAI", "messages"), _fn) return None - assert helper == "experimental.initEvaluations", helper + assert helper == "client.initEvaluations", helper try: init_evaluations() except Exception: @@ -193,7 +193,7 @@ async def client() -> Any: "client.runJudge", "client.buildJudgeTasks", "client.createHandler", - "experimental.initEvaluations", + "client.initEvaluations", ], ) async def test_helper_sends_one_usage_event(client: MagicMock, helper: str) -> None: diff --git a/release-please-config.json b/release-please-config.json index 9c94b155..79ae254a 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -158,9 +158,6 @@ } }, "changelog-sections": [ - { "type": "feat", "scope": "evaluations", "section": "Experimental" }, - { "type": "fix", "scope": "evaluations", "section": "Experimental" }, - { "type": "perf", "scope": "evaluations", "section": "Experimental" }, { "type": "feat", "section": "Features" }, { "type": "fix", "section": "Bug Fixes" }, { "type": "perf", "section": "Performance Improvements" },