diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 30f55777..cb48b3a2 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.14.0", + "version": "0.15.0", "keywords": [ "agentops", "evaluation", diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 30f55777..cb48b3a2 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "agentops-accelerator", "source": "../../plugins/agentops", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Toolkit and Microsoft Foundry agents.", - "version": "0.14.0", + "version": "0.15.0", "keywords": [ "agentops", "evaluation", diff --git a/CHANGELOG.md b/CHANGELOG.md index 01b7eba7..27232c85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,40 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +## [0.15.0] - 2026-09-06 + +### Added +- **`execution: azd` supports the current `azd ai eval` command surface.** + AgentOps now discovers `evals/azure.eval.yaml` alongside the legacy + `eval.yaml`, classifies the recipe by content, and delegates to the matching + azd surface: `azd ai eval` via the `azure.ai.evaluations` extension (azd + 1.27.1+), or the existing `azd ai agent eval` via `azure.ai.agents`. The + evaluation is created, started, polled to a terminal state, and read back + per-sample, then normalized into the same `results.json` and `report.md` + contract as every other execution mode. Threshold binding, fail-closed + behavior, baseline comparison, and the `0`/`2`/`1` exit codes are unchanged. + Raw azd output is retained for successful and failed runs alike. + + The current surface fills `results.json` rows with one entry per sample, + including failed and errored samples, and computes aggregate metrics as the + mean of the per-sample scores, because the azd run object exposes only counts. + Threshold keys that name a metric the recipe cannot produce are now rejected + *before* the evaluation is submitted, so a typo never consumes a cloud run. + + `azure.ai.evaluations` is in preview and is not yet published to the default + azd extension registry. The new surface is strictly opt-in: it activates only + when a current-surface recipe exists, and `agentops eval init` continues to + generate a legacy recipe while the extension is unavailable, so existing + workspaces and fresh clones are unaffected. + +### Changed +- **azd extension detection now reads structured output.** Availability is + determined from `azd extension list --installed -o json` and matched on + extension id, falling back to the previous text scan only when the structured + form is unsupported. The human-readable table also lists uninstalled registry + entries, so scanning it could report an extension as available before it was + actually installed. + ## [0.14.0] - 2026-09-01 ### Added diff --git a/docs/evaluation.md b/docs/evaluation.md index 02eebd47..6684e246 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -66,7 +66,7 @@ separate from the Evaluations page. !!! note "The azd dataset remains recipe-owned" When `execution: azd` is selected, azd continues to read the dataset declared - in `eval.yaml`. AgentOps does not rewrite that external recipe from the + in the azd recipe. AgentOps does not rewrite that external recipe from the `dataset` value in `agentops.yaml`. ```mermaid @@ -284,7 +284,9 @@ eval assets. The `execution:` field decides where the evaluation actually runs. Local is the default and works for every target. Cloud runs a Foundry agent server-side. The -azd recipe path delegates to an existing `azd ai agent eval` flow. +azd recipe path delegates to an azd evaluation flow — either the current +`azd ai eval` surface or the legacy `azd ai agent eval` one, chosen by the +recipe. See [Delegating to azd](#delegating-to-azd) below. | Target | Cloud (`execution: cloud`) | Local runner | Recommended default | |---|---|---|---| @@ -297,12 +299,77 @@ For prompt-agent CI pipelines that need a merge or deploy gate, prefer cloud eval. Foundry executes the managed evaluation and AgentOps enforces thresholds, baselines, Doctor readiness, and release evidence. -!!! info "Reusing an azd eval recipe" - If a Foundry project already uses the public-preview `azd ai agent eval` - recipe, set `execution: azd` and `eval_recipe: eval.yaml`. AgentOps - delegates execution to azd, normalizes the metrics, binds thresholds, writes - `results.json`, and fails closed for any threshold that has no emitted - metric. Rubric evaluator dimensions are treated as first-class metric names. +## Delegating to azd + +`execution: azd` hands the evaluation to the Azure Developer CLI and keeps the +AgentOps contract around it: normalized `results.json`, `report.md`, threshold +gating, baseline comparison, and the same exit codes. Foundry runs the +evaluation; AgentOps decides whether the release is ready. + +Two azd evaluation surfaces are supported. **The recipe decides which one is +used** — there is no extra setting. + +| Surface | Recipe location | azd commands | Extension | azd version | +|---|---|---|---|---| +| Current | `evals/azure.eval.yaml` | `azd ai eval` | `azure.ai.evaluations` | 1.27.1+ | +| Legacy | `eval.yaml`, or `src//eval.yaml` | `azd ai agent eval` | `azure.ai.agents` | as previously documented | + +Classification is by file content, not by filename: a recipe whose root has a +sequence-valued `evals:` key is current, while a mapping-valued `agent:` key or +a `dataset_reference:` key is legacy. + +!!! warning "`azure.ai.evaluations` is preview and not yet published" + As of this writing the extension exists only as an unmerged pull request + against `Azure/azure-dev`. It is absent from the default azd extension + registry, so `azd extension install azure.ai.evaluations` does not resolve + yet. Until it ships, the current surface is only usable where the extension + has been built and published into a local azd extension source. + + This affects nobody who does not opt in. The current surface activates only + when `evals/azure.eval.yaml` exists. A workspace without it behaves exactly + as it did before, and `agentops eval init` keeps generating a legacy recipe + while the extension is unavailable. + +### Discovery precedence + +1. `eval_recipe:` in `agentops.yaml`, when set, wins outright. +2. If recipes from both surfaces are discoverable, the current surface wins and + the run reports which recipe it skipped. +3. More than one recipe within a single surface is rejected as ambiguous; set + `eval_recipe:` to choose. +4. No recipe is a configuration error naming both supported locations. + +### Thresholds against an azd run + +Threshold keys bind to the metric names the recipe declares — builtin evaluator +references such as `builtin.task_adherence` (the bare `task_adherence` alias +works), evaluator labels, and rubric dimension ids. Binding uses narrow aliases +only; nothing is fuzzy-matched, because a wrong match here would create a +false-green gate. + +For the current surface, binding happens in two stages: + +| Condition | When | Result | Exit code | +|---|---|---|---| +| Threshold names a metric no evaluator declares | before any azd command runs | configuration error | `1` | +| Threshold is ambiguous across declared metrics | before any azd command runs | configuration error | `1` | +| Declared metric produced no score in the run | after the run | threshold recorded as failed | `2` | +| Bound threshold not satisfied | after the run | gate failure | `2` | + +Catching a typo before submission means a misconfigured threshold never consumes +a cloud evaluation. + +### What the current surface adds + +The current surface exposes per-sample output, so AgentOps populates +`results.json` rows with one entry per sample, including failed and errored +samples, and computes each aggregate metric as the mean of its per-sample +scores. The run object itself carries only counts. The legacy surface remains +aggregate-only. + +A run that produced zero samples, no decodable metrics, or a non-`completed` +terminal status never reports a pass. Raw azd output is retained in the run's +artifact directory for successful and failed runs alike. ## Input mapping diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 68ae6a7e..caa3dc82 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -150,7 +150,7 @@ flowchart TD RS --> F F --> G["Invoke target per row or submit Foundry cloud eval"] G --> H["Run/collect evaluator scores"] - E -->|azd| AZD["Call azd ai agent eval using eval.yaml"] + E -->|azd| AZD["Call azd (ai eval or ai agent eval) using the discovered recipe"] AZD --> H H --> I["Evaluate thresholds"] I --> J["Write results.json + report.md"] @@ -181,7 +181,31 @@ federated, workload, managed, or service-principal identity. The identity needs tokens, SAS, account keys, connection strings, query strings, fragments, and embedded credentials are unsupported. Storage firewall and private-endpoint connectivity remain runner responsibilities. `execution: azd` is unchanged and -continues to use the dataset declared by `eval.yaml`. +continues to use the dataset declared by the azd recipe. + +### The two azd evaluation surfaces + +`execution: azd` supports two azd command families. The discovered recipe +decides which one runs; there is no extra setting. + +| Recipe | azd commands | Extension | +|---|---|---| +| `evals/azure.eval.yaml` | `azd ai eval` | `azure.ai.evaluations` (preview, azd 1.27.1+) | +| `eval.yaml`, `src//eval.yaml` | `azd ai agent eval` | `azure.ai.agents` | + +Recipes are classified by content rather than filename, and when both surfaces +are discoverable the current one wins with the skipped recipe reported. + +For the current surface AgentOps reconciles the evaluation definition, submits +the run without waiting, polls it to a terminal state under its own timeout, and +then reads the run object plus every per-sample output item. Submitting without +waiting is deliberate: azd's blocking mode has an internal wait budget that, on +expiry, exits zero with an unfinished run and a differently shaped payload. + +AgentOps computes aggregate metrics itself for that surface, because the azd run +object exposes only counts, never scores. It also never passes a failure-gating +flag to azd — the release gate stays in AgentOps so a gate breach remains +distinguishable from an operational failure. ## POC-to-production readiness flow diff --git a/docs/tutorial-hosted-agent.md b/docs/tutorial-hosted-agent.md index d716f1cd..66983a34 100644 --- a/docs/tutorial-hosted-agent.md +++ b/docs/tutorial-hosted-agent.md @@ -274,6 +274,60 @@ That is expected. A hosted endpoint is evaluated with AgentOps local eval so the repo can invoke the endpoint, normalize results, apply thresholds, and keep a stable `results.json` contract, in sandbox and in CI alike. +### Optional: delegate the run to azd instead + +If your Foundry project already drives evaluations through the Azure Developer +CLI, you can keep that flow and still get the AgentOps gate around it. Set +`execution: azd` in `agentops.yaml` and commit an azd recipe. Which azd surface +runs is decided by the recipe, not by a setting: + +| Recipe you commit | azd commands | Extension | +|---|---|---| +| `evals/azure.eval.yaml` | `azd ai eval` | `azure.ai.evaluations` (preview, azd 1.27.1+) | +| `eval.yaml` | `azd ai agent eval` | `azure.ai.agents` | + +A minimal current-surface recipe: + +```yaml +# evals/azure.eval.yaml +datasets: + - name: smoke + file: ../.agentops/data/smoke.jsonl + +evals: + - name: hosted-agent-regression + dataset: smoke + evaluation_level: turn + evaluators: + - evaluator: builtin.task_adherence + initialization_parameters: + model: gpt-4o + target: + type: agent + name: +``` + +Thresholds in `agentops.yaml` bind to the metric names the recipe declares, so +`task_adherence: ">=4"` gates `builtin.task_adherence`. AgentOps checks that +binding *before* submitting, so a typo fails immediately with exit code `1` +instead of burning a cloud run. + +!!! warning "The current-surface extension is not published yet" + `azure.ai.evaluations` exists today only as an unmerged pull request against + `Azure/azure-dev`, so `azd extension install azure.ai.evaluations` does not + resolve from the default registry. To exercise this path now you must build + and publish the extension into a local azd extension source: + + ```powershell + azd extension install microsoft.azd.extensions + azd x build; azd x pack; azd x publish + azd extension install azure.ai.evaluations --source local + ``` + + Skip this section entirely if that is not set up. Nothing else in the + tutorial depends on it, and a workspace without `evals/azure.eval.yaml` is + completely unaffected. + ## 8. Run a local eval Replay the dataset against the sandbox endpoint and score it: diff --git a/plugins/agentops/package.json b/plugins/agentops/package.json index 0d28bdd1..de687bbb 100644 --- a/plugins/agentops/package.json +++ b/plugins/agentops/package.json @@ -2,7 +2,7 @@ "name": "agentops-accelerator", "displayName": "AgentOps Accelerator — Skills for GitHub Copilot", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.14.0", + "version": "0.15.0", "publisher": "AgentOpsAccelerator", "icon": "icon.png", "license": "MIT", diff --git a/plugins/agentops/plugin.json b/plugins/agentops/plugin.json index 138237e0..2887fa4c 100644 --- a/plugins/agentops/plugin.json +++ b/plugins/agentops/plugin.json @@ -1,7 +1,7 @@ { "name": "agentops-accelerator", "description": "Copilot agent skills for running standardized evaluation workflows with AgentOps Accelerator and Microsoft Foundry agents.", - "version": "0.14.0", + "version": "0.15.0", "author": { "name": "AgentOps Accelerator", "url": "https://github.com/Azure/agentops" diff --git a/requirements-docs.txt b/requirements-docs.txt index 18e74a23..d09d5c37 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -4,4 +4,4 @@ mkdocs-material-extensions==1.3.1 # Needed for dynamic release version in docs mkdocs-macros-plugin==1.5.0 -requests==2.33.0 +requests==2.34.2 diff --git a/specs/011-azd-ai-eval-surface/checklists/requirements.md b/specs/011-azd-ai-eval-surface/checklists/requirements.md new file mode 100644 index 00000000..4063f0d3 --- /dev/null +++ b/specs/011-azd-ai-eval-surface/checklists/requirements.md @@ -0,0 +1,44 @@ +# Specification Quality Checklist: Current azd AI Evaluation Surface Support + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-09-06 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- Iteration 1: One open [NEEDS CLARIFICATION] marker in Edge Cases, covering recipe resolution + when both a legacy and a current-surface recipe are discoverable and no explicit `eval_recipe` + is set. +- Iteration 2: Resolved. The user chose "prefer the current surface and report the choice". + Encoded as FR-004a (cross-surface precedence), FR-004b (same-surface ambiguity is still + rejected), SC-009 (deterministic, always-reported resolution), an Edge Cases entry, and an + Assumptions entry recording the rationale and the `eval_recipe` escape hatch. All checklist + items now pass. +- Command names, file paths, and configuration field names retained in the spec are existing + public contracts of this product, not implementation choices, and are required for the + requirements to be testable. diff --git a/specs/011-azd-ai-eval-surface/contracts/azd-eval-surface.md b/specs/011-azd-ai-eval-surface/contracts/azd-eval-surface.md new file mode 100644 index 00000000..97f45cf3 --- /dev/null +++ b/specs/011-azd-ai-eval-surface/contracts/azd-eval-surface.md @@ -0,0 +1,230 @@ +# Contract: azd Evaluation Surfaces + +This contract covers what AgentOps accepts from the user, what it invokes on the +Azure Developer CLI, and what it guarantees back. It does not restate the +`results.json` schema, which is unchanged; see [data-model.md](../data-model.md) +for the fields the current surface fills. + +## Configuration + +No new field is introduced. Both surfaces are driven by the existing keys. + +### Auto-discovered current surface + +```yaml +version: 1 +agent: "travel-agent:1" +dataset: .agentops/data/smoke.jsonl +execution: azd +thresholds: + task_adherence: ">=4" + accuracy: ">=3" +``` + +With `evals/azure.eval.yaml` present, this resolves to the current surface. The +`dataset` field remains required by the AgentOps schema and continues to drive +readiness analysis; the evaluation itself uses the dataset the recipe declares, +which is existing `execution: azd` behavior. + +### Explicit recipe + +```yaml +version: 1 +agent: "travel-agent:1" +dataset: .agentops/data/smoke.jsonl +execution: azd +eval_recipe: config/nightly/azure.eval.yaml +``` + +`eval_recipe` bypasses discovery for either surface. The surface is still +determined by the file's content. + +### Unchanged legacy surface + +```yaml +version: 1 +agent: "reservation-agent:3" +dataset: .agentops/data/smoke.jsonl +execution: azd +``` + +With `eval.yaml` at the workspace root or under `src//` and no +current-surface recipe, behavior is identical to previous releases. + +## Recipe discovery + +| Surface | Discovery locations | +|---|---| +| Current | `evals/azure.eval.yaml` | +| Legacy | `eval.yaml`, `eval.yml`, `src/*/eval.yaml`, `src/*/eval.yml` | + +Precedence: + +1. `eval_recipe`, when set, wins outright. +2. Candidates from both surfaces → the current surface is selected, and the run + reports the selected and skipped paths. +3. More than one candidate within one surface → rejected as ambiguous, listing + the candidates. +4. No candidate → configuration error naming both supported locations. + +## Recipe schema classification + +Classification is by content, not filename. + +| Observed at the document root | Surface | +|---|---| +| `evals` is a sequence | Current | +| `agent` is a mapping, or `dataset_reference` is present, and no `evals` sequence | Legacy | +| Neither | Configuration error | + +## Accepted current-surface recipe + +```yaml +datasets: + - name: support-agent-regression + file: ./datasets/support-agent-regression.jsonl + +evaluators: + - name: support-agent-quality + source: ./evaluators/support-agent-quality.json + - name: brevity + definition: + type: rubric + dimensions: + - id: length + weight: 1 + description: Answers the question without restating it. + +evals: + - name: support-agent-regression-eval + dataset: support-agent-regression + evaluation_level: turn + max_samples: 50 + evaluators: + - evaluator: builtin.task_adherence + initialization_parameters: + model: gpt-4o + - evaluator: support-agent-quality + version: "2" + data_mapping: + ground_truth: "{{item.expected}}" + target: + type: agent + name: support-agent +``` + +A trace-sourced evaluation replaces `dataset` with `source`: + +```yaml +evals: + - name: support-agent-trace-eval + source: + type: traces + agent_name: support-agent + lookback_hours: 24 + max_traces: 20 + evaluation_level: turn + evaluators: + - evaluator: builtin.task_adherence + initialization_parameters: + model: gpt-4o +``` + +Notes that a parser must respect: + +- The dataset key is `file`. `source` on a `datasets[]` entry is invalid. +- `dataset` and `source` on an `evals[]` entry are mutually exclusive. +- Unknown fields are preserved, not rejected. + +## Threshold binding + +Threshold keys in `agentops.yaml` bind to the metric names a recipe declares: + +| Threshold key | Binds to | +|---|---| +| `task_adherence` | `builtin.task_adherence`, via the existing `builtin.` alias rule | +| `builtin.task_adherence` | itself, exactly | +| `support-agent-quality` | an evaluator reference name or label | +| `accuracy` | a rubric dimension id | + +Binding uses the existing narrow alias rules. No fuzzy matching is added. + +| Condition | Stage | Outcome | Exit code | +|---|---|---|---| +| Threshold matches no declared metric | Pre-flight, before any azd call | Configuration error | `1` | +| Threshold matches more than one declared metric | Pre-flight, before any azd call | Configuration error listing candidates | `1` | +| Declared metric produced no score in the run | Post-run | Threshold recorded as failed | `2` | +| All thresholds bound and satisfied | Post-run | Pass | `0` | +| Bound threshold not satisfied | Post-run | Gate failure | `2` | + +Pre-flight binding applies to the current surface only. Legacy binding behavior +is unchanged. + +## Commands AgentOps invokes + +Every invocation is non-interactive and requests structured output. AgentOps +never passes a failure-gating flag; the release gate stays in AgentOps. + +| Step | Command | Purpose | +|---|---|---| +| Probe | `azd version` | Confirm azd is present | +| Probe | `azd extension list --installed -o json` | Confirm the required extension is installed, matching on extension id | +| Reconcile | `azd ai eval create --path -o json` | Idempotently register the declared datasets, evaluators, and evaluation | +| Submit | `azd ai eval run start --eval --path --no-wait -o json` | Start the run and capture identifiers | +| Poll | `azd ai eval run show --path -o json` | Observe status until terminal or timeout | +| Retrieve run | `azd ai eval run show --path -o json` | Final run object with counts and portal link | +| Retrieve samples | `azd ai eval run output list --all --output-file -o json` | Full per-sample output, unpaged | + +Constraints this encodes: + +- `--path` names the *directory* holding the recipe, not the file. +- `run start` takes no positional argument; the evaluation is named by `--eval`. +- `run output list` takes the run id positionally, which takes precedence over + the `--run` flag. +- `--all` and `--output-file` together defeat the default page bound; the output + file is read instead of stdout. +- The legacy surface continues to invoke `azd ai agent eval run` and + `azd ai agent eval show` exactly as before. Note that the legacy surface spells + its file flag `--out-file`, while the current surface spells it + `--output-file`. + +## Run status handling + +| Status | Treatment | +|---|---| +| `completed` | Terminal, success | +| `failed`, `error`, `canceled`, `cancelled` | Terminal, runtime error | +| Anything else | Non-terminal; keep polling until the AgentOps timeout | + +Comparison is case-insensitive. On timeout, AgentOps raises a runtime error that +includes the evaluation id, the run id, the last observed status, and the portal +link when known, so the operator can reattach in Foundry. + +## Environment requirements + +| Surface | azd | Extension | +|---|---|---| +| Current | `>= 1.27.1` | `azure.ai.evaluations` | +| Legacy | as previously documented | `azure.ai.agents` | + +When azd or the required extension is missing, AgentOps exits `1` with a message +naming the missing component and the install command. AgentOps never installs +tooling and never falls back to another execution engine. + +> The `azure.ai.evaluations` extension is preview and, as of this writing, is not +> yet present in the default azd extension registry. Until it ships, the current +> surface can only be exercised where the extension has been published into a +> local extension source. AgentOps treats its absence as a normal, clearly +> reported condition. + +## Guarantees + +- The evaluation executes exactly once per `agentops eval run`. Retrieving + results never re-executes it. +- Normalized `results.json` and `report.md` are produced with the same structure + for both surfaces. +- A run that produced zero samples, no decodable metrics, or a non-`completed` + terminal status never reports a pass. +- Raw azd output is retained in the run's artifact directory for successful and + failed runs alike. +- Existing legacy configurations continue to work with no edits. diff --git a/specs/011-azd-ai-eval-surface/data-model.md b/specs/011-azd-ai-eval-surface/data-model.md new file mode 100644 index 00000000..8513f3c0 --- /dev/null +++ b/specs/011-azd-ai-eval-surface/data-model.md @@ -0,0 +1,309 @@ +# Phase 1 Data Model: Current azd AI Evaluation Surface + +**Feature**: `specs/011-azd-ai-eval-surface` +**Date**: 2026-09-06 + +Entities are grouped by layer. Pure recipe entities live in `core/`; adapter +entities live in `pipeline/`; the normalized output entity is the existing +`RunResult` contract, which is extended additively. + +## Layer 1 — Recipe entities (`core/azd_eval.py`) + +### EvalSurface + +An enumeration identifying which azd command family and extension a recipe +requires. + +| Value | Extension | Command family | Recipe schema | +|---|---|---|---| +| `legacy` | `azure.ai.agents` | `azd ai agent eval` | flat mapping with `agent` and `dataset_reference` | +| `current` | `azure.ai.evaluations` | `azd ai eval` | `datasets` / `evaluators` / `evals` arrays | + +Every discovered recipe resolves to exactly one value. There is no `unknown` +value: a document that classifies as neither is rejected as a configuration +error, so downstream code never handles an indeterminate surface. + +### RecipeClassification + +Result of classifying a parsed YAML document. + +| Field | Type | Notes | +|---|---|---| +| `surface` | `EvalSurface` | Resolved surface | +| `path` | `Path` | Absolute path to the recipe file | + +**Classification rules**, applied in order against the parsed root mapping: + +1. Root `evals` is a sequence → `current`. +2. Root `agent` is a mapping, or root has `dataset_reference` → `legacy`. +3. Otherwise → configuration error naming the path and stating that the document + matches neither supported schema. + +Rule 1 precedes rule 2 so that a current-surface document is never misread. The +two shapes cannot both match, because the current schema forbids top-level keys +outside its three arrays. + +### RecipeResolution + +Outcome of discovery, carrying both the choice and what was passed over so the +choice can be reported rather than made silently. + +| Field | Type | Notes | +|---|---|---| +| `path` | `Path` | Selected recipe | +| `surface` | `EvalSurface` | Surface of the selected recipe | +| `skipped` | `tuple[Path, ...]` | Discoverable recipes not selected | +| `explicit` | `bool` | True when `eval_recipe` selected the path | + +**Discovery locations**: + +| Surface | Locations searched | +|---|---| +| `current` | `/evals/azure.eval.yaml` | +| `legacy` | `/eval.yaml`, `/eval.yml`, `/src/*/eval.yaml`, `/src/*/eval.yml` | + +**Resolution rules**: + +- An explicit `eval_recipe` selects that path directly; classification still + applies, and `skipped` is empty. +- Candidates from more than one surface → select the `current` candidate and + record the others in `skipped`. +- More than one candidate within a single surface → ambiguity error listing the + candidates and directing the user to set `eval_recipe`. This preserves existing + legacy behavior. +- No candidates → configuration error naming both supported locations. + +### CurrentEvalRecipe + +Tolerant model of `evals/azure.eval.yaml`. Unknown fields are preserved for +forward compatibility with the preview schema, mirroring how the existing legacy +recipe model is built. + +| Field | Type | Notes | +|---|---|---| +| `datasets` | `list[CurrentDatasetDecl]` | Optional | +| `evaluators` | `list[CurrentEvaluatorDecl]` | Optional; locally declared rubric evaluators | +| `evals` | `list[CurrentEval]` | Optional in schema; required in practice for a run | + +### CurrentDatasetDecl + +| Field | Type | Notes | +|---|---|---| +| `name` | `str` | Required | +| `file` | `Optional[str]` | Local `.jsonl` path. **Key is `file`, not `source`** | +| `version` | `Optional[str]` | Coerced to string, matching existing version handling | + +### CurrentEvaluatorDecl + +A locally declared evaluator, typically a weighted rubric. + +| Field | Type | Notes | +|---|---|---| +| `name` | `str` | Required; referenced by `CurrentEvaluatorRef.evaluator` | +| `source` | `Optional[str]` | Path to a rubric definition file | +| `definition` | `Optional[dict]` | Inline rubric, or a nested file reference | +| `version` | `Optional[str]` | Coerced to string | + +Rubric dimensions are read from `definition.dimensions` when the rubric is +inline, and from the referenced file when `source` is set. Each dimension +contributes a declared metric name from its `id`, falling back to its `name`, +reusing the existing dimension metric-name rule. + +### CurrentEval + +| Field | Type | Notes | +|---|---|---| +| `name` | `str` | Required; the value passed to `--eval` | +| `id` | `Optional[str]` | Pre-existing evaluation created outside this recipe | +| `description` | `Optional[str]` | | +| `dataset` | `Optional[str]` | Name of a `datasets[]` entry; mutually exclusive with `source` | +| `source` | `Optional[CurrentSourceDecl]` | Trace or response source; mutually exclusive with `dataset` | +| `evaluation_level` | `Optional[str]` | `turn` or `conversation` | +| `max_samples` | `Optional[int]` | Recipe-owned; AgentOps does not override it | +| `evaluators` | `list[CurrentEvaluatorRef]` | | +| `target` | `Optional[CurrentTarget]` | Omitted when grading responses already present in rows | + +### CurrentSourceDecl + +| Field | Type | Notes | +|---|---|---| +| `type` | `str` | `traces` or `responses` | +| `agent_name`, `agent_version` | `Optional[str]` | Trace sources | +| `lookback_hours`, `max_traces`, `max_turns` | `Optional[int]` | | +| `response_ids` | `list[str]` | Response sources | +| `start_time`, `end_time` | `Optional[str]` | Absolute window | + +Used for provenance in the normalized result's dataset description. A +trace-sourced evaluation has no local dataset file, so the recorded dataset path +describes the source instead. + +### CurrentEvaluatorRef + +| Field | Type | Notes | +|---|---|---| +| `evaluator` | `str` | Required. `builtin.` or a name from `evaluators[]` | +| `name` | `Optional[str]` | Label used when one evaluator is referenced twice | +| `version` | `Optional[str]` | Coerced to string | +| `initialization_parameters` | `dict` | Carries the judge `model` | +| `data_mapping` | `dict` | Binds evaluator inputs to dataset columns | + +### CurrentTarget + +| Field | Type | Notes | +|---|---|---| +| `type` | `str` | `agent` or `model` | +| `name` | `str` | Agent or deployment name | + +### Declared metric names + +The set of metric names a current-surface recipe can produce, used for +pre-flight threshold binding: + +- For each `CurrentEvaluatorRef`: its `name` when set, otherwise its `evaluator`. +- For each rubric dimension of each referenced local evaluator: the dimension's + metric name. + +These feed the existing threshold binding helper unchanged, so the established +narrow alias rules (including the `builtin.` prefix rule) apply identically to +both surfaces. + +## Layer 2 — Adapter entities (`pipeline/azd_eval_runner.py`) + +### CurrentEvalRun + +Everything captured from one delegated evaluation. + +| Field | Type | Notes | +|---|---|---| +| `recipe_path` | `Path` | Selected recipe | +| `eval_id` | `Optional[str]` | Evaluation identifier | +| `eval_name` | `Optional[str]` | Evaluation name from the recipe | +| `run_id` | `str` | Run identifier; required to retrieve output | +| `status` | `str` | Last observed run status | +| `run_payload` | `dict` | Raw run object | +| `output_items` | `list[dict]` | Raw per-sample items | +| `report_url` | `Optional[str]` | Portal link | +| `error_message` | `Optional[str]` | Non-empty run-level error message, if any | +| `stdout`, `stderr` | `str` | Concatenated command streams | +| `duration_seconds` | `float` | Wall time for the delegated sequence | + +### RunCounts + +Sample tallies read from the run object. + +| Field | Type | +|---|---| +| `total`, `passed`, `failed`, `errored`, `skipped` | `int` | + +Authoritative for sample totals and pass counts. Cross-checked against the number +of retrieved output items; a mismatch is recorded as a retrieval warning and the +smaller pass count is used, so an incomplete retrieval can never inflate the pass +rate. + +### SampleOutcome + +Per-sample status vocabulary: `passed`, `failed`, `errored`, `skipped`. + +### SampleScore + +One metric score for one sample. + +| Field | Type | Notes | +|---|---|---| +| `metric` | `str` | `metric` when present, otherwise `name` | +| `score` | `Optional[float]` | Tolerant decode; `None` when absent or undecodable | +| `passed` | `Optional[bool]` | Three-valued: pass, judged failure, or unjudged | +| `label` | `Optional[str]` | | +| `reason` | `Optional[str]` | Carried into the normalized row metric | + +**Invariants**: + +- `passed = None` is never collapsed to `False`. +- `score = None` is never coerced to `0.0`; the sample is excluded from that + metric's mean. +- A sample with an empty score list is an outcome of `failed`, not `passed`. + +### Aggregation rule + +For each distinct `metric` across all samples: + +```text +aggregate_metrics[metric] = mean(score for score in samples if score is not None) +``` + +A metric with no decodable scores is omitted from `aggregate_metrics`. Any +threshold bound to an omitted metric therefore fails closed at evaluation time. + +## Layer 3 — Normalized output (`core/results.py`) + +`RunResult` is unchanged structurally. The current-surface adapter fills existing +fields that the legacy adapter leaves empty, and adds one nested mapping inside +the existing free-form `config` field. + +| `RunResult` field | Current-surface content | +|---|---| +| `target` | Derived from the configured `agent`, as today | +| `dataset_path` | Recipe dataset file when present, otherwise a description of the declared source | +| `evaluators` | Declared evaluator reference names | +| `rows` | One `RowResult` per retrieved sample | +| `aggregate_metrics` | Computed means, per the aggregation rule | +| `thresholds` | Evaluated against `aggregate_metrics`, with declared-but-missing metrics recorded as failed | +| `summary` | Counts from `RunCounts`; `overall_passed` requires a `completed` status and a full threshold pass rate | +| `config.azd_evaluation` | Provenance block, extended for this surface | + +### RowResult mapping + +| `RowResult` field | Source | +|---|---| +| `row_index` | Position in the retrieved output | +| `input` | Best-effort extraction from the sample's data item | +| `expected` | Best-effort extraction from the sample's data item | +| `response` | Best-effort extraction from the sample's data item | +| `metrics` | One `RowMetric` per `SampleScore`, carrying `name`, `value`, `reason` | +| `error` | Set for `errored` samples, from the run-level error when no per-sample message exists | + +The surface has no per-sample error field; an execution failure appears as an +`errored` status, an empty score list, or an unjudged verdict. Text extraction is +best-effort and never affects the gate: metric scores are read from the score +list, not from the data item, so an extraction miss degrades readability only. +The raw item is always retained in the raw artifact. + +### `config.azd_evaluation` provenance block + +Existing keys are preserved. Keys marked new are additive **and are emitted by +the current surface only** — the legacy adapter's provenance block is left +byte-for-byte as it was, because FR-013 requires the legacy artifact contract to +be unchanged. A downstream consumer therefore reads an absent `surface` as +`legacy`. + +| Key | Notes | +|---|---| +| `recipe_path` | Existing | +| `run_id`, `eval_id`, `status`, `report_url` | Existing | +| `metric_binding`, `unused_metrics` | Existing | +| `surface` | **New, current surface only**: `current` | +| `extension` | **New, current surface only**: extension id used | +| `eval_name` | **New, current surface only** | +| `skipped_recipes` | **New, current surface only**: discoverable recipes not selected | +| `result_counts` | **New, current surface only**: total, passed, failed, errored, skipped | +| `missing_metrics` | **New, current surface only**: declared metrics the run did not emit | +| `retrieval_warnings` | **New, current surface only**: count mismatches or truncation notices | +| `error_message` | **New, current surface only**: run-level error message when non-empty | + +`config.result_granularity` becomes `row` for the current surface; it remains +`aggregate` for the legacy surface. + +## Raw artifacts + +Written to the run's output directory for successful and failed runs alike, +alongside the existing legacy artifact names. + +| File | Content | +|---|---| +| `azd_evaluation.json` | Raw run object | +| `azd_eval_output_items.json` | Raw per-sample items | +| `azd_stdout.log`, `azd_stderr.log` | Concatenated command streams | + +Per-sample items contain prompts and model responses. They stay inside the +results directory, which the generated workspace `.gitignore` already excludes +from version control. diff --git a/specs/011-azd-ai-eval-surface/plan.md b/specs/011-azd-ai-eval-surface/plan.md new file mode 100644 index 00000000..5cd8200e --- /dev/null +++ b/specs/011-azd-ai-eval-surface/plan.md @@ -0,0 +1,260 @@ +# Implementation Plan: Current azd AI Evaluation Surface Support + +**Branch**: `placerda-azd-ai-eval-surface` | **Date**: 2026-09-06 | **Spec**: [spec.md](spec.md) + +**Input**: Feature specification from `/specs/011-azd-ai-eval-surface/spec.md` + +## Summary + +Teach `execution: azd` to speak two azd evaluation surfaces instead of one. A +pure core module gains a second recipe schema and a content-based classifier, so +a discovered recipe declares which surface it belongs to. A new pipeline adapter +drives the current surface — reconcile the evaluation, submit the run +asynchronously, poll to a terminal state, then read the run object and its +per-sample output — and normalizes that output into the existing `RunResult` +contract. The existing legacy adapter is left untouched and continues to serve +legacy recipes. + +The current surface exposes no aggregate numeric metrics, only counts, so the new +adapter computes each metric as the mean of its per-sample scores and populates +per-sample rows. That is a richer result than the legacy adapter's +aggregate-only output, and it arrives through the same schema, so reporting, +threshold gating, baseline comparison, evidence, and exit codes need no +surface-specific branching. + +Discovery adds `evals/azure.eval.yaml`. When both surfaces are discoverable and +no explicit `eval_recipe` is set, the current surface wins and the choice is +reported. No new command, flag, or configuration field is introduced. + +## Technical Context + +**Language/Version**: Python 3.11+ + +**Primary Dependencies**: Pydantic v2 and `ruamel.yaml` for the second recipe +schema; the standard library `subprocess` for the azd boundary. No new +third-party dependency. No Azure SDK is involved — azd is the only integration +point and it is an external process. + +**External tooling**: Azure Developer CLI `>= 1.27.1` plus the +`azure.ai.evaluations` extension for the current surface; the existing +`azure.ai.agents` extension for the legacy surface. Neither is installed by +AgentOps. + +**Storage**: Recipe YAML read from the workspace; run artifacts written under the +existing timestamped results directory; a temporary file for the per-sample +output that azd writes directly. + +**Testing**: pytest with the azd subprocess boundary mocked, matching the +existing azd adapter tests. No azd installation, no Azure credentials, and no +network access are required to run the suite. + +**Target Platform**: Windows, Linux, and macOS workstations plus non-interactive +CI runners. + +**Project Type**: Python CLI and library. + +**Performance Goals**: Adapter overhead outside the cloud run itself stays +negligible relative to evaluation latency. Progress output during polling +appears at the existing heartbeat interval so a long run never looks hung. + +**Constraints**: No new command, flag, or required configuration field; the flat +`agentops.yaml` schema evolves additively only; the `RunResult` schema evolves +additively only; exit codes keep their existing meanings; legacy recipe behavior +is unchanged; the evaluation executes exactly once per invocation; the gate stays +in AgentOps and is never delegated to azd; unbound thresholds and missing metrics +fail closed. + +**Scale/Scope**: One recipe, one evaluation, and one run per `agentops eval run` +invocation. Per-sample output is retrieved in full rather than paged by the +caller. + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +| Gate | Pre-design assessment | Post-design assessment | +|---|---|---| +| Preserve public contracts | PASS: `execution: azd` and `eval_recipe` keep their meaning and gain a second supported recipe schema. No new command or flag. Exit codes unchanged. | PASS: `RunResult` gains only additive content under the existing `config` mapping and populates the already-present `rows` list. Legacy recipes resolve, execute, and normalize exactly as before. Cross-surface discovery precedence is deterministic and reported, never silent. | +| Enforce architectural boundaries | PASS: recipe schema, classification, and threshold binding are pure and belong in `core/`; subprocess execution and normalization belong in `pipeline/`; readiness reporting belongs in `services/`; the CLI is unchanged. | PASS: the new adapter module holds every azd command construction and every JSON field name, so the volatile external surface is contained in one place. `core/` performs no I/O. `pathlib.Path` throughout. | +| Isolate Azure runtime integration | PASS: no Azure SDK is used. The integration is an external process, and it is invoked lazily from within the execution path. | PASS: tests mock the subprocess boundary, so the suite runs with no azd, no credentials, and no network. Missing tooling produces an explicit, actionable error naming the extension, never a silent fallback to another engine. | +| Keep release evidence trustworthy | PASS: Foundry still owns evaluation execution; AgentOps owns the gate and the normalized evidence. Doctor and Cockpit are untouched and stay read-only. | PASS: the gate is never delegated to azd; a null verdict is never read as a pass; missing metrics and zero-sample runs cannot report a pass; raw azd output is retained for audit inside the results directory the workspace already excludes from version control. | +| Verify every behavior change | PASS: schema classification, discovery precedence, command construction, polling and terminal-state handling, normalization, threshold binding, and readiness output each need focused coverage. | PASS: unit coverage is identified per module and integration coverage asserts exit codes for pass, gate failure, and error. Every existing azd test must pass unmodified. | +| Product and workflow constraints | PASS: schema evolution is additive; existing helpers and patterns are reused rather than replaced. | PASS: user-visible documentation, initialization guidance, and a changelog entry are included. No constitutional exception is required. | + +**Notable risk carried forward, not a gate failure**: the `azure.ai.evaluations` +extension is not yet published (see [research.md](research.md), Decision 2). The +design treats absence as an ordinary handled outcome, so nothing regresses while +the extension is unavailable, and the surface details are pinned to a verified +schema rather than guessed. + +## Project Structure + +### Documentation (this feature) + +```text +specs/011-azd-ai-eval-surface/ +├── plan.md +├── research.md +├── data-model.md +├── quickstart.md +├── contracts/ +│ └── azd-eval-surface.md +├── checklists/ +│ └── requirements.md +└── tasks.md # created by /speckit-tasks, not by this command +``` + +### Source Code (repository root) + +```text +CHANGELOG.md +docs/ +├── evaluation.md # both surfaces, versions, extension ids +├── how-it-works.md # azd execution path description +└── tutorial-hosted-agent.md # current-surface walkthrough + +src/agentops/ +├── core/ +│ └── azd_eval.py # + current recipe model, classifier, +│ # evals/ discovery, declared-metric names +├── pipeline/ +│ ├── azd_runner.py # legacy adapter: unchanged behavior; +│ │ # shared availability + command helpers +│ ├── azd_eval_runner.py # NEW current-surface adapter +│ └── orchestrator.py # dispatch by resolved surface +├── services/ +│ ├── eval_analysis.py # resolved recipe + surface readiness +│ └── azd_eval_init.py # initialization guidance per surface +└── templates/ + └── agentops.yaml # comment guidance for both surfaces + +tests/ +├── unit/ +│ ├── test_azd_eval.py # classifier, discovery precedence, parsing +│ ├── test_azd_eval_runner.py # NEW adapter, polling, normalization +│ ├── test_azd_runner.py # legacy regression, unchanged +│ ├── test_eval_analysis.py # recipe/surface readiness reporting +│ └── test_azd_eval_init.py # initialization guidance +└── integration/ + └── test_cli_flat_schema.py # end-to-end exit codes per outcome +``` + +**Structure Decision**: Add a second adapter module rather than branching inside +the existing one. The legacy adapter carries behavior that must not change, while +the current surface needs a different command sequence, a different result +retrieval model, and its own aggregation step. Keeping them separate makes the +backward-compatibility requirement verifiable by inspection and confines the +volatile preview surface to one file. Shared, surface-agnostic concerns — +extension availability probing, subprocess execution with progress heartbeat, +failure formatting, and raw artifact writing — stay in the existing module and +are reused by both. + +## Design Decisions + +1. **Recipe classification is content-based.** A parsed document whose root + `evals` key is a sequence is a current-surface recipe; a document with a + mapping-valued `agent` key or a `dataset_reference` key and no `evals` + sequence is a legacy recipe. Filenames and directories are discovery hints + only. A file matching neither shape is a configuration error naming the path + and the reason, never a best-effort run. + +2. **Discovery precedence is fixed and reported.** Auto-discovery collects + candidates from the current-surface location and the legacy locations. Across + surfaces, the current surface wins and the run reports which recipe was chosen + and which was skipped. Within a surface, more than one candidate is rejected + as ambiguous with the candidates listed, preserving today's behavior. An + explicit `eval_recipe` bypasses all of this. + +3. **Threshold binding is split into pre-flight and post-run stages.** Before any + azd command runs, configured thresholds are bound against the metric names the + recipe declares; unmatched or ambiguous keys are a configuration error with + exit code `1`, raised before a run is billed. After the run, a declared metric + the run did not emit is recorded as a failed threshold, producing a gate + failure with exit code `2`. Both fail closed. The existing narrow alias rules + are reused; no broader matching is introduced. This staging applies to the + current surface only, leaving legacy binding behavior untouched. + +4. **Submission is asynchronous and AgentOps owns the wait.** The run is started + in no-wait mode to capture the evaluation and run identifiers deterministically + in a single predictable payload shape, then polled to a terminal state under + the AgentOps timeout, emitting the existing progress heartbeat. A timeout + raises a runtime error that carries both identifiers and the last observed + status, so the operator can reattach in Foundry. This avoids the surface's + blocking mode, which can exit successfully with an unfinished run when its + internal wait budget expires. + +5. **The gate is never delegated.** AgentOps does not pass a failure-gating flag + to azd. A completed run with failing samples exits zero from azd, and AgentOps + decides the release outcome from the normalized results. This keeps a gate + breach distinguishable from an operational failure, which azd's collapsed exit + codes cannot express. + +6. **Aggregate metrics are computed, not read.** The run object carries only + counts. Each aggregate metric is the mean of the per-sample scores recorded + under that metric name. A sample that produced no score for a metric is + excluded from that metric's mean rather than counted as zero. A metric with no + scores at all is absent from the aggregate map, which makes any threshold + bound to it fail closed by construction. + +7. **Per-sample results are first-class.** The current surface populates the + existing `rows` list with one entry per sample, carrying its metric scores, + reasons, and outcome, and marks the result granularity accordingly. Failed and + errored samples appear as rows with their outcome recorded; they are never + dropped from counts nor averaged away. + +8. **Verdicts use three-valued logic.** A sample verdict is a pass, a judged + failure, or absent. Absent means the evaluator did not judge the sample and is + never collapsed into a failure. Sample totals and pass counts come from the + run object's counts, cross-checked against the retrieved rows, so a truncated + retrieval cannot inflate the pass rate. + +9. **Availability detection prefers structured output.** Extension presence is + determined from the structured installed-extension listing, matching on + extension id, because the human-readable listing also includes uninstalled + registry entries and is truncated to terminal width. The existing text scan + remains as a fallback so legacy detection behavior is preserved. + +10. **Value parsing is deliberately tolerant.** Scores accept both numeric and + string encodings; unknown recipe and payload fields are preserved rather than + rejected; the run's always-present error object is tested by message content + rather than existence. The preview surface is expected to drift, and rejecting + unknown fields would turn additive upstream changes into outages. + +11. **Evaluation definitions are reconciled before each run.** The create step + runs first and idempotently registers what the recipe declares, so a run + started from a clean CI checkout resolves the evaluation by name without + depending on local state from a prior interactive session. + +12. **Raw output retention is unchanged in shape and location.** The current + surface writes its native run payload, per-sample output, and command streams + into the same run artifact directory the legacy adapter already uses, for both + successful and failed runs. Per-sample content includes prompts and model + responses, and the workspace already excludes that directory from version + control. + +13. **The new surface is strictly opt-in, and initialization never opts a user + in to something they cannot install.** The current-surface code path is + reachable only when a current-surface recipe exists — through discovery at + `evals/azure.eval.yaml` or through an explicit `eval_recipe`. A workspace + without such a file behaves exactly as it does today, so no existing user and + no fresh clone can encounter a new failure. + + This constrains initialization directly: `agentops eval init` must keep + generating a legacy recipe while the current-surface extension is + unavailable, because generating a current-surface recipe would produce a + workspace whose very next command fails on a dependency the user cannot + install. Initialization probes for the current-surface extension and targets + it only when it is present; otherwise it targets the legacy surface, which is + published and installable. Documentation describes both surfaces regardless, + so a reader can choose deliberately. + +## Complexity Tracking + +No constitution violations and no justified complexity exceptions are required. + +The one design choice worth naming is the second adapter module. It is not an +unnecessary abstraction: the alternative — branching inside the existing adapter +— would interleave a frozen legacy code path with a volatile preview one in the +same functions, making the "legacy behavior is unchanged" requirement +unverifiable by inspection and making every future upstream surface change a +regression risk for existing adopters. diff --git a/specs/011-azd-ai-eval-surface/quickstart.md b/specs/011-azd-ai-eval-surface/quickstart.md new file mode 100644 index 00000000..1ef21b99 --- /dev/null +++ b/specs/011-azd-ai-eval-surface/quickstart.md @@ -0,0 +1,256 @@ +# Quickstart: Validating Current azd Evaluation Surface Support + +**Feature**: `specs/011-azd-ai-eval-surface` +**Date**: 2026-09-06 + +Validation is split into two tiers. **Tier 1 runs anywhere** — no azd, no Azure +credentials, no network — and covers every guarantee in +[contracts/azd-eval-surface.md](contracts/azd-eval-surface.md) through the mocked +subprocess boundary. **Tier 2 requires a live environment** and is only possible +where the `azure.ai.evaluations` extension has been published into an azd +extension source. + +Tier 1 is the acceptance bar for this feature. Tier 2 is confirmation, and until +the extension ships it cannot run on a clean machine. + +## Prerequisites + +```bash +python -m pip install -e . +python -m pip install pytest +``` + +Tier 2 additionally needs azd `>= 1.27.1`, the `azure.ai.evaluations` extension, +an authenticated Azure session, and a Foundry project with a deployed agent. + +## Tier 1 — Offline validation + +### T1.1 Full suite, no regressions + +```bash +python -m pytest tests/ -x -q +``` + +**Expected**: all tests pass. Every pre-existing azd test must pass **without +modification** — that is the mechanical proof of the backward-compatibility +requirement. A test that had to be edited to accommodate this feature is a +finding, not a fix. + +### T1.2 Recipe classification and discovery precedence + +```bash +python -m pytest tests/unit/test_azd_eval.py -q +``` + +**Expected coverage**: + +| Workspace state | Expected resolution | +|---|---| +| `evals/azure.eval.yaml` only | current surface, no skipped entries | +| `eval.yaml` only | legacy surface, unchanged from today | +| `src//eval.yaml` only | legacy surface, unchanged from today | +| both surfaces present | current selected, legacy recorded as skipped | +| two legacy candidates | ambiguity error listing both | +| `eval_recipe` set | that path used, classified by content | +| document matching neither schema | configuration error naming the path | + +Also assert that a current-surface recipe using `datasets[].file` parses, that an +unknown top-level key is preserved rather than rejected, and that declared metric +names include builtin evaluator references, evaluator labels, and rubric +dimension ids. + +### T1.3 Adapter behavior against a mocked azd + +```bash +python -m pytest tests/unit/test_azd_eval_runner.py -q +``` + +**Expected coverage**, each with the subprocess boundary faked: + +- **Command sequence**: probe, create, `run start --no-wait`, poll, final + `run show`, `run output list --all --output-file`. Assert the evaluation is + started exactly once and that no failure-gating flag is passed. +- **Flag correctness**: `--path` receives the recipe's *directory*; `run start` + receives no positional argument; `run output list` receives the run id + positionally; the current surface uses `--output-file`, never `--out-file`. +- **Polling**: a sequence of non-terminal statuses followed by `completed` + resolves; an unknown status keeps polling; timeout raises a runtime error that + contains the evaluation id, run id, and last observed status. +- **Aggregation**: metric means exclude samples with no score rather than + treating them as zero; a metric with no scores anywhere is absent from the + aggregate map. +- **Tolerant decoding**: a score arriving as the string `"4"` decodes to `4.0`; a + score that is neither number nor numeric string is recorded as absent. +- **Three-valued verdicts**: a sample with `passed: null` is not counted as a + judged failure and is not counted as a pass; a sample with an empty result list + is a failure. +- **Run-level error**: an error object whose members are null is not treated as a + failure; a non-empty error message is. +- **Portal link**: `report_url` wins over `portal_url`. +- **Raw artifacts**: the run payload, the per-sample items, and both command + streams are written for a successful run *and* for a failed run. + +### T1.4 Availability detection + +Still in `tests/unit/test_azd_eval_runner.py`: + +- A structured installed-extension listing that includes the extension id → + available. +- A listing that omits it → unavailable, with an error naming + `azure.ai.evaluations` and the install command. +- A **human-readable** listing containing the extension id with a not-installed + status → **unavailable**. This is the false-positive guard; a substring scan + would wrongly report available. +- The legacy detection path continues to resolve through its existing fallback, + proven by the unmodified legacy tests. + +### T1.5 Exit codes end to end + +```bash +python -m pytest tests/integration/test_cli_flat_schema.py -q +``` + +**Expected**, with azd faked at the subprocess boundary: + +| Scenario | Exit code | +|---|---| +| Completed run, all thresholds satisfied | `0` | +| Completed run, a bound threshold not satisfied | `2` | +| Completed run, a declared metric produced no score | `2` | +| Threshold naming a metric no evaluator declares | `1`, **before any azd command runs** | +| Threshold ambiguous across declared metrics | `1`, before any azd command runs | +| Terminal status of `failed`, `error`, or either cancelled spelling | `1` | +| Poll timeout | `1`, with identifiers in the message | +| Run completed with zero samples | not `0` | +| azd or extension missing | `1`, naming the missing component | + +For the two pre-flight cases, assert that the fake azd was **never invoked**. +That is the observable proof that a configuration typo does not bill a cloud run. + +### T1.6 Readiness analysis + +```bash +python -m pytest tests/unit/test_eval_analysis.py -q +agentops eval analyze --format json +``` + +**Expected**: for an `execution: azd` workspace, the report names the resolved +recipe path and its surface. With no discoverable recipe, it reports the gap and +the action required. With both surfaces present, it reports the selection and +what was skipped. + +### T1.7 Nobody is opted in by accident + +The current surface must be unreachable unless a current-surface recipe exists. + +```bash +# Workspace with a legacy eval.yaml and no evals/ directory +agentops eval run +agentops eval analyze +``` + +**Expected**: identical output to the previous release. No mention of the current +surface, no new warning, no new error. This is the regression guard for existing +users and fresh clones. + +```bash +# Fresh workspace, current-surface extension not installed +agentops eval init +``` + +**Expected**: a **legacy** recipe is generated, and the workspace is immediately +runnable. Initialization must not generate `evals/azure.eval.yaml` while the +current-surface extension is unavailable, because that would produce a workspace +whose next command fails on a dependency the user cannot install. Assert that +initialization probes for the extension and that the probe result selects the +target surface. + +### T1.8 Unavailable-extension experience + +In a workspace where someone deliberately added `evals/azure.eval.yaml`, on a +machine without the extension: + +```bash +agentops eval run +echo "exit=$?" +``` + +**Expected**: exit `1`. The message names `azure.ai.evaluations`, states the azd +version floor, and gives the install command. It must not suggest that AgentOps +fell back to another engine, and it must not mention the legacy extension. + +Because the extension is not yet in the default registry, this is what every +opt-in user sees today, so its wording carries more weight than usual. + +## Tier 2 — Live validation + +Only runnable where the extension is installed. + +### T2.1 Happy path + +Given a Foundry project, a deployed agent, `evals/azure.eval.yaml` declaring a +dataset and at least one builtin evaluator, and matching thresholds: + +```bash +agentops eval run +echo "exit=$?" +``` + +**Expected**: + +- Progress output names the resolved recipe and the current surface. +- A heartbeat appears while the run is in flight; the command never looks hung. +- Exit `0` when thresholds pass. +- `.agentops/results/latest/results.json` contains computed `aggregate_metrics`, + one row per sample, and a `config.azd_evaluation` block whose `surface` is + `current`. +- `.agentops/results/latest/report.md` renders thresholds, rubric dimensions, and + any failed samples. +- The raw run payload and per-sample items are present in the same directory. +- **The Foundry portal shows exactly one run** for this invocation. This is the + observable check that results retrieval did not re-execute the evaluation. + +### T2.2 Gate failure + +Tighten one threshold above the observed score and rerun. + +**Expected**: exit `2`. The report names the failing threshold. The run itself is +still recorded as completed — a gate failure is not an execution failure. + +### T2.3 Rubric dimensions + +With a recipe declaring a local rubric evaluator and thresholds naming its +dimension ids: + +**Expected**: each dimension appears in `aggregate_metrics` under its id and is +gated exactly like a builtin metric. + +### T2.4 Legacy regression + +In a workspace with only a legacy `eval.yaml`: + +```bash +agentops eval run +``` + +**Expected**: identical behavior to the previous release — same commands, same +normalization, same `result_granularity` of `aggregate`, same exit code. + +### T2.5 Baseline comparison + +```bash +agentops eval run --baseline .agentops/results//results.json +``` + +**Expected**: the comparison renders against the current-surface run exactly as +it does for any other execution mode. + +## Documentation check + +Confirm before closing the feature: + +- `docs/evaluation.md` documents both surfaces, their extension ids, their azd + version floors, and the discovery precedence rule. +- The preview status of `azure.ai.evaluations`, including that it is not yet in + the default extension registry, is stated plainly rather than implied. +- `CHANGELOG.md` records the added support under the unreleased section. diff --git a/specs/011-azd-ai-eval-surface/research.md b/specs/011-azd-ai-eval-surface/research.md new file mode 100644 index 00000000..5a2e84fd --- /dev/null +++ b/specs/011-azd-ai-eval-surface/research.md @@ -0,0 +1,315 @@ +# Phase 0 Research: Current azd AI Evaluation Surface + +**Feature**: `specs/011-azd-ai-eval-surface` +**Date**: 2026-09-06 + +All command names, flags, YAML keys, and JSON field names below are verified +against source in the `azure.ai.evaluations` extension implementation. Items +that could not be verified are listed under Open Risks and are designed around +defensively rather than assumed. + +## Decision 1: Target the `azure.ai.evaluations` extension and the `ai.eval` namespace + +**Decision**: The current surface is the azd extension `azure.ai.evaluations`, +which registers the command namespace `ai.eval`, producing `azd ai eval ...`. +It declares a minimum azd version of `1.27.1`. The legacy surface remains the +`azure.ai.agents` extension with the namespace `ai.agent`, producing +`azd ai agent eval ...`. + +**Rationale**: The extension manifest states the id, namespace, and azd version +floor directly. AgentOps must name the correct extension in its availability +error message, otherwise the user installs the wrong thing. + +**Alternatives considered**: Treating the two surfaces as one command family +with different subcommands was rejected. They are separate extensions with +separate install steps and separate version floors, so availability must be +checked per surface. + +## Decision 2: The extension is not yet published, so absence is the normal state + +**Decision**: Build the integration now, but treat "extension not installed" as +an ordinary, well-handled outcome rather than an exceptional one. Detection +failure must produce an actionable configuration error naming +`azure.ai.evaluations`, and must never degrade into another execution engine. + +**Rationale**: The extension exists only as an unmerged pull request against +`Azure/azure-dev`. It is absent from both the public and dev extension +registries, and installing it from the default source fails today. It can only +be obtained by building and publishing it into a local extension source. This +means that for every user, on every machine, the unavailable path is the path +they will hit first. + +**Consequences**: End-to-end validation against a live extension is not +possible in CI or on a clean workstation until the extension ships. All +automated coverage is therefore built on mocked subprocess boundaries, which the +project already requires for Azure interactions. The published documentation +must state the install path honestly, including that the extension is preview +and not yet in the default registry. + +**Alternatives considered**: Deferring the feature until the extension ships was +rejected because the recipe schema and command surface are already stable enough +to code against, and the consuming lab work is blocked now. Vendoring or +auto-installing the extension was rejected: AgentOps does not install tooling. + +## Decision 3: Detect availability with structured output, not substring scanning + +**Decision**: Detect extension availability by parsing +`azd extension list --installed -o json`, which returns a bare JSON array whose +entries carry `id` and `installedVersion`. Match on `id` equality. Retain the +existing text scan only as a fallback when the JSON form is unavailable, so the +legacy detection path keeps its current observable behavior. + +**Rationale**: The default table form of `azd extension list` lists every +registry extension, not just installed ones, and marks uninstalled entries with +a `Not installed` status. A substring scan for an extension id therefore reports +a false positive as soon as the extension appears in the registry but before the +user installs it. The table is also truncated to terminal width, so version +values are unreliable. azd itself directs users to `-o json` for full detail. + +**Alternatives considered**: Parsing the table's `STATUS` column was rejected as +brittle across azd versions and terminal widths. Skipping detection and letting +the command fail was rejected because the resulting error would not tell the +user which extension to install. + +## Decision 4: Distinguish recipe schemas structurally, not by filename + +**Decision**: Classify a discovered recipe by inspecting its parsed content. A +document whose root `evals` key is a sequence is a current-surface recipe. A +document with a mapping-valued `agent` key, or a `dataset_reference` key, and no +`evals` sequence, is a legacy recipe. Filename and directory are discovery hints +only, never the classifier. + +**Rationale**: The two schemas differ at the root in ways that cannot collide. +The current schema has exactly three optional top-level array keys — `datasets`, +`evaluators`, `evals` — and forbids additional properties. The legacy schema is +a flat mapping with `name`, `agent`, `dataset_reference`, `evaluators`, and +`options`, and has no `evals` key at all. Classifying by content also handles the +case where the current-surface configuration is embedded in `azure.yaml` under a +service with host `azure.ai.eval`, and the case where a recipe is referenced from +an arbitrary path. + +**Alternatives considered**: Keying off the `evals/` directory or the +`azure.eval.yaml` basename alone was rejected because the directory is only a +default, the path is overridable by flag, and content classification is needed +anyway to parse the file. + +## Decision 5: Canonical discovery location is `evals/azure.eval.yaml` + +**Decision**: Add `evals/azure.eval.yaml` to auto-discovery, alongside the +existing legacy locations. The azd commands accept a `--path` flag that names +the *directory* holding the file, not the file itself, so AgentOps passes the +recipe's parent directory when invoking azd. + +**Rationale**: The extension defines the default evaluation directory as `evals` +and the configuration basename as `azure.eval.yaml`. The basename is a constant +with no flag to override it; only the containing directory is configurable. + +**Alternatives considered**: Recursively scanning the workspace for any +`*.eval.yaml` was rejected as both slow and ambiguous. Users with a non-default +location already have the explicit `eval_recipe` setting. + +## Decision 6: Compute aggregate metrics from per-sample scores + +**Decision**: AgentOps computes each aggregate metric as the mean of the +per-sample scores for that metric, read from the per-sample output. The run +object's counts are used for sample totals and pass/fail/error/skip tallies, not +for metric values. + +**Rationale**: This is the single most consequential finding. The run object +exposes `result_counts` and `per_testing_criteria_results`, but both are +**counts**, not scores. There is no aggregate numeric metric anywhere on the run +object. The only numeric scores in the surface are per-sample. The legacy +adapter's assumption that the payload carries aggregate numeric metrics does not +hold for the current surface, so aggregation moves into AgentOps. + +**Consequences**: Per-sample output retrieval is mandatory, not optional — the +gate cannot be evaluated without it. It also means AgentOps now populates +per-sample rows for this surface, where the legacy adapter emits an empty row +list and marks the result aggregate-only. + +**Alternatives considered**: Deriving a pass rate from `result_counts` and gating +only on that was rejected: it would silently discard every threshold that names a +specific evaluator or rubric dimension, which is the primary use case. + +## Decision 7: Submit asynchronously and poll, rather than relying on `--wait` + +**Decision**: Invoke `run start` with `--no-wait` to obtain the evaluation and +run identifiers deterministically, then poll `run show` at a fixed interval until +the run reaches a terminal state or the AgentOps timeout expires. + +**Rationale**: `run start` blocks by default, but it has an internal wait budget. +When that budget expires it prints reattachment instructions and **exits 0**, +emitting a small handoff object instead of the full run object. A caller that +trusts the exit code would treat an unfinished run as a success. Worse, the same +command emits two structurally different JSON shapes depending on which path it +took. Forcing `--no-wait` collapses this into one predictable shape, puts the +timeout under AgentOps control, and lets AgentOps emit its own progress +heartbeat during long runs, consistent with the existing legacy adapter. + +It also directly satisfies the specification's requirement that a run which never +reaches a terminal state fails with the identifiers preserved, because those +identifiers are captured before polling begins. + +**Terminal states**: `completed`, `failed`, `canceled`, `cancelled`, `error`. +Both spellings of cancelled occur, and `error` is distinct from `failed`. Only +`completed` is treated as success. Status comparison is case-insensitive. + +**Alternatives considered**: Using the default blocking `--wait` and detecting +the handoff shape post hoc was rejected: it is a second code path guarding a +silent-success bug, and it forfeits control of the timeout. + +## Decision 8: Never pass `--fail-on`; AgentOps owns the gate + +**Decision**: AgentOps does not pass `--fail-on` to `run start`. Thresholds +configured in `agentops.yaml` remain the only gate, and are applied to the +normalized results. + +**Rationale**: Without `--fail-on`, a completed run with failing samples exits 0, +which is exactly what AgentOps needs: azd reports execution outcome, AgentOps +decides release outcome. Passing `--fail-on` would fold a gate breach into the +same non-zero exit code as an operational failure, and azd collapses extension +exit codes so the two would be indistinguishable without parsing message text. +Keeping the gate in AgentOps preserves the project's exit-code contract. + +**Alternatives considered**: Translating configured thresholds into `--fail-on` +expressions was rejected. `--fail-on` supports only a pass-rate or +any-failure form, which cannot express per-metric thresholds, and it would move +the gate outside the artifact that CI reviews. + +## Decision 9: Retrieve results with two verified calls + +**Decision**: After the run reaches a terminal state, call `run show` for the run +object and `run output list` with `--all` and `--output-file` for the per-sample +items. Read the per-sample items from the written file rather than from stdout. + +**Rationale**: `run output list` emits a bare JSON array by design, which means +the paging fields carried by the underlying list envelope are stripped from JSON +output. A default call returns only a bounded first page with no indication that +more exists. Passing `--all` or `--output-file` switches the command to an +unbounded page size, so both are passed. The file is written atomically with +owner-only permissions because rows contain prompts and model responses. + +The positional run identifier takes precedence over the `--run` flag, so the +positional form is used. + +**Alternatives considered**: `run output export`, which writes the run and its +items as a single document, is appealing as a one-call retrieval, but its exact +document shape was not verified against source. Two individually verified calls +were preferred over one partially verified call. Export can replace both later +without changing the normalized contract. + +## Decision 10: Bind thresholds before submitting, and fail closed after + +**Decision**: Split threshold validation into two stages. + +1. **Pre-flight, before any azd command runs**: bind configured threshold keys + against the metric names the *recipe declares* — builtin evaluator names, + evaluator labels, and rubric dimension ids. An unmatched or ambiguous + threshold is a configuration error, exit code `1`, raised before an evaluation + is created or a run is billed. +2. **Post-run**: a metric that the recipe declared but the completed run did not + emit is recorded as a failed threshold, producing a gate failure with exit + code `2`. + +**Rationale**: These are genuinely different failures and deserve different exit +codes under the project's contract. A threshold naming a metric that no evaluator +in the recipe can ever produce is a misconfiguration, and catching it before +submission saves a cloud run. A threshold naming a declared metric that the run +failed to produce means execution succeeded but the evidence is incomplete, which +is a gate outcome, not a configuration error. Both fail closed; neither can +report a pass. + +The existing narrow alias rules are reused unchanged. Broad fuzzy matching is not +introduced, because a wrong match here creates a false-green release gate. + +**Scope**: Pre-flight binding applies to the current surface only. The legacy +adapter's post-run binding behavior is left exactly as it is, so existing +behavior and existing tests are unaffected. + +**Alternatives considered**: Treating every binding failure as a runtime error, +matching the legacy adapter, was rejected because it makes a missing score +indistinguishable from a typo in the configuration. Treating every binding +failure as a gate failure was rejected because it would bill a cloud run to +discover a typo. + +## Decision 11: Parse defensively at every value boundary + +**Decision**: The per-sample parser treats the following as expected variation +rather than corruption: + +- `score` may arrive as a JSON number or a string, and is decoded through a + tolerant numeric conversion. A value that is neither is recorded as a missing + score for that metric on that sample, not as a zero. +- `passed` is nullable and carries three-valued meaning. `true` is a pass, + `false` is a judged failure, and `null` means the evaluator did not judge the + sample. `null` must never be collapsed to `false`. +- A sample with an empty result list is a failure, not a pass. +- The run's `error` object is always present with null members on success, so + failure is detected by testing for a non-empty `error.message`, never by + testing whether `error` exists. +- `report_url` is preferred for the portal link, with `portal_url` as fallback. + +**Rationale**: Each of these is a documented property of the surface, and each +has a plausible naive reading that produces a false pass or a false failure. +Collapsing a null verdict to `false` would fail runs that merely lacked a +judgement; treating a missing score as `0.0` would fail a threshold that was +never actually evaluated. + +**Alternatives considered**: Strict schema validation that rejects any deviation +was rejected. The surface is preview and explicitly volatile; unknown fields must +be tolerated and preserved, exactly as the legacy recipe model already does. + +## Decision 12: Reconcile the evaluation definition before each run + +**Decision**: Invoke `azd ai eval create` against the recipe directory before +starting a run, treating it as idempotent reconciliation of the declared +datasets, evaluators, and evaluation. + +**Rationale**: The create step registers what the recipe declares and resolves +the evaluation by name. A CI runner is a clean environment with no cached local +state from a prior interactive session, so an evaluation referenced only by name +would not resolve without it. Reconciling first makes the run reproducible from a +fresh clone. + +This step registers definitions in Foundry. That is consistent with an +evaluation run, which already publishes results, and is unrelated to the +read-only guarantee that constrains Doctor and Cockpit. + +**Alternatives considered**: Calling `create` only when `run start` fails to +resolve the evaluation was rejected as a fragile error-string dependency. + +## Decision 13: `-o json` implies non-interactive + +**Decision**: Pass `-o json` on every invocation and rely on it to suppress +prompts, while continuing to pass the explicit non-interactive flag for clarity +and for commands where output format is not requested. + +**Rationale**: The extension makes JSON output imply non-interactive mode, so +JSON invocations never block waiting for input. This matters because the +evaluation runs from CI with no attached terminal. + +## Open Risks + +| Risk | Impact | Mitigation | +|---|---|---| +| The extension is unshipped and its command surface is explicitly described as subject to change before merge. | Flags or field names could shift, breaking the adapter. | Isolate every azd command construction and every JSON field name in one adapter module with focused unit tests, so a surface change is a localized edit. Pin the documented supported version range. | +| The exact key names inside a sample's data item, which carry the input, expected value, and response, were not verified. | Per-sample rows could be populated with empty or wrong text. | Extract by documented key preference with a safe fallback, always retain the raw item in the raw artifact, and never let extraction failure fail the run or the gate. Metric scores, which the gate depends on, do not come from this structure. | +| Whether `score` is emitted as a number or a string was not verified. | A strict numeric parse would fail every sample. | Accept both, per Decision 11. | +| The service could report a run status outside the client's known terminal set. | The poller could wait until timeout on a finished run. | Treat unknown statuses as non-terminal and let the AgentOps timeout bound the wait, surfacing the last observed status in the error. | +| `run output export` shape unverified. | None today. | Not used; noted as a future simplification. | +| Per-sample output contains prompts and model responses. | Sensitive content could reach a shared location. | Raw artifacts stay in the run's results directory, which the workspace already excludes from version control, matching how existing raw azd artifacts are handled. | +| Learn documentation for this surface is not published; only the legacy surface is documented. | Users cannot self-serve setup. | AgentOps documentation states the extension id, the azd version floor, and the install path explicitly rather than linking to a page that does not exist. | + +## Sources + +- `azure.ai.evaluations` extension manifest, command implementations, and JSON + models, from the open pull request `Azure/azure-dev#9500`. +- `schemas/azure.ai.eval.json` — the formal recipe schema, including the + `additionalProperties: false` constraints relied on for schema discrimination. +- `schemas/examples/inline.azure.yaml` and `schemas/examples/ref.azure.yaml` — + official example recipes. +- `Azure/azure-dev` `cli/azd/extensions/registry.json` and `registry.dev.json` — + confirmation that the extension is absent from both registries. +- Microsoft Learn, "Evaluate with the Azure Developer CLI" — the legacy + `azd ai agent eval` surface and `eval.yaml` schema, used here as the + backward-compatibility reference. diff --git a/specs/011-azd-ai-eval-surface/spec.md b/specs/011-azd-ai-eval-surface/spec.md new file mode 100644 index 00000000..cdf4526e --- /dev/null +++ b/specs/011-azd-ai-eval-surface/spec.md @@ -0,0 +1,165 @@ +# Feature Specification: Current azd AI Evaluation Surface Support + +**Feature Branch**: `011-azd-ai-eval-surface` + +**Created**: 2026-09-06 + +**Status**: Draft + +**Input**: GitHub issue [Azure/agentops#484](https://github.com/Azure/agentops/issues/484) - "Support the current azd ai eval command surface". AgentOps currently delegates `execution: azd` to the legacy `azd ai agent eval` command surface and discovers `eval.yaml` recipes. Current Microsoft Foundry guidance uses the `azure.ai.evaluations` extension with `azd ai eval`, `evals/azure.eval.yaml`, `azd ai eval create`, `azd ai eval run start`, and `azd ai eval run output list`. This gap blocks the AgentOps VBD Evaluate lab (related: [#483](https://github.com/Azure/agentops/issues/483)) from using `agentops eval run` as the evidence and release-readiness wrapper around the current Foundry cloud evaluation flow. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Evaluate a Foundry hosted agent through the current azd evaluation surface (Priority: P1) + +A release engineer following current Microsoft Foundry guidance has an `evals/azure.eval.yaml` recipe in their workspace. They set `execution: azd` in `agentops.yaml` and run `agentops eval run`. AgentOps discovers the current-surface recipe, delegates the cloud evaluation to the current azd evaluation commands, and produces the same normalized `results.json` and `report.md` that every other AgentOps execution mode produces. + +**Why this priority**: This is the entire point of the feature. Without it, teams on current Foundry guidance cannot use AgentOps as their release-readiness wrapper at all, and the VBD Evaluate lab has no supported path. Every other story in this specification is a refinement of this one. + +**Independent Test**: Can be fully tested by placing a current-surface recipe at `evals/azure.eval.yaml`, setting `execution: azd`, running `agentops eval run`, and confirming that the evaluation is submitted and retrieved through the current azd evaluation surface exactly once and that a normalized `results.json` and `report.md` are written to the run's artifact directory. + +**Acceptance Scenarios**: + +1. **Given** a workspace containing exactly one current-surface recipe at `evals/azure.eval.yaml` and `execution: azd` in `agentops.yaml`, **When** `agentops eval run` is executed, **Then** the recipe is auto-discovered without requiring an explicit `eval_recipe` path. +2. **Given** the same setup, **When** the run executes, **Then** the cloud evaluation is created, started, and its per-sample output retrieved through the current azd evaluation surface, and the evaluation itself is executed exactly once (no duplicate submission and no re-execution to retrieve results). +3. **Given** a completed current-surface run, **When** artifacts are written, **Then** run-level metrics, rubric dimension scores, per-sample outcomes, and any per-sample failures are normalized into the existing `results.json` schema and rendered into `report.md`, with the same field names and shapes produced by local, cloud, and legacy azd execution. +4. **Given** `execution: azd` with an explicit `eval_recipe` pointing at a current-surface recipe outside the default discovery locations, **When** `agentops eval run` is executed, **Then** that recipe is used instead of auto-discovery. +5. **Given** a workspace where the azd tooling or the evaluation extension required by the discovered recipe is unavailable, **When** `agentops eval run` is executed, **Then** the run fails with a configuration error that names the missing dependency and the action required, and AgentOps does not silently fall back to another execution engine. + +--- + +### User Story 2 - Gate the release on thresholds and rubric dimensions from a current-surface run (Priority: P2) + +A release engineer has thresholds in `agentops.yaml` that reference both built-in evaluator metrics and named rubric dimensions declared in their current-surface recipe. They run `agentops eval run` in CI and expect the gate to behave exactly as it does for every other execution mode: thresholds bind to the emitted metrics, missing metrics fail the run rather than silently passing, baseline comparison still works, and the process exit code communicates pass, gate failure, or error. + +**Why this priority**: Delivering normalized artifacts without a trustworthy gate would produce false-green releases, which directly violates the project's release-evidence guarantees. This story is what turns a working integration into release-readiness evidence, but it depends on Story 1 producing a run at all. + +**Independent Test**: Can be fully tested by running a current-surface evaluation with a threshold that binds to an emitted metric (expect pass), a threshold whose metric is absent from the run (expect gate failure, not a pass), and a `--baseline` comparison against a previous run, then asserting the process exit code for each case. + +**Acceptance Scenarios**: + +1. **Given** thresholds referencing metrics that the current-surface run emits, **When** all thresholds are satisfied, **Then** the run reports a pass and exits with code `0`. +2. **Given** thresholds referencing metrics that the current-surface run emits, **When** at least one threshold is not satisfied, **Then** the run reports a gate failure and exits with code `2`. +3. **Given** a threshold whose metric is not present in the completed run's emitted metrics, **When** the run finishes, **Then** the threshold is treated as failed and the run does not report a pass, and the report names the unbound threshold. +4. **Given** a threshold key that could bind to more than one emitted metric name, **When** the run finishes, **Then** the ambiguity is surfaced as an explicit failure rather than resolved by guessing. +5. **Given** thresholds that reference rubric dimensions declared in the current-surface recipe, **When** the run emits per-dimension scores, **Then** those dimension scores bind to the corresponding thresholds and are gated identically to built-in evaluator metrics. +6. **Given** a completed current-surface run and a `--baseline` pointing at a previous run's results, **When** `agentops eval run` is executed, **Then** the baseline comparison operates on the normalized result shape identically to any other execution mode. +7. **Given** the azd evaluation surface reports an execution error for the run as a whole, **When** the run terminates, **Then** AgentOps reports a runtime error with exit code `1`, distinct from a threshold gate failure. + +--- + +### User Story 3 - Keep existing legacy azd recipes working unchanged (Priority: P3) + +A team that already integrated AgentOps with the legacy `azd ai agent eval` surface and a root-level `eval.yaml` recipe upgrades AgentOps. Their existing configuration, commands, CI workflows, and results continue to work with no configuration changes. + +**Why this priority**: Backward compatibility is a hard project constraint on public contracts, and breaking existing adopters would be a regression regardless of how well the new surface works. It is prioritized after the gate because it protects existing value rather than delivering new value. + +**Independent Test**: Can be fully tested by running the existing legacy-recipe test suite and a legacy end-to-end scenario against the updated build, with no changes to `agentops.yaml`, and confirming identical discovery, execution, normalization, and exit-code behavior. + +**Acceptance Scenarios**: + +1. **Given** an unchanged workspace containing only a legacy recipe at the workspace root or under `src//` and `execution: azd`, **When** `agentops eval run` is executed, **Then** the legacy recipe is discovered and executed through the legacy azd evaluation surface exactly as before. +2. **Given** an unchanged `agentops.yaml` from a previous AgentOps version, **When** it is loaded, **Then** it remains valid and requires no new or renamed configuration fields to keep working. +3. **Given** a legacy-surface run, **When** artifacts are written, **Then** the `results.json` and `report.md` contract is unchanged from the previous behavior. + +--- + +### User Story 4 - Diagnose azd-backed evaluation setup and failures (Priority: P4) + +A release engineer preparing an azd-backed evaluation runs `agentops eval analyze` to understand which recipe will be used and which surface it belongs to before spending time on a cloud run. When a run does fail, they need the raw azd output retained alongside the normalized artifacts so they can reproduce and troubleshoot the failure directly against azd. + +**Why this priority**: This story reduces time-to-diagnosis and onboarding friction but is not required for a correct evaluation or a correct gate. It is valuable polish layered on the three stories above. + +**Independent Test**: Can be fully tested by running `agentops eval analyze` in workspaces containing a current-surface recipe, a legacy recipe, and no recipe, confirming each case reports the resolved recipe and surface or an actionable gap, then forcing a run failure and confirming the raw azd output is retained in the run's artifact directory. + +**Acceptance Scenarios**: + +1. **Given** a workspace with a discoverable recipe and `execution: azd`, **When** `agentops eval analyze` is executed, **Then** the report identifies the resolved recipe path and which azd evaluation surface it will use. +2. **Given** a workspace with `execution: azd` and no discoverable recipe, **When** `agentops eval analyze` is executed, **Then** the report identifies the missing recipe as a gap and states the action required to resolve it. +3. **Given** any azd-backed run, whether it succeeds or fails, **When** the run terminates, **Then** the raw azd command output is retained in the run's artifact directory for diagnostics and audit. +4. **Given** a user setting up an azd-backed evaluation for the first time, **When** they consult the published documentation, **Then** the documentation states the minimum azd version and the required evaluation extension for each supported surface. + +--- + +### Edge Cases + +- What happens when the workspace contains both a legacy recipe and a current-surface recipe and no explicit `eval_recipe` is set? The current-surface recipe MUST win, and the run MUST report which recipe was selected and that another discoverable recipe was skipped, so the choice is never silent. +- What happens when the workspace contains more than one recipe of the *same* surface and no explicit `eval_recipe` is set? The run MUST be rejected as ambiguous, since there is no basis to prefer one over the other. +- What happens when a discovered recipe file exists but its schema matches neither the legacy nor the current shape? The run MUST fail with a configuration error naming the recipe path and the reason, rather than attempting a best-effort run against an unknown schema. +- What happens when the evaluation is created and started successfully but the run never reaches a terminal state within the allowed time? The run MUST terminate with a runtime error that preserves the created evaluation and run identifiers so the operator can inspect the run in Foundry, rather than reporting a pass or a gate failure. +- What happens when the run completes but returns zero samples? The run MUST NOT report a pass; a zero-sample completion is treated as a failure to produce evidence. +- What happens when the run completes but individual samples failed? Failed samples MUST be represented in the normalized results and MUST NOT be silently dropped from sample counts or averaged away as if they had not been attempted. +- What happens when the run completes but emits no readable metrics at all? The run MUST fail with a runtime error rather than producing an empty-but-passing result. +- What happens when a rubric dimension is declared in the recipe but the completed run emits no score for it? Any threshold bound to that dimension MUST fail closed; an undeclared-but-emitted metric MUST NOT be treated as an error. +- How does the system behave when the required extension for the current surface is installed but the discovered recipe is a legacy recipe (or vice versa)? The run MUST fail with an error naming the specific missing extension for the surface the recipe requires. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The system MUST discover current-surface evaluation recipes at the location prescribed by current Microsoft Foundry guidance (`evals/azure.eval.yaml`) in addition to the existing legacy recipe locations, without requiring new configuration. +- **FR-002**: The system MUST determine which azd evaluation surface a discovered recipe belongs to by inspecting the recipe itself, and MUST NOT require the user to declare the surface in `agentops.yaml`. +- **FR-003**: The system MUST parse both the legacy recipe schema and the current recipe schema, extracting at minimum the evaluation identity, the agent under evaluation, the dataset reference, the declared evaluators, and any declared rubric dimensions. +- **FR-004**: The system MUST support an explicit `eval_recipe` path that overrides auto-discovery for both recipe schemas. +- **FR-004a**: When auto-discovery finds recipes from both surfaces and no explicit `eval_recipe` is set, the system MUST select the current-surface recipe and MUST report both the selected recipe and the skipped recipe to the user. +- **FR-004b**: When auto-discovery finds more than one recipe belonging to the same surface and no explicit `eval_recipe` is set, the system MUST reject the run with an error that lists the candidates and directs the user to set `eval_recipe`. +- **FR-005**: For a current-surface recipe, the system MUST create the evaluation, start the run, and retrieve the run's per-sample output through the current azd evaluation commands, executing the evaluation exactly once per `agentops eval run` invocation. +- **FR-006**: The system MUST normalize run-level metrics, rubric dimension scores, per-sample outcomes, per-sample failures, and execution errors from the current surface into the existing `results.json` schema, producing the same row and metric shapes as every other execution mode. +- **FR-007**: The system MUST bind configured thresholds to the metrics and rubric dimensions emitted by a current-surface run using the same binding rules already applied to legacy azd runs, resolving exact metric names before any alias. +- **FR-008**: The system MUST treat a threshold that cannot be bound to exactly one emitted metric as a failed gate, and MUST NOT report the run as passing when any threshold is unbound or ambiguous. +- **FR-009**: The system MUST preserve the existing exit-code contract for azd-backed runs: `0` when execution succeeded and all gates passed, `2` when execution succeeded and at least one gate failed, and `1` for runtime or configuration errors. +- **FR-010**: The system MUST support `--baseline` comparison for current-surface runs using the same normalized result shape and comparison behavior applied to all other execution modes. +- **FR-011**: The system MUST render `report.md` for current-surface runs from the normalized results, including bound thresholds, unbound thresholds, rubric dimension outcomes, and failed samples. +- **FR-012**: The system MUST retain the raw azd command output for every azd-backed run in that run's artifact directory, for both successful and failed runs. +- **FR-013**: The system MUST continue to discover, execute, and normalize legacy recipes through the legacy azd evaluation surface with unchanged behavior, and MUST NOT require any configuration change from existing adopters. +- **FR-014**: The system MUST fail with an actionable configuration error, naming the missing dependency and the required action, when the azd tooling or the evaluation extension required by the discovered recipe is unavailable, and MUST NOT switch execution engines implicitly. +- **FR-015**: `agentops eval analyze` MUST report, for an azd-backed workspace, the resolved recipe path and the azd evaluation surface it belongs to, or an actionable gap when no recipe can be resolved. +- **FR-016**: Workspace initialization MUST continue to generate a recipe for a surface that is actually installable in the user's environment. It MUST default to the legacy surface while the current-surface extension is unavailable, MUST prefer the current surface once that extension is detected as installed, and MUST NOT produce a workspace whose first evaluation run fails for lack of an uninstallable dependency. +- **FR-017**: Published documentation MUST state the minimum azd version and the required evaluation extension for each supported azd evaluation surface, and the release changelog MUST record the added support. +- **FR-018**: The system MUST NOT report a passing run when the completed evaluation produced zero samples or no readable metrics. + +### Key Entities + +- **Evaluation Recipe**: A workspace file describing an azd-backed evaluation. Two schemas are supported: the legacy schema discovered at the workspace root or under `src//`, and the current schema discovered under the `evals/` directory. Carries the evaluation identity, agent, dataset reference, evaluators, and rubric dimensions. +- **Evaluation Surface**: The azd command family and extension a recipe requires. Each discovered recipe resolves to exactly one surface, which determines how the evaluation is created, started, and read back. +- **Evaluation Run Reference**: The identifiers returned when an evaluation is created and started, used to retrieve output and to point an operator at the run in Foundry when something goes wrong. +- **Rubric Dimension**: A named scoring dimension declared in a recipe and emitted per sample and in aggregate by the evaluation run. Bindable to a threshold exactly like a built-in evaluator metric. +- **Failed Sample**: A dataset sample that the evaluation attempted but could not score or complete. Represented in the normalized results rather than dropped. +- **Raw azd Result**: The unmodified azd command output retained alongside the normalized artifacts for troubleshooting and audit. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A Foundry hosted agent with a current-surface recipe can be evaluated end to end through `agentops eval run` and produces a normalized result set and a human-readable report, with no manual post-processing. +- **SC-002**: A single `agentops eval run` invocation against a current-surface recipe causes the cloud evaluation to execute exactly once; retrieving results never re-executes the evaluation. +- **SC-003**: The row and metric structure of a current-surface run is identical in shape and field names to a legacy azd run, a local run, and a cloud run, so downstream reporting, comparison, evidence, and readiness tooling require no branching on which surface produced the run. +- **SC-004**: 100% of configured thresholds that cannot be bound to exactly one emitted metric result in a non-passing run; no configuration of thresholds and emitted metrics produces a false pass. +- **SC-005**: Every existing legacy-recipe scenario and its automated coverage continues to pass unchanged against the updated build, with no edits to existing configuration files. +- **SC-010**: A user who initializes a new azd-backed workspace and immediately runs an evaluation never fails because of an uninstallable dependency; initialization only targets a surface whose extension is installable in that environment. +- **SC-006**: For every azd-backed run that fails, an operator can locate the raw azd output and the run identifiers in the run's artifact directory without re-running the evaluation. +- **SC-007**: `agentops eval analyze` correctly identifies the resolved recipe and surface, or the specific gap, in each of these workspace states: current-surface recipe only, legacy recipe only, both surfaces present, and no recipe. +- **SC-009**: Recipe resolution is deterministic: the same workspace always resolves to the same recipe, and whenever a discoverable recipe is skipped the user is told which one was chosen and which was skipped. +- **SC-008**: A new user following the published documentation can determine the required azd version and extension for their chosen surface without reading source code or issue history. + +## Assumptions + +- The Azure Developer CLI and the evaluation extension required by the user's chosen surface are installed and authenticated by the operator. Installing, upgrading, or authenticating azd and its extensions remains out of scope for AgentOps, which only detects their absence and reports it. +- `execution: azd` remains the single configuration value that selects azd-backed execution for both surfaces; no new execution mode value and no new required configuration field are introduced, in line with the existing configuration contract. +- The current surface's supported agent targets remain the same Foundry prompt and Foundry hosted agent targets already accepted for `execution: azd`; targets rejected today remain rejected. +- Foundry remains the system of record for the cloud evaluation run itself. AgentOps reads and normalizes the run's output and does not reimplement evaluation, scoring, or run management. +- The existing `results.json` and `report.md` contracts are sufficient to represent current-surface output; any schema evolution required to represent rubric dimensions or failed samples is additive and backward compatible. +- Threshold binding for the current surface reuses the existing narrow alias rules rather than introducing broader fuzzy matching, so that gate behavior stays predictable and cannot create false-green results. +- The dataset referenced by a current-surface recipe is resolved and supplied by the recipe and the azd evaluation surface; AgentOps does not need to re-upload or re-shape it. +- Both azd evaluation surfaces are expected to coexist for some period, so surface support is additive rather than a migration that removes the legacy path. +- The current surface is strictly opt-in. It activates only when a current-surface recipe is present, so a workspace that does not have one is unaffected by this feature in every respect, including error messages. +- When both surfaces are present in one workspace, the current surface is assumed to represent the team's intended direction, so it is preferred over the legacy recipe. The selection is always reported, so a team that wants the legacy recipe can pin it with an explicit `eval_recipe`. + +## Out of Scope + +- Installing, upgrading, configuring, or authenticating the Azure Developer CLI or any of its extensions. +- Authoring or generating the content of a current-surface recipe beyond the initialization guidance required by FR-016; recipe authoring belongs to the azd tooling and Foundry guidance. +- Removing, deprecating, or migrating away from the legacy azd evaluation surface. +- Changing the local or cloud execution modes, the threshold expression language, the exit-code contract, or the `results.json` and `report.md` schemas beyond additive changes needed to represent current-surface output. +- Doctor readiness checks, release evidence composition, Cockpit presentation, and CI/CD workflow generation, which consume these results but are specified separately. +- Any change to how Foundry executes, scores, or stores the evaluation run. diff --git a/specs/011-azd-ai-eval-surface/tasks.md b/specs/011-azd-ai-eval-surface/tasks.md new file mode 100644 index 00000000..67747373 --- /dev/null +++ b/specs/011-azd-ai-eval-surface/tasks.md @@ -0,0 +1,281 @@ +# Tasks: Current azd AI Evaluation Surface Support + +**Input**: Design documents from `/specs/011-azd-ai-eval-surface/` + +**Prerequisites**: [plan.md](plan.md), [spec.md](spec.md), [research.md](research.md), [data-model.md](data-model.md), [contracts/azd-eval-surface.md](contracts/azd-eval-surface.md), [quickstart.md](quickstart.md) + +**Tests**: Test tasks are **required**, not optional. Constitution Principle V +mandates focused automated coverage for every behavior change, and Principle III +requires the azd boundary to be mocked so the suite runs without azd, Azure +credentials, or network access. Test tasks are written before their +implementation task in each story. + +**Organization**: Tasks are grouped by user story so each story can be +implemented, tested, and merged independently. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (US1, US2, US3, US4) +- Exact file paths are included in every task + +## Path Conventions + +Single Python project at repository root: `src/agentops/`, `tests/unit/`, +`tests/integration/`, `docs/`. Paths below follow the structure recorded in +plan.md. + +--- + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Establish a clean baseline and the shared test double that every +later phase depends on. + +- [X] T001 Run `python -m pytest tests/ -x -q` and record the green baseline, so any later failure is attributable to this feature rather than pre-existing state +- [X] T002 [P] Add a reusable azd subprocess test double in `tests/fixtures/azd_stub.py` that lets a test script an ordered sequence of `(command, returncode, stdout, stderr)` responses, assert the exact argv of each invocation, and assert that no unexpected command was run + +**Checkpoint**: Baseline green and the azd boundary is fakeable. + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: The recipe layer. Every user story depends on being able to +discover, classify, and parse a current-surface recipe. + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete. + +**Note on parallelism**: T003–T006 all edit `src/agentops/core/azd_eval.py`, so +they are strictly sequential despite being separable concerns. + +- [X] T003 Add the `EvalSurface` enum (`legacy`, `current`) and a content-based `classify_recipe_document()` in `src/agentops/core/azd_eval.py`, applying the rules from data-model.md: root `evals` as a sequence means current; a mapping-valued `agent` or a `dataset_reference` key with no `evals` sequence means legacy; anything else raises `AzdEvalRecipeError` naming the path and stating that the document matches neither supported schema +- [X] T004 Add the current-surface recipe models to `src/agentops/core/azd_eval.py` — `CurrentEvalRecipe`, `CurrentDatasetDecl`, `CurrentEvaluatorDecl`, `CurrentEval`, `CurrentSourceDecl`, `CurrentEvaluatorRef`, `CurrentTarget` — as Pydantic v2 models with `extra="allow"` so unknown preview fields are preserved, coercing every `version` field to string exactly as the existing legacy models do, and reading the dataset path from `file` (never `source`) +- [X] T005 Add `current_recipe_metric_names()` to `src/agentops/core/azd_eval.py` returning every metric name a current recipe can produce: each evaluator reference's `name` when set otherwise its `evaluator`, plus each rubric dimension's metric name resolved from local `CurrentEvaluatorDecl` entries via inline `definition.dimensions` or the referenced `source` file, reusing the existing dimension metric-name rule +- [X] T006 Add `RecipeResolution` and extend discovery in `src/agentops/core/azd_eval.py` to search `evals/azure.eval.yaml` alongside the existing legacy locations, implementing the precedence from contracts: explicit `eval_recipe` wins outright; a cross-surface tie selects the current surface and records the skipped paths; two candidates within one surface raise the existing ambiguity error listing candidates; no candidate raises a configuration error naming both supported locations +- [X] T007 Add unit tests to `tests/unit/test_azd_eval.py` covering the seven workspace states in quickstart T1.2, plus: a current recipe using `datasets[].file` parses, an unknown top-level key is preserved rather than rejected, and declared metric names include builtin references, evaluator labels, and rubric dimension ids +- [X] T008 Add a shared extension availability probe to `src/agentops/pipeline/azd_runner.py` that parses `azd extension list --installed -o json` and matches on extension `id`, falling back to the existing text scan only when the structured form is unavailable, and accepts the extension id as a parameter so both surfaces reuse it +- [X] T009 Add unit tests to `tests/unit/test_azd_runner.py` for the probe: a structured listing containing the id means available; a listing omitting it means unavailable; and a human-readable listing that contains the id with a not-installed status means **unavailable** — the false-positive guard from quickstart T1.4. Existing legacy tests in this file must pass unmodified + +**Checkpoint**: A recipe can be found, classified, parsed, and reduced to its +declared metric names, and extension presence can be determined reliably. + +--- + +## Phase 3: User Story 1 - Evaluate through the current azd surface (Priority: P1) 🎯 MVP + +**Goal**: `agentops eval run` against `evals/azure.eval.yaml` delegates the +evaluation to the current azd surface, executes it exactly once, and produces +normalized `results.json` and `report.md`. + +**Independent Test**: With the azd boundary faked, run `agentops eval run` in a +workspace holding a current-surface recipe and confirm the exact command +sequence, a single evaluation execution, and a normalized result set with +computed aggregate metrics and one row per sample. + +### Tests for User Story 1 + +> Write these first and confirm they fail before implementing. + +- [X] T010 [US1] Write failing tests in `tests/unit/test_azd_eval_runner.py` for the command sequence and flag correctness per contracts: probe, `create`, `run start --no-wait`, poll, final `run show`, `run output list --all --output-file`; assert `--path` receives the recipe's parent directory, `run start` receives no positional argument, `run output list` receives the run id positionally, the current surface uses `--output-file` and never `--out-file`, and no failure-gating flag is ever passed +- [X] T011 [US1] Write failing tests in `tests/unit/test_azd_eval_runner.py` for polling: non-terminal statuses followed by `completed` resolve; each of `failed`, `error`, `canceled`, and `cancelled` is terminal and raises a runtime error; an unrecognized status keeps polling; and a timeout raises a runtime error whose message contains the evaluation id, run id, and last observed status +- [X] T012 [US1] Write failing tests in `tests/unit/test_azd_eval_runner.py` for output parsing and aggregation: a metric mean excludes samples with no score rather than treating them as zero; a metric with no scores anywhere is absent from the aggregate map; a score arriving as the string `"4"` decodes to `4.0`; a non-numeric score is recorded as absent; `passed: null` is neither a pass nor a judged failure; a sample with an empty result list is a failure; a run error object with null members is not a failure while a non-empty `error.message` is; and `report_url` wins over `portal_url` +- [X] T013 [US1] Write failing tests in `tests/unit/test_azd_eval_runner.py` asserting that raw artifacts — `azd_evaluation.json`, `azd_eval_output_items.json`, `azd_stdout.log`, `azd_stderr.log` — are written to the run output directory for a successful run and for a failed run + +### Implementation for User Story 1 + +- [X] T014 [US1] Create `src/agentops/pipeline/azd_eval_runner.py` with the `CurrentEvalRun` dataclass, `RunCounts`, `SampleScore`, and the `CURRENT_EXTENSION_NAME` / minimum azd version constants, plus every azd command builder for this surface, so all volatile surface details live in one module +- [X] T015 [US1] Implement the delegated sequence in `src/agentops/pipeline/azd_eval_runner.py`: probe availability via the shared helper from T008 and raise an actionable error naming `azure.ai.evaluations`, its azd version floor, and the install command when absent; run `create` for idempotent reconciliation; run `run start --no-wait` and capture the evaluation and run identifiers; then poll `run show` at the existing heartbeat interval under the AgentOps timeout until a terminal status +- [X] T016 [US1] Implement result retrieval in `src/agentops/pipeline/azd_eval_runner.py`: a final `run show` for the run object and `run output list --all --output-file ` for the per-sample items, reading the items from the written temporary file rather than stdout, and cleaning the temporary file up afterwards +- [X] T017 [US1] Implement per-sample parsing and aggregation in `src/agentops/pipeline/azd_eval_runner.py` per data-model.md: build `SampleScore` entries keyed by `metric` falling back to `name`, decode scores tolerantly, preserve three-valued verdicts, and compute each aggregate metric as the mean of its non-absent scores +- [X] T018 [US1] Implement `normalize_to_results()` in `src/agentops/pipeline/azd_eval_runner.py` producing a `RunResult` with one `RowResult` per sample (best-effort input/expected/response extraction that never affects the gate), `aggregate_metrics` from T017, counts from `RunCounts` cross-checked against retrieved rows, `result_granularity` of `row`, and the extended `config.azd_evaluation` provenance block including `surface`, `extension`, `skipped_recipes`, `result_counts`, `missing_metrics`, `retrieval_warnings`, and `error_message` +- [X] T019 [US1] Implement raw artifact writing in `src/agentops/pipeline/azd_eval_runner.py` for both success and failure paths, matching the existing legacy artifact naming and writing into the run's output directory +- [X] T020 [US1] Update the azd branch in `src/agentops/pipeline/orchestrator.py` to resolve the recipe through `RecipeResolution`, dispatch to `azd_eval_runner` for the current surface and the untouched `azd_runner` for the legacy surface, and emit progress naming the resolved recipe, the selected surface, and any skipped recipe +- [X] T021 [US1] Add an end-to-end test in `tests/integration/test_cli_flat_schema.py` that runs the CLI against a faked azd with a current-surface recipe and asserts exit code `0`, a written `results.json` with populated rows and computed aggregate metrics, a written `report.md`, and exactly one evaluation execution + +**Checkpoint**: A current-surface evaluation runs end to end and produces +normalized artifacts. This is the MVP. + +--- + +## Phase 4: User Story 2 - Gate the release on thresholds and rubric dimensions (Priority: P2) + +**Goal**: Thresholds bind to emitted metrics and rubric dimensions, unbound and +missing metrics fail closed, baseline comparison works, and exit codes follow the +project contract. + +**Independent Test**: Run the same faked evaluation with a satisfied threshold, +an unsatisfied threshold, a threshold naming an undeclared metric, and a +threshold naming a declared-but-unemitted metric, asserting the exit code for +each. + +### Tests for User Story 2 + +- [X] T022 [US2] Write failing tests in `tests/unit/test_azd_eval_runner.py` for pre-flight binding: a threshold matching no declared metric and a threshold matching more than one declared metric each raise a configuration error, and in both cases the faked azd records **zero invocations** — the observable proof that a typo never bills a cloud run +- [X] T023 [US2] Write failing tests in `tests/unit/test_azd_eval_runner.py` for post-run behavior: a threshold bound to a declared metric that the run did not emit is recorded as a failed `ThresholdEvaluation` and appears in `config.azd_evaluation.missing_metrics`, the run does not report a pass, and a run with zero samples or no decodable metrics never reports a pass +- [X] T024 [US2] Write failing tests in `tests/unit/test_azd_eval_runner.py` asserting that thresholds naming rubric dimension ids bind to the corresponding per-dimension scores and are gated identically to builtin evaluator metrics + +### Implementation for User Story 2 + +- [X] T025 [US2] Implement pre-flight threshold binding in `src/agentops/pipeline/azd_eval_runner.py`, binding configured threshold keys against the declared metric names from T005 using the existing `bind_threshold_metrics` helper, raising a configuration error before any azd command runs when a key is unmatched or ambiguous, and listing the candidates for the ambiguous case +- [X] T026 [US2] Implement post-run gate handling in `src/agentops/pipeline/azd_eval_runner.py`: a declared metric absent from `aggregate_metrics` produces a failed `ThresholdEvaluation` rather than an exception, `overall_passed` requires both a `completed` status and a full threshold pass rate, and a zero-sample or zero-metric run cannot pass +- [X] T027 [US2] Add integration coverage in `tests/integration/test_cli_flat_schema.py` for the full exit-code matrix from quickstart T1.5 — `0` for a satisfied gate, `2` for an unsatisfied threshold, `2` for a declared-but-unemitted metric, `1` for each pre-flight binding failure with zero azd invocations, `1` for each non-`completed` terminal status, `1` for a poll timeout, and `1` when azd or the extension is missing +- [X] T028 [US2] Add a test in `tests/integration/test_cli_flat_schema.py` confirming `--baseline` comparison against a current-surface run behaves identically to any other execution mode + +**Checkpoint**: The release gate is trustworthy and fails closed in every +identified failure mode. + +--- + +## Phase 5: User Story 3 - Keep existing legacy azd recipes working unchanged (Priority: P3) + +**Goal**: Existing adopters upgrade with no configuration changes and no +behavioral difference, and nobody is opted in to the new surface by accident. + +**Independent Test**: Run the pre-existing legacy test suite unmodified against +the updated build, and run the CLI in a legacy workspace confirming identical +commands, normalization, and exit code. + +- [X] T029 [US3] Verify by inspection and by test that `src/agentops/pipeline/azd_runner.py` retains its original discovery, command sequence, post-run binding semantics, and `result_granularity` of `aggregate`, with the only changes being the extracted shared availability probe from T008 and any helper reuse that leaves observable behavior identical +- [X] T030 [US3] Confirm every pre-existing test in `tests/unit/test_azd_runner.py`, `tests/unit/test_azd_eval.py`, and `tests/unit/test_azd_eval_init.py` passes **without modification**; a test that required editing is a finding to resolve in the implementation, not by changing the test +- [X] T031 [US3] Add a regression guard test in `tests/integration/test_cli_flat_schema.py` proving the current surface is strictly opt-in: in a workspace with a legacy `eval.yaml` and no `evals/` directory, `agentops eval run` and `agentops eval analyze` produce output identical to the previous release, with no mention of the current surface and no new warning or error +- [X] T032 [US3] Add a test in `tests/integration/test_cli_flat_schema.py` confirming that a legacy workspace dispatches to `azd ai agent eval` commands and never to `azd ai eval` commands + +**Checkpoint**: Existing adopters are provably unaffected. + +--- + +## Phase 6: User Story 4 - Diagnose azd-backed setup and failures (Priority: P4) + +**Goal**: `agentops eval analyze` reports the resolved recipe and its surface, +initialization only targets an installable surface, and raw azd output is +available for troubleshooting. + +**Independent Test**: Run `agentops eval analyze` in workspaces with a +current-surface recipe, a legacy recipe, both, and none, and confirm the resolved +recipe and surface or the specific gap in each case. + +- [X] T033 [US4] Write failing tests in `tests/unit/test_eval_analysis.py` for the four workspace states, asserting that the report names the resolved recipe path and surface, or reports the gap and the required action when no recipe resolves, and reports the selection and the skipped path when both surfaces are present +- [X] T034 [US4] Implement an azd recipe readiness signal in `src/agentops/services/eval_analysis.py` that reuses `RecipeResolution` from T006, replacing the current weak `azure.yaml`-presence signal with the resolved recipe path and surface, and reporting resolution errors as actionable gaps rather than raising +- [X] T035 [US4] Write failing tests in `tests/unit/test_azd_eval_init.py` asserting that initialization probes for the current-surface extension and selects the legacy surface when it is absent, selects the current surface when it is present, and never generates `evals/azure.eval.yaml` in an environment where the required extension cannot be installed +- [X] T036 [US4] Implement installable-surface selection in `src/agentops/services/azd_eval_init.py` per plan design decision 13 and spec FR-016, keeping the existing legacy generation path as the default while the current-surface extension is unavailable so a freshly initialized workspace is always immediately runnable +- [X] T037 [US4] Add a test in `tests/unit/test_azd_eval_runner.py` asserting that a run failing at the poll, retrieval, or normalization stage still writes every raw artifact, so a failed run is always diagnosable without re-running the evaluation + +**Checkpoint**: Setup and failure diagnosis are self-service. + +--- + +## Phase 7: Polish & Cross-Cutting Concerns + +**Purpose**: User-visible documentation and final validation. + +- [X] T038 [P] Document both surfaces in `docs/evaluation.md`: extension ids, azd version floors, discovery locations, the cross-surface precedence rule, the threshold binding table, and a plain statement that `azure.ai.evaluations` is preview and not yet in the default azd extension registry +- [X] T039 [P] Update the azd execution path description in `docs/how-it-works.md` to describe two surfaces, the delegated command sequence, and the fact that AgentOps computes aggregate metrics from per-sample scores for the current surface +- [X] T040 [P] Add a current-surface walkthrough to `docs/tutorial-hosted-agent.md` using an `evals/azure.eval.yaml` recipe, including the local-source install path for the preview extension +- [X] T041 [P] Add a `CHANGELOG.md` entry under the unreleased section recording current-surface support, the preserved legacy path, and the preview status of the required extension +- [X] T042 [P] Update comment guidance in `src/agentops/templates/agentops.yaml` so `execution: azd` mentions both recipe locations and the precedence rule +- [X] T043 Run `python -m pytest tests/ -x -q` and confirm the full suite passes with no pre-existing test modified +- [X] T044 Walk the Tier 1 scenarios in [quickstart.md](quickstart.md) (T1.1 through T1.8) and confirm each expectation, in particular the opt-in regression guard and the wording of the unavailable-extension message + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies +- **Foundational (Phase 2)**: Depends on Setup — **blocks every user story** +- **US1 (Phase 3)**: Depends on Foundational +- **US2 (Phase 4)**: Depends on US1, because gating operates on the normalized result US1 produces +- **US3 (Phase 5)**: Depends on Foundational only; can run in parallel with US1 and US2 +- **US4 (Phase 6)**: Depends on Foundational for T033–T036; T037 depends on US1 +- **Polish (Phase 7)**: Depends on all desired stories + +### User Story Dependencies + +- **US1 (P1)**: Independent once Foundational completes. This is the MVP. +- **US2 (P2)**: The one genuine cross-story dependency — it gates the result US1 normalizes. Sequential after US1. +- **US3 (P3)**: Fully independent. It is verification and guard work over code the other stories must not disturb, so it can start as soon as Foundational lands and acts as a continuous safety net. +- **US4 (P4)**: Largely independent. Analyze and init work touch different service modules from the adapter. + +### Within Each User Story + +- Tests are written before their implementation task and must fail first +- Core models before services before adapters before orchestration +- Every task in a story completes before that story's checkpoint is claimed + +### Parallel Opportunities + +Genuine parallelism is limited by file ownership rather than by logic: + +- T003–T006 all edit `core/azd_eval.py` and are strictly sequential +- T010–T013 and T022–T024 all edit `tests/unit/test_azd_eval_runner.py` and are strictly sequential +- T014–T019 all edit `pipeline/azd_eval_runner.py` and are strictly sequential +- Across stories: US3 (T029–T032) and US4 (T033–T036) touch different modules from US1/US2 and can proceed concurrently +- All of Phase 7 except T043 and T044 is parallelizable + +## Parallel Example: after Foundational completes + +```bash +# Three developers, no file conflicts: +Developer A: US1 (Phase 3) - src/agentops/pipeline/azd_eval_runner.py +Developer B: US3 (Phase 5) - regression guards in tests/ +Developer C: US4 T033-T036 - src/agentops/services/{eval_analysis,azd_eval_init}.py +``` + +```bash +# Phase 7 documentation, all parallel: +Task: "Document both surfaces in docs/evaluation.md" +Task: "Update azd execution path in docs/how-it-works.md" +Task: "Add current-surface walkthrough to docs/tutorial-hosted-agent.md" +Task: "Add CHANGELOG.md entry" +Task: "Update templates/agentops.yaml comments" +``` + +--- + +## Implementation Strategy + +### MVP scope + +**Phases 1, 2, and 3 (T001–T021).** That delivers a Foundry hosted agent +evaluated end to end through the current azd surface, producing normalized +`results.json` and `report.md`. It satisfies the primary acceptance criterion of +issue #484 and unblocks the VBD Evaluate lab. + +Thresholds still function in the MVP through the existing evaluation path; US2 +adds the fail-closed guarantees and the pre-flight check that make the gate +trustworthy for CI. + +### Recommended delivery order + +1. **Phases 1–2** — foundation; nothing user-visible yet +2. **Phase 3 (US1)** — MVP; stop and validate against quickstart Tier 1 +3. **Phase 5 (US3)** — land the regression guards early so every later change is protected; cheap and independent +4. **Phase 4 (US2)** — make the gate trustworthy; required before recommending this path in CI +5. **Phase 6 (US4)** — diagnosis and safe initialization +6. **Phase 7** — documentation and final validation + +Landing US3 before US2 is deliberate: the guards are inexpensive, and having them +in place makes every subsequent change provably non-breaking for existing +adopters. + +### Do not ship without + +- **T031** — the opt-in regression guard. Without it there is no proof that + existing users and fresh clones are unaffected. +- **T036** — installable-surface selection at init. Without it a freshly + initialized workspace could fail on a dependency the user cannot install. +- **T027** — the exit-code matrix. Exit codes are a public contract. + +--- + +## Notes + +- `[P]` marks tasks in different files with no incomplete dependencies +- Every azd command construction and every external JSON field name belongs in + `src/agentops/pipeline/azd_eval_runner.py`, so an upstream surface change stays + a localized edit +- No test may reach the network, require azd, or require Azure credentials +- A pre-existing test that needs editing is a finding, not a fix +- Commit after each task or logical group; stop at any checkpoint to validate diff --git a/src/agentops/core/azd_eval.py b/src/agentops/core/azd_eval.py index 008364e9..fded000c 100644 --- a/src/agentops/core/azd_eval.py +++ b/src/agentops/core/azd_eval.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from enum import Enum from pathlib import Path from typing import Any, Dict, Iterable, Optional @@ -13,6 +14,24 @@ EVAL_RECIPE_FILENAMES = ("eval.yaml", "eval.yml") +#: Default directory holding a current-surface recipe, per the +#: ``azure.ai.evaluations`` extension. The basename is a constant upstream; +#: the ``.yml`` spelling is tolerated for convenience. +CURRENT_EVAL_DIRNAME = "evals" +CURRENT_EVAL_FILENAMES = ("azure.eval.yaml", "azure.eval.yml") + + +class EvalSurface(str, Enum): + """Which azd evaluation command family and extension a recipe requires. + + ``legacy`` -> ``azure.ai.agents`` -> ``azd ai agent eval ...`` + ``current`` -> ``azure.ai.evaluations`` -> ``azd ai eval ...`` + """ + + LEGACY = "legacy" + CURRENT = "current" + + class AzdEvalRecipeError(ValueError): """Raised when an azd evaluation recipe cannot be discovered or parsed.""" @@ -178,18 +197,8 @@ def ok(self) -> bool: } -def find_eval_yaml(workspace: Path, explicit_path: Optional[Path] = None) -> Optional[Path]: - """Find an azd ``eval.yaml`` recipe under ``workspace``. - - Discovery is deterministic. A single recipe can live at the workspace root - or under ``src//``. Multiple candidates require an explicit - ``eval_recipe`` path. - """ - - root = workspace.resolve() - if explicit_path is not None: - path = explicit_path if explicit_path.is_absolute() else root / explicit_path - return path.resolve() +def _legacy_candidates(root: Path) -> list[Path]: + """Return legacy recipe candidates, deterministically ordered.""" candidates: list[Path] = [] for filename in EVAL_RECIPE_FILENAMES: @@ -207,11 +216,37 @@ def find_eval_yaml(workspace: Path, explicit_path: Optional[Path] = None) -> Opt if candidate.exists(): candidates.append(candidate) - unique = sorted({candidate.resolve() for candidate in candidates}) + return sorted({candidate.resolve() for candidate in candidates}) + + +def _display_path(path: Path, root: Path) -> str: + try: + return str(path.relative_to(root)) + except ValueError: + return str(path) + + +def find_eval_yaml(workspace: Path, explicit_path: Optional[Path] = None) -> Optional[Path]: + """Find an azd ``eval.yaml`` recipe under ``workspace``. + + Discovery is deterministic. A single recipe can live at the workspace root + or under ``src//``. Multiple candidates require an explicit + ``eval_recipe`` path. + + This function covers the legacy surface only. Cross-surface resolution + lives in :func:`resolve_recipe`. + """ + + root = workspace.resolve() + if explicit_path is not None: + path = explicit_path if explicit_path.is_absolute() else root / explicit_path + return path.resolve() + + unique = _legacy_candidates(root) if not unique: return None if len(unique) > 1: - display = ", ".join(str(path.relative_to(root)) for path in unique) + display = ", ".join(_display_path(path, root) for path in unique) raise AzdEvalRecipeAmbiguous( "multiple azd eval recipes found; set 'eval_recipe' in agentops.yaml " f"to choose one: {display}" @@ -219,6 +254,7 @@ def find_eval_yaml(workspace: Path, explicit_path: Optional[Path] = None) -> Opt return unique[0] + def load_eval_recipe(path: Path) -> EvalRecipe: """Load and validate an azd ``eval.yaml`` recipe.""" @@ -306,3 +342,467 @@ def bind_threshold_metrics( ambiguous=ambiguous, unused_metrics=unused, ) + + +# --------------------------------------------------------------------------- +# Current surface (``azure.ai.evaluations`` / ``azd ai eval``) +# --------------------------------------------------------------------------- + + +class CurrentDatasetDecl(BaseModel): + """A ``datasets[]`` entry in ``evals/azure.eval.yaml``. + + The local file key is ``file``. ``source`` is *not* valid here; that + spelling belongs to ``evals[].source``. + """ + + name: str + file: Optional[str] = None + version: Optional[str] = None + + model_config = ConfigDict(extra="allow") + + @field_validator("version", mode="before") + @classmethod + def _version_to_string(cls, value: Any) -> Optional[str]: + if value is None: + return None + return str(value) + + +class CurrentEvaluatorDecl(BaseModel): + """An ``evaluators[]`` entry: a locally declared evaluator, usually a rubric.""" + + name: str + source: Optional[str] = None + definition: Optional[Dict[str, Any]] = None + version: Optional[str] = None + + model_config = ConfigDict(extra="allow") + + @field_validator("version", mode="before") + @classmethod + def _version_to_string(cls, value: Any) -> Optional[str]: + if value is None: + return None + return str(value) + + +class CurrentEvaluatorRef(BaseModel): + """An ``evals[].evaluators[]`` entry.""" + + evaluator: str + name: Optional[str] = None + version: Optional[str] = None + initialization_parameters: Dict[str, Any] = Field(default_factory=dict) + data_mapping: Dict[str, Any] = Field(default_factory=dict) + + model_config = ConfigDict(extra="allow") + + @model_validator(mode="before") + @classmethod + def _coerce_shorthand(cls, data: Any) -> Any: + if isinstance(data, str): + return {"evaluator": data} + return data + + @field_validator("evaluator") + @classmethod + def _evaluator_non_empty(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("azd eval evaluator reference must be non-empty") + return value + + @field_validator("version", mode="before") + @classmethod + def _version_to_string(cls, value: Any) -> Optional[str]: + if value is None: + return None + return str(value) + + @property + def metric_name(self) -> str: + """Return the metric key this reference produces.""" + + if self.name and self.name.strip(): + return self.name.strip() + return self.evaluator + + +class CurrentSourceDecl(BaseModel): + """An ``evals[].source`` block: trace-sourced or response-sourced input.""" + + type: str + agent_name: Optional[str] = None + agent_version: Optional[str] = None + lookback_hours: Optional[int] = None + max_traces: Optional[int] = None + max_turns: Optional[int] = None + response_ids: list[str] = Field(default_factory=list) + start_time: Optional[str] = None + end_time: Optional[str] = None + + model_config = ConfigDict(extra="allow") + + @field_validator("agent_version", mode="before") + @classmethod + def _version_to_string(cls, value: Any) -> Optional[str]: + if value is None: + return None + return str(value) + + def describe(self) -> str: + """Return a short human-readable provenance description.""" + + parts = [f"{self.type}"] + if self.agent_name: + parts.append(f"agent={self.agent_name}") + if self.agent_version: + parts.append(f"version={self.agent_version}") + if self.lookback_hours is not None: + parts.append(f"lookback_hours={self.lookback_hours}") + if self.max_traces is not None: + parts.append(f"max_traces={self.max_traces}") + return f"azd:{' '.join(parts)}" + + +class CurrentTarget(BaseModel): + """An ``evals[].target`` block.""" + + type: str + name: str + + model_config = ConfigDict(extra="allow") + + +class CurrentEval(BaseModel): + """An ``evals[]`` entry.""" + + name: str + id: Optional[str] = None + description: Optional[str] = None + dataset: Optional[str] = None + source: Optional[CurrentSourceDecl] = None + evaluation_level: Optional[str] = None + max_samples: Optional[int] = None + evaluators: list[CurrentEvaluatorRef] = Field(default_factory=list) + target: Optional[CurrentTarget] = None + + model_config = ConfigDict(extra="allow") + + @field_validator("name") + @classmethod + def _name_non_empty(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("azd eval name must be non-empty") + return value + + +class CurrentEvalRecipe(BaseModel): + """Tolerant model for ``evals/azure.eval.yaml``. + + Unknown fields are preserved so that additive upstream changes to the + preview schema do not turn into AgentOps outages. + """ + + datasets: list[CurrentDatasetDecl] = Field(default_factory=list) + evaluators: list[CurrentEvaluatorDecl] = Field(default_factory=list) + evals: list[CurrentEval] = Field(default_factory=list) + + model_config = ConfigDict(extra="allow") + + def primary_eval(self) -> CurrentEval: + """Return the single declared evaluation. + + AgentOps runs exactly one evaluation per invocation, so a recipe + declaring several requires the user to say which one. + """ + + if not self.evals: + raise AzdEvalRecipeError( + "azd eval recipe declares no evaluations under 'evals:'. " + "Add one evaluation, or point 'eval_recipe:' at a recipe that has one." + ) + if len(self.evals) > 1: + names = ", ".join(item.name for item in self.evals) + raise AzdEvalRecipeAmbiguous( + "azd eval recipe declares multiple evaluations and AgentOps runs one " + f"per invocation: {names}. Split them into separate recipes and select " + "one with 'eval_recipe:' in agentops.yaml." + ) + return self.evals[0] + + +def classify_recipe_document( + data: Any, + *, + path: Optional[Path] = None, + hint: Optional[EvalSurface] = None, +) -> EvalSurface: + """Classify a parsed recipe document by content. + + Content always wins. ``hint`` (derived from the discovery location) is used + only when the document is structurally inconclusive, so a minimal legacy + recipe found at a legacy location still classifies deterministically. + """ + + location = f" at {path}" if path is not None else "" + if not isinstance(data, dict): + raise AzdEvalRecipeError( + f"azd eval recipe{location} is not a YAML mapping, so AgentOps cannot " + "determine which azd evaluation surface it belongs to." + ) + if isinstance(data.get("evals"), list): + return EvalSurface.CURRENT + if isinstance(data.get("agent"), dict) or "dataset_reference" in data: + return EvalSurface.LEGACY + if hint is not None: + return hint + raise AzdEvalRecipeError( + f"azd eval recipe{location} matches neither supported schema. " + "A current-surface recipe has a sequence-valued 'evals:' key; a legacy " + "recipe has a mapping-valued 'agent:' key or a 'dataset_reference:' key." + ) + + +def _surface_hint_for(path: Path) -> Optional[EvalSurface]: + if path.name in CURRENT_EVAL_FILENAMES: + return EvalSurface.CURRENT + if path.name in EVAL_RECIPE_FILENAMES: + return EvalSurface.LEGACY + return None + + +def classify_recipe_file(path: Path) -> EvalSurface: + """Load ``path`` and classify which azd evaluation surface it targets.""" + + try: + data = load_yaml(path) + except Exception as exc: # noqa: BLE001 - surfaced as a config error + raise AzdEvalRecipeError(f"could not read azd eval recipe {path}: {exc}") from exc + return classify_recipe_document(data, path=path, hint=_surface_hint_for(path)) + + +def load_current_eval_recipe(path: Path) -> CurrentEvalRecipe: + """Load and validate a current-surface ``azure.eval.yaml`` recipe.""" + + try: + data = load_yaml(path) + return CurrentEvalRecipe.model_validate(data) + except ValidationError as exc: + raise AzdEvalRecipeError(f"invalid azd eval recipe {path}: {exc}") from exc + + +def _resolve_relative(reference: str, base_dir: Path) -> Path: + candidate = Path(reference) + if not candidate.is_absolute(): + candidate = base_dir / candidate + return candidate + + +def _load_dimensions_document(path: Path) -> list[EvalRubricDimension]: + if not path.exists(): + raise AzdEvalRecipeError( + f"azd eval recipe references a rubric definition that does not exist: {path}. " + "Commit the rubric file or remove the reference." + ) + try: + data = load_yaml(path) + except Exception as exc: # noqa: BLE001 - surfaced as a config error + raise AzdEvalRecipeError(f"could not read rubric definition {path}: {exc}") from exc + return _dimensions_from_mapping(data) + + +def _dimensions_from_mapping(data: Any) -> list[EvalRubricDimension]: + if not isinstance(data, dict): + return [] + raw = data.get("dimensions") + if not isinstance(raw, list): + return [] + dimensions: list[EvalRubricDimension] = [] + for entry in raw: + if isinstance(entry, dict): + try: + dimensions.append(EvalRubricDimension.model_validate(entry)) + except ValidationError: + continue + return dimensions + + +def current_evaluator_dimensions( + declaration: CurrentEvaluatorDecl, + recipe_dir: Path, +) -> list[EvalRubricDimension]: + """Return the rubric dimensions a locally declared evaluator produces.""" + + definition = declaration.definition + if isinstance(definition, dict): + ref = definition.get("$ref") + if isinstance(ref, str) and ref.strip(): + return _load_dimensions_document(_resolve_relative(ref.strip(), recipe_dir)) + dimensions = _dimensions_from_mapping(definition) + if dimensions: + return dimensions + if declaration.source and declaration.source.strip(): + return _load_dimensions_document(_resolve_relative(declaration.source.strip(), recipe_dir)) + return [] + + +def current_recipe_metric_names(recipe: CurrentEvalRecipe, recipe_path: Path) -> set[str]: + """Return every metric name a current-surface recipe can produce. + + This is the set that pre-flight threshold binding validates against, so it + must cover builtin evaluator references, evaluator labels, and the rubric + dimensions of locally declared evaluators. + """ + + recipe_dir = recipe_path.parent + declared = { + declaration.name: declaration + for declaration in recipe.evaluators + if declaration.name + } + + names: set[str] = set() + for evaluation in recipe.evals: + for reference in evaluation.evaluators: + metric = reference.metric_name + if metric: + names.add(metric) + declaration = declared.get(reference.evaluator) + if declaration is None: + continue + for dimension in current_evaluator_dimensions(declaration, recipe_dir): + dimension_name = dimension.metric_name + if dimension_name: + names.add(dimension_name) + return names + + +def current_recipe_dataset_path( + recipe: CurrentEvalRecipe, + evaluation: CurrentEval, + recipe_path: Path, +) -> str: + """Describe the data an evaluation reads, for result provenance.""" + + if evaluation.dataset: + for declaration in recipe.datasets: + if declaration.name != evaluation.dataset: + continue + if declaration.file: + return str(_resolve_relative(declaration.file, recipe_path.parent)) + return declaration.name + return evaluation.dataset + if evaluation.source is not None: + return evaluation.source.describe() + return "" + + +# --------------------------------------------------------------------------- +# Cross-surface discovery +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class RecipeResolution: + """The recipe AgentOps selected, and what it passed over to get there.""" + + path: Path + surface: EvalSurface + skipped: tuple[Path, ...] = () + explicit: bool = False + + @property + def extension(self) -> str: + """Return the azd extension id this recipe's surface requires.""" + + if self.surface is EvalSurface.CURRENT: + return "azure.ai.evaluations" + return "azure.ai.agents" + + +def _current_candidates(root: Path) -> list[Path]: + eval_dir = root / CURRENT_EVAL_DIRNAME + if not eval_dir.is_dir(): + return [] + candidates = [ + eval_dir / filename + for filename in CURRENT_EVAL_FILENAMES + if (eval_dir / filename).exists() + ] + return sorted({candidate.resolve() for candidate in candidates}) + + +def resolve_recipe(workspace: Path, explicit_path: Optional[Path] = None) -> RecipeResolution: + """Resolve which azd evaluation recipe a run should use. + + Precedence, per the feature contract: + + 1. An explicit ``eval_recipe`` wins outright. + 2. Candidates from both surfaces select the current surface, recording the + skipped legacy paths so the choice is reported rather than silent. + 3. More than one candidate within a single surface is ambiguous. + 4. No candidate is a configuration error naming both supported locations. + """ + + root = workspace.resolve() + + if explicit_path is not None: + path = _resolve_relative(str(explicit_path), root).resolve() + if not path.exists(): + raise AzdEvalRecipeError( + f"azd eval recipe not found at {path}. Update 'eval_recipe:' in " + "agentops.yaml, or remove it to use auto-discovery." + ) + return RecipeResolution( + path=path, + surface=classify_recipe_file(path), + skipped=(), + explicit=True, + ) + + current = _current_candidates(root) + legacy = _legacy_candidates(root) + + if len(current) > 1: + display = ", ".join(_display_path(path, root) for path in current) + raise AzdEvalRecipeAmbiguous( + "multiple azd eval recipes found; set 'eval_recipe' in agentops.yaml " + f"to choose one: {display}" + ) + + if current: + return RecipeResolution( + path=current[0], + surface=classify_recipe_file(current[0]), + skipped=tuple(legacy), + explicit=False, + ) + + if len(legacy) > 1: + display = ", ".join(_display_path(path, root) for path in legacy) + raise AzdEvalRecipeAmbiguous( + "multiple azd eval recipes found; set 'eval_recipe' in agentops.yaml " + f"to choose one: {display}" + ) + + if legacy: + return RecipeResolution( + path=legacy[0], + surface=classify_recipe_file(legacy[0]), + skipped=(), + explicit=False, + ) + + raise AzdEvalRecipeError( + "azd eval recipe not found. AgentOps looks for " + f"'{CURRENT_EVAL_DIRNAME}/{CURRENT_EVAL_FILENAMES[0]}' (azd ai eval) and " + "'eval.yaml' at the workspace root or under 'src//' " + "(azd ai agent eval). Generate one with `agentops eval init`, or set " + "'eval_recipe:' in agentops.yaml. If you want the AgentOps local engine " + "instead, set 'execution: local'." + ) + diff --git a/src/agentops/pipeline/azd_eval_runner.py b/src/agentops/pipeline/azd_eval_runner.py new file mode 100644 index 00000000..fd6f2d52 --- /dev/null +++ b/src/agentops/pipeline/azd_eval_runner.py @@ -0,0 +1,889 @@ +"""Subprocess adapter for the current ``azd ai eval`` surface. + +This module owns every command construction and every external JSON field name +for the ``azure.ai.evaluations`` extension, so an upstream change to that +preview surface stays a localized edit. + +Key differences from the legacy ``azd ai agent eval`` adapter in +:mod:`agentops.pipeline.azd_runner`: + +* The run object exposes only *counts*, never aggregate numeric metrics, so + AgentOps computes each metric as the mean of its per-sample scores. +* Per-sample output is retrieved and normalized into ``RunResult.rows``, where + the legacy adapter emits an aggregate-only result. +* The run is submitted with ``--no-wait`` and polled by AgentOps. The surface's + blocking mode has an internal wait budget that, when it expires, exits zero + with an unfinished run and a differently shaped payload. +* AgentOps never passes a failure-gating flag; the release gate stays here. +""" + +from __future__ import annotations + +import json +import math +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Callable, Dict, Iterable, Optional, Sequence + +from agentops.core.agentops_config import AgentOpsConfig, Threshold, classify_agent +from agentops.core.azd_eval import ( + CurrentEval, + CurrentEvalRecipe, + MetricBinding, + bind_threshold_metrics, + current_recipe_dataset_path, +) +from agentops.core.results import ( + RowMetric, + RowResult, + RunResult, + RunSummary, + TargetInfo, +) +from agentops.pipeline import thresholds +from agentops.pipeline.azd_runner import ( + AZD_PROGRESS_INTERVAL_SECONDS, + AzdBackendError, + _format_command_failure, + _parse_json_object, + _persist_failure_logs, + _run_command, + probe_extension, +) + + +#: azd extension providing the current evaluation surface. +CURRENT_EXTENSION_NAME = "azure.ai.evaluations" + +#: Minimum azd version the extension manifest requires. +CURRENT_MIN_AZD_VERSION = "1.27.1" + +CURRENT_EVAL_TIMEOUT_SECONDS = 1800.0 +CURRENT_POLL_INTERVAL_SECONDS = 10.0 + +#: Statuses that end a run. Both spellings of cancelled occur upstream, and +#: ``error`` is distinct from ``failed``. +TERMINAL_SUCCESS_STATUSES = frozenset({"completed"}) +TERMINAL_FAILURE_STATUSES = frozenset({"failed", "error", "canceled", "cancelled"}) +TERMINAL_STATUSES = TERMINAL_SUCCESS_STATUSES | TERMINAL_FAILURE_STATUSES + +#: Per-sample outcome vocabulary. +SAMPLE_OUTCOMES = frozenset({"passed", "failed", "errored", "skipped"}) + +_INPUT_KEYS = ("input", "query", "question", "prompt", "user_input") +_EXPECTED_KEYS = ( + "expected", + "expected_response", + "expected_output", + "ground_truth", + "reference", +) +_RESPONSE_KEYS = ( + "response", + "output", + "answer", + "completion", + "output_text", + "sample.output_text", +) + + +class CurrentSurfaceUnavailable(AzdBackendError): + """Raised when azd or the evaluations extension cannot be used.""" + + +@dataclass(frozen=True) +class RunCounts: + """Sample tallies reported by the run object.""" + + total: int = 0 + passed: int = 0 + failed: int = 0 + errored: int = 0 + skipped: int = 0 + + def as_dict(self) -> Dict[str, int]: + return { + "total": self.total, + "passed": self.passed, + "failed": self.failed, + "errored": self.errored, + "skipped": self.skipped, + } + + +@dataclass(frozen=True) +class SampleScore: + """One metric score for one sample. + + ``score`` is ``None`` when absent or undecodable; it is never coerced to + zero. ``passed`` is three-valued: ``True`` pass, ``False`` judged failure, + ``None`` not judged. + """ + + metric: str + score: Optional[float] = None + passed: Optional[bool] = None + label: Optional[str] = None + reason: Optional[str] = None + + +@dataclass(frozen=True) +class SampleResult: + """One normalized per-sample output item.""" + + index: int + identifier: Optional[str] + outcome: str + scores: tuple[SampleScore, ...] + input_text: str = "" + expected_text: Optional[str] = None + response_text: str = "" + raw: Dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CurrentEvalRun: + """Everything captured from one delegated evaluation.""" + + recipe_path: Path + run_id: str + status: str + run_payload: Dict[str, Any] + output_items: tuple[Dict[str, Any], ...] + eval_id: Optional[str] = None + eval_name: Optional[str] = None + report_url: Optional[str] = None + error_message: Optional[str] = None + stdout: str = "" + stderr: str = "" + duration_seconds: float = 0.0 + + +# --------------------------------------------------------------------------- +# Command construction +# --------------------------------------------------------------------------- + + +def _command(*args: str) -> list[str]: + """Build a non-interactive azd invocation requesting structured output.""" + + return ["azd", "--no-prompt", *args, "-o", "json"] + + +def _require_success( + label: str, + completed: Any, + command: list[str], + debug_dir: Optional[Path], + step: str, +) -> None: + if completed.returncode == 0: + return + debug_paths = _persist_failure_logs( + debug_dir, step=step, completed=completed, command=command + ) + raise AzdBackendError( + _format_command_failure(label, completed, command=command, debug_paths=debug_paths) + ) + + +def _unavailable_error() -> CurrentSurfaceUnavailable: + return CurrentSurfaceUnavailable( + "The current azd evaluation surface is not available. It needs the Azure " + f"Developer CLI {CURRENT_MIN_AZD_VERSION} or newer plus the " + f"`{CURRENT_EXTENSION_NAME}` extension:\n" + f" azd extension install {CURRENT_EXTENSION_NAME}\n" + f"`{CURRENT_EXTENSION_NAME}` is in preview and is not yet published to the " + "default azd extension registry, so the install may not resolve until it " + "ships. Until then, use a legacy `eval.yaml` recipe, or set " + "`execution: local` to run the AgentOps engine instead." + ) + + +# --------------------------------------------------------------------------- +# Execution +# --------------------------------------------------------------------------- + + +def run_current_eval( + recipe_path: Path, + evaluation: CurrentEval, + *, + workspace: Path, + progress: Optional[Callable[[str], None]] = None, + timeout_seconds: float = CURRENT_EVAL_TIMEOUT_SECONDS, + poll_interval_seconds: float = CURRENT_POLL_INTERVAL_SECONDS, + debug_dir: Optional[Path] = None, + sleep: Callable[[float], None] = time.sleep, +) -> CurrentEvalRun: + """Reconcile, start, poll, and read back one current-surface evaluation.""" + + notify = progress or (lambda _message: None) + if not probe_extension(CURRENT_EXTENSION_NAME, cwd=workspace): + raise _unavailable_error() + + recipe_dir = str(recipe_path.parent) + started = time.perf_counter() + stdout_parts: list[str] = [] + stderr_parts: list[str] = [] + + def _record(completed: Any) -> None: + if completed.stdout: + stdout_parts.append(completed.stdout) + if completed.stderr: + stderr_parts.append(completed.stderr) + + # 1. Reconcile the declared datasets, evaluators, and evaluation. + notify("azd ai eval create: reconciling the evaluation definition.") + create_command = _command("ai", "eval", "create", "--path", recipe_dir) + created = _run_command(create_command, cwd=workspace, timeout_seconds=timeout_seconds) + _record(created) + _require_success("azd ai eval create", created, create_command, debug_dir, "azd_eval_create") + + # 2. Submit without waiting, so identifiers arrive in one predictable shape. + notify(f"azd ai eval run start: submitting '{evaluation.name}'.") + start_command = _command( + "ai", + "eval", + "run", + "start", + "--eval", + evaluation.name, + "--path", + recipe_dir, + "--no-wait", + ) + started_run = _run_command(start_command, cwd=workspace, timeout_seconds=timeout_seconds) + _record(started_run) + _require_success( + "azd ai eval run start", started_run, start_command, debug_dir, "azd_eval_run_start" + ) + + handoff = _parse_json_object(started_run.stdout) + run_id = _first_str(handoff, ("run_id", "runId", "id")) + eval_id = _first_str(handoff, ("eval_id", "evalId")) + eval_name = _first_str(handoff, ("eval_name", "evalName")) or evaluation.name + if not run_id: + raise AzdBackendError( + "azd ai eval run start did not return a run identifier, so AgentOps " + "cannot retrieve results. Inspect the run with `azd ai eval run list`." + ) + + # 3. Poll to a terminal state under the AgentOps timeout. + def _partial_run(status: str, payload: Dict[str, Any]) -> CurrentEvalRun: + return CurrentEvalRun( + recipe_path=recipe_path, + run_id=run_id, + status=status, + run_payload=payload, + output_items=(), + eval_id=eval_id or _first_str(payload, ("eval_id", "evalId")), + eval_name=eval_name, + report_url=_first_str(payload, ("report_url", "reportUrl")) + or _first_str(payload, ("portal_url", "portalUrl")), + error_message=_run_error_message(payload), + stdout="\n".join(stdout_parts), + stderr="\n".join(stderr_parts), + duration_seconds=time.perf_counter() - started, + ) + + poll_state: Dict[str, Any] = {"status": "", "payload": {}} + try: + run_payload, status = _poll_until_terminal( + run_id, + recipe_dir=recipe_dir, + workspace=workspace, + timeout_seconds=timeout_seconds, + poll_interval_seconds=poll_interval_seconds, + notify=notify, + record=_record, + debug_dir=debug_dir, + sleep=sleep, + eval_id=eval_id, + state=poll_state, + ) + except AzdBackendError: + # The evaluation exists in Foundry even though AgentOps could not finish + # reading it. Persist what we have so the failure stays diagnosable + # without re-running the evaluation. + if debug_dir is not None: + write_raw_artifacts( + _partial_run(poll_state["status"], poll_state["payload"]), debug_dir + ) + raise + + eval_id = eval_id or _first_str(run_payload, ("eval_id", "evalId")) + report_url = _first_str(run_payload, ("report_url", "reportUrl")) or _first_str( + run_payload, ("portal_url", "portalUrl") + ) + error_message = _run_error_message(run_payload) + + # 4. Read every per-sample item. `--all` plus `--output-file` defeats the + # default page bound; the bare-array JSON form drops paging metadata. + output_items: tuple[Dict[str, Any], ...] = () + try: + with TemporaryDirectory() as temp_dir: + items_path = Path(temp_dir) / "azd-eval-output-items.json" + list_command = _command( + "ai", + "eval", + "run", + "output", + "list", + run_id, + "--all", + "--output-file", + str(items_path), + "--path", + recipe_dir, + ) + listed = _run_command(list_command, cwd=workspace, timeout_seconds=timeout_seconds) + _record(listed) + _require_success( + "azd ai eval run output list", + listed, + list_command, + debug_dir, + "azd_eval_run_output_list", + ) + output_items = _load_output_items(items_path, listed.stdout) + except AzdBackendError: + if debug_dir is not None: + write_raw_artifacts(_partial_run(status, run_payload), debug_dir) + raise + + + return CurrentEvalRun( + recipe_path=recipe_path, + run_id=run_id, + status=status, + run_payload=run_payload, + output_items=output_items, + eval_id=eval_id, + eval_name=eval_name, + report_url=report_url, + error_message=error_message, + stdout="\n".join(stdout_parts), + stderr="\n".join(stderr_parts), + duration_seconds=time.perf_counter() - started, + ) + + +def _poll_until_terminal( + run_id: str, + *, + recipe_dir: str, + workspace: Path, + timeout_seconds: float, + poll_interval_seconds: float, + notify: Callable[[str], None], + record: Callable[[Any], None], + debug_dir: Optional[Path], + sleep: Callable[[float], None], + eval_id: Optional[str], + state: Optional[Dict[str, Any]] = None, +) -> tuple[Dict[str, Any], str]: + """Poll ``run show`` until the run ends, or the AgentOps timeout expires. + + ``state`` is updated in place with the last observed status and payload so + the caller can persist diagnostics if this raises. + """ + + show_command = _command( + "ai", "eval", "run", "show", run_id, "--path", recipe_dir + ) + deadline = time.monotonic() + timeout_seconds + next_heartbeat = time.monotonic() + AZD_PROGRESS_INTERVAL_SECONDS + status = "" + payload: Dict[str, Any] = {} + + while True: + completed = _run_command( + show_command, cwd=workspace, timeout_seconds=timeout_seconds + ) + record(completed) + _require_success( + "azd ai eval run show", completed, show_command, debug_dir, "azd_eval_run_show" + ) + payload = _parse_json_object(completed.stdout) + status = _status_of(payload) + if state is not None: + state["status"] = status + state["payload"] = payload + if status in TERMINAL_STATUSES: + break + + now = time.monotonic() + if now >= deadline: + raise AzdBackendError( + "azd evaluation run did not reach a terminal state within " + f"{timeout_seconds:g}s. The run is still recorded in Foundry:\n" + f" eval id: {eval_id or '(unknown)'}\n" + f" run id: {run_id}\n" + f" last observed status: {status or '(none reported)'}\n" + f"Reattach with `azd ai eval run show {run_id} --wait`." + ) + if now >= next_heartbeat: + elapsed = timeout_seconds - (deadline - now) + notify( + f"azd ai eval run: still running ({elapsed / 60:.1f} min elapsed, " + f"status {status or 'pending'})." + ) + next_heartbeat = now + AZD_PROGRESS_INTERVAL_SECONDS + sleep(poll_interval_seconds) + + if status in TERMINAL_FAILURE_STATUSES: + detail = _run_error_message(payload) + raise AzdBackendError( + f"azd evaluation run ended with status '{status}'.\n" + f" eval id: {eval_id or '(unknown)'}\n" + f" run id: {run_id}\n" + + (f" error: {detail}\n" if detail else "") + + "AgentOps did not evaluate thresholds because the run did not complete." + ) + return payload, status + + +def _load_output_items(items_path: Path, fallback_stdout: str) -> tuple[Dict[str, Any], ...]: + """Read the per-sample items azd wrote, falling back to stdout.""" + + raw: Any = None + if items_path.exists(): + try: + raw = json.loads(items_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + raw = None + if raw is None: + text = (fallback_stdout or "").strip() + if text: + try: + raw = json.loads(text) + except json.JSONDecodeError: + raw = None + if isinstance(raw, dict): + for key in ("items", "data", "output_items"): + nested = raw.get(key) + if isinstance(nested, list): + raw = nested + break + if not isinstance(raw, list): + return () + return tuple(entry for entry in raw if isinstance(entry, dict)) + + +# --------------------------------------------------------------------------- +# Payload readers +# --------------------------------------------------------------------------- + + +def _first_str(payload: Dict[str, Any], keys: Sequence[str]) -> Optional[str]: + for key in keys: + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _status_of(payload: Dict[str, Any]) -> str: + value = payload.get("status") + if isinstance(value, str): + return value.strip().lower() + return "" + + +def _run_error_message(payload: Dict[str, Any]) -> Optional[str]: + """Return a run-level error message, if one is actually present. + + The error object is always present with null members on success, so its + existence says nothing. Only a non-empty message is a failure signal. + """ + + error = payload.get("error") + if not isinstance(error, dict): + return None + message = error.get("message") + if isinstance(message, str) and message.strip(): + code = error.get("code") + if isinstance(code, str) and code.strip(): + return f"{code.strip()}: {message.strip()}" + return message.strip() + return None + + +def _coerce_score(value: Any) -> Optional[float]: + """Decode a score tolerantly; return ``None`` rather than a false zero.""" + + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + number = float(value) + return number if math.isfinite(number) else None + if isinstance(value, str): + try: + number = float(value.strip()) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + return None + + +def _coerce_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + try: + return json.dumps(value, ensure_ascii=False, default=str) + except (TypeError, ValueError): + return str(value) + + +def _first_text(payload: Dict[str, Any], keys: Sequence[str]) -> str: + for key in keys: + if key in payload: + text = _coerce_text(payload.get(key)) + if text: + return text + return "" + + +def _parse_scores(item: Dict[str, Any]) -> tuple[SampleScore, ...]: + raw = item.get("results") + if not isinstance(raw, list): + return () + scores: list[SampleScore] = [] + for entry in raw: + if not isinstance(entry, dict): + continue + metric = _first_str(entry, ("metric", "name")) + if not metric: + continue + passed = entry.get("passed") + scores.append( + SampleScore( + metric=metric, + score=_coerce_score(entry.get("score")), + passed=passed if isinstance(passed, bool) else None, + label=_first_str(entry, ("label",)), + reason=_first_str(entry, ("reason",)), + ) + ) + return tuple(scores) + + +def _derive_outcome(scores: Sequence[SampleScore]) -> str: + """Classify a sample when the payload did not state an outcome. + + A sample with no results is a failure, and any verdict that is not an + explicit pass counts as a failure. + """ + + if not scores: + return "failed" + if all(score.passed is True for score in scores): + return "passed" + return "failed" + + +def parse_output_item(index: int, item: Dict[str, Any]) -> SampleResult: + """Normalize one raw per-sample output item.""" + + scores = _parse_scores(item) + status = str(item.get("status") or "").strip().lower() + outcome = status if status in SAMPLE_OUTCOMES else _derive_outcome(scores) + + data_item = item.get("datasource_item") + if not isinstance(data_item, dict): + data_item = {} + + return SampleResult( + index=index, + identifier=_first_str(item, ("id",)), + outcome=outcome, + scores=scores, + input_text=_first_text(data_item, _INPUT_KEYS), + expected_text=_first_text(data_item, _EXPECTED_KEYS) or None, + response_text=_first_text(data_item, _RESPONSE_KEYS), + raw=item, + ) + + +def _run_counts(payload: Dict[str, Any], samples: Sequence[SampleResult]) -> RunCounts: + raw = payload.get("result_counts") + if isinstance(raw, dict): + def _count(key: str) -> int: + value = raw.get(key) + if isinstance(value, bool): + return 0 + if isinstance(value, (int, float)): + return int(value) + if isinstance(value, str): + try: + return int(value.strip()) + except (TypeError, ValueError): + return 0 + return 0 + + counts = RunCounts( + total=_count("total"), + passed=_count("passed"), + failed=_count("failed"), + errored=_count("errored"), + skipped=_count("skipped"), + ) + if counts.total or counts.passed or counts.failed or counts.errored or counts.skipped: + return counts + + return RunCounts( + total=len(samples), + passed=sum(1 for sample in samples if sample.outcome == "passed"), + failed=sum(1 for sample in samples if sample.outcome == "failed"), + errored=sum(1 for sample in samples if sample.outcome == "errored"), + skipped=sum(1 for sample in samples if sample.outcome == "skipped"), + ) + + +def aggregate_scores(samples: Iterable[SampleResult]) -> Dict[str, float]: + """Compute each metric as the mean of its non-absent per-sample scores. + + The run object carries only counts, so this is the only source of numeric + metrics for this surface. A metric with no decodable score anywhere is + omitted, which makes any threshold bound to it fail closed. + """ + + totals: Dict[str, float] = {} + counts: Dict[str, int] = {} + for sample in samples: + for score in sample.scores: + if score.score is None: + continue + totals[score.metric] = totals.get(score.metric, 0.0) + score.score + counts[score.metric] = counts.get(score.metric, 0) + 1 + return {metric: totals[metric] / counts[metric] for metric in sorted(totals)} + + +# --------------------------------------------------------------------------- +# Threshold binding +# --------------------------------------------------------------------------- + + +def preflight_bind_thresholds( + threshold_names: Iterable[str], + declared_metrics: Iterable[str], +) -> MetricBinding: + """Bind thresholds against what the recipe declares, before running anything. + + A threshold naming a metric no evaluator in the recipe can produce is a + configuration error, not a gate outcome, and catching it here means a typo + never bills a cloud run. + """ + + names = list(threshold_names) + metrics = sorted({metric for metric in declared_metrics if metric}) + binding = bind_threshold_metrics(names, metrics) + + if binding.unmatched: + unmatched = ", ".join(sorted(binding.unmatched)) + available = ", ".join(metrics) if metrics else "(none declared)" + raise AzdBackendError( + f"threshold metric(s) not declared by the azd eval recipe: {unmatched}.\n" + f"Metrics this recipe can produce: {available}.\n" + "Fix the threshold names in agentops.yaml, or add the evaluator to the " + "recipe. AgentOps checked this before starting the evaluation, so no " + "cloud run was consumed." + ) + if binding.ambiguous: + details = "; ".join( + f"{name} -> {', '.join(matches)}" + for name, matches in sorted(binding.ambiguous.items()) + ) + raise AzdBackendError( + f"ambiguous threshold metric binding: {details}.\n" + "Use the fully qualified metric name in agentops.yaml so the gate is " + "unambiguous. No cloud run was consumed." + ) + return binding + + +# --------------------------------------------------------------------------- +# Normalization +# --------------------------------------------------------------------------- + + +def normalize_to_results( + azd_run: CurrentEvalRun, + *, + config: AgentOpsConfig, + recipe: CurrentEvalRecipe, + evaluation: CurrentEval, + metric_binding: MetricBinding, + started_at: datetime, + resolution: Optional[Any] = None, +) -> RunResult: + """Normalize a current-surface run to the stable ``results.json`` schema.""" + + target = classify_agent(config.agent, config.protocol) + samples = [ + parse_output_item(index, item) for index, item in enumerate(azd_run.output_items) + ] + aggregate_metrics = aggregate_scores(samples) + counts = _run_counts(azd_run.run_payload, samples) + + retrieval_warnings: list[str] = [] + retrieved_passed = sum(1 for sample in samples if sample.outcome == "passed") + if counts.total and len(samples) != counts.total: + retrieval_warnings.append( + f"retrieved {len(samples)} of {counts.total} reported samples; " + "aggregate metrics are computed from the retrieved subset" + ) + items_passed = min(counts.passed, retrieved_passed) + else: + items_passed = counts.passed + + items_total = counts.total or len(samples) + + # Bound thresholds whose metric the run actually produced. Anything else is + # left out so `thresholds.evaluate` records it as missing and fails closed. + threshold_metrics = { + threshold_name: aggregate_metrics[actual_name] + for threshold_name, actual_name in metric_binding.bound.items() + if actual_name in aggregate_metrics + } + missing_metrics = sorted( + actual_name + for actual_name in metric_binding.bound.values() + if actual_name not in aggregate_metrics + ) + + threshold_rules = [ + Threshold.from_expression(metric, expression) + for metric, expression in config.thresholds.items() + ] + threshold_results = thresholds.evaluate(threshold_rules, threshold_metrics) + thresholds_total = len(threshold_results) + thresholds_passed = sum(1 for item in threshold_results if item.passed) + threshold_pass_rate = thresholds_passed / thresholds_total if thresholds_total else 1.0 + + status_ok = azd_run.status in TERMINAL_SUCCESS_STATUSES and not azd_run.error_message + # A run with no samples or no decodable metrics produced no evidence, so it + # cannot pass regardless of how few thresholds were configured. + evidence_ok = items_total > 0 and bool(aggregate_metrics) + overall_passed = status_ok and evidence_ok and threshold_pass_rate == 1.0 + + rows = [_to_row(sample, azd_run.error_message) for sample in samples] + finished_at = datetime.now(timezone.utc) + + return RunResult( + started_at=started_at.isoformat(), + finished_at=finished_at.isoformat(), + duration_seconds=azd_run.duration_seconds, + target=TargetInfo( + kind=target.kind, + raw=target.raw, + protocol=target.protocol, + name=target.name, + version=target.version, + url=target.url, + deployment=target.deployment, + ), + dataset_path=current_recipe_dataset_path(recipe, evaluation, azd_run.recipe_path), + evaluators=[reference.metric_name for reference in evaluation.evaluators], + rows=rows, + aggregate_metrics=aggregate_metrics, + thresholds=threshold_results, + summary=RunSummary( + items_total=items_total, + items_passed_all=items_passed, + items_pass_rate=(items_passed / items_total if items_total else 0.0), + thresholds_total=thresholds_total, + thresholds_passed=thresholds_passed, + threshold_pass_rate=threshold_pass_rate, + overall_passed=overall_passed, + ), + config={ + "version": config.version, + "agent": config.agent, + "thresholds": dict(config.thresholds), + "dataset_kind": config.dataset_kind, + "rubrics": [rubric.model_dump(mode="json") for rubric in config.rubrics], + "execution": "azd", + "backend_requested": "azd", + "backend_effective": "azd", + "degraded": False, + "result_granularity": "row", + "azd_evaluation": { + "recipe_path": str(azd_run.recipe_path), + "surface": "current", + "extension": CURRENT_EXTENSION_NAME, + "eval_name": azd_run.eval_name, + "run_id": azd_run.run_id, + "eval_id": azd_run.eval_id, + "status": azd_run.status, + "report_url": azd_run.report_url, + "error_message": azd_run.error_message, + "dataset": evaluation.dataset, + "metric_binding": dict(metric_binding.bound), + "unused_metrics": list(metric_binding.unused_metrics), + "missing_metrics": missing_metrics, + "result_counts": counts.as_dict(), + "retrieval_warnings": retrieval_warnings, + "skipped_recipes": [ + str(path) for path in getattr(resolution, "skipped", ()) or () + ], + }, + }, + ) + + +def _to_row(sample: SampleResult, run_error: Optional[str]) -> RowResult: + metrics = [ + RowMetric(name=score.metric, value=score.score, reason=score.reason) + for score in sample.scores + ] + error: Optional[str] = None + if sample.outcome == "errored": + error = run_error or "azd reported this sample as errored" + elif sample.outcome == "skipped": + error = "azd skipped this sample" + elif not sample.scores: + error = "azd returned no evaluator results for this sample" + return RowResult( + row_index=sample.index, + input=sample.input_text, + expected=sample.expected_text, + response=sample.response_text, + metrics=metrics, + error=error, + ) + + +# --------------------------------------------------------------------------- +# Raw artifacts +# --------------------------------------------------------------------------- + + +def write_raw_artifacts(azd_run: CurrentEvalRun, output_dir: Path) -> None: + """Persist the native azd payloads and command streams for diagnostics. + + Per-sample items carry prompts and model responses. They stay inside the + run's results directory, which the generated workspace ``.gitignore`` + already excludes from version control. + """ + + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "azd_evaluation.json").write_text( + json.dumps(azd_run.run_payload, indent=2, ensure_ascii=False, default=str), + encoding="utf-8", + ) + (output_dir / "azd_eval_output_items.json").write_text( + json.dumps(list(azd_run.output_items), indent=2, ensure_ascii=False, default=str), + encoding="utf-8", + ) + if azd_run.stdout: + (output_dir / "azd_stdout.log").write_text(azd_run.stdout, encoding="utf-8") + if azd_run.stderr: + (output_dir / "azd_stderr.log").write_text(azd_run.stderr, encoding="utf-8") diff --git a/src/agentops/pipeline/azd_runner.py b/src/agentops/pipeline/azd_runner.py index bcfc598f..d419db31 100644 --- a/src/agentops/pipeline/azd_runner.py +++ b/src/agentops/pipeline/azd_runner.py @@ -45,8 +45,19 @@ class AzdEvalRun: duration_seconds: float -def azd_available(*, cwd: Optional[Path] = None) -> bool: - """Return whether azd and the AI agents extension are available.""" +def probe_extension(extension_name: str, *, cwd: Optional[Path] = None) -> bool: + """Return whether azd and ``extension_name`` are both available. + + Detection prefers ``azd extension list --installed -o json``, which returns + a bare array whose entries carry an ``id``. The human-readable table lists + *every* registry extension, including uninstalled ones marked + ``Not installed``, so substring scanning it reports a false positive as soon + as an extension appears in the registry but before the user installs it. The + table is also truncated to terminal width. + + The text scan is retained only as a fallback for azd builds that do not + support the structured form, so existing behavior is preserved there. + """ try: subprocess.run( @@ -57,6 +68,27 @@ def azd_available(*, cwd: Optional[Path] = None) -> bool: timeout=AZD_AVAILABILITY_TIMEOUT_SECONDS, check=True, ) + except (FileNotFoundError, subprocess.SubprocessError): + return False + + try: + structured = subprocess.run( + ["azd", "extension", "list", "--installed", "-o", "json"], + cwd=str(cwd) if cwd else None, + text=True, + capture_output=True, + timeout=AZD_AVAILABILITY_TIMEOUT_SECONDS, + check=False, + ) + except (FileNotFoundError, subprocess.SubprocessError): + return False + + if structured.returncode == 0: + installed = _installed_extension_ids(structured.stdout) + if installed is not None: + return extension_name in installed + + try: extensions = subprocess.run( ["azd", "extension", "list"], cwd=str(cwd) if cwd else None, @@ -67,7 +99,41 @@ def azd_available(*, cwd: Optional[Path] = None) -> bool: ) except (FileNotFoundError, subprocess.SubprocessError): return False - return AZD_EXTENSION_NAME in (extensions.stdout + extensions.stderr) + return extension_name in (extensions.stdout + extensions.stderr) + + +def _installed_extension_ids(payload: str) -> Optional[set[str]]: + """Parse installed extension ids from structured azd output. + + Returns ``None`` when the payload is not the expected bare array, so the + caller falls back rather than treating an unparseable response as + "nothing installed". + """ + + text = (payload or "").strip() + if not text: + return None + try: + parsed = json.loads(text) + except json.JSONDecodeError: + return None + if not isinstance(parsed, list): + return None + identifiers: set[str] = set() + for entry in parsed: + if not isinstance(entry, dict): + continue + identifier = entry.get("id") + if isinstance(identifier, str) and identifier.strip(): + identifiers.add(identifier.strip()) + return identifiers + + +def azd_available(*, cwd: Optional[Path] = None) -> bool: + """Return whether azd and the AI agents extension are available.""" + + return probe_extension(AZD_EXTENSION_NAME, cwd=cwd) + def resolve_eval_recipe(workspace: Path, config: AgentOpsConfig) -> Path: diff --git a/src/agentops/pipeline/orchestrator.py b/src/agentops/pipeline/orchestrator.py index a0a8d1a4..d408c381 100644 --- a/src/agentops/pipeline/orchestrator.py +++ b/src/agentops/pipeline/orchestrator.py @@ -592,7 +592,12 @@ def _run_evaluation_azd( *, options: RunOptions, ) -> RunResult: - """azd execution: delegate to ``azd ai agent eval`` with no hidden fallback.""" + """azd execution: delegate to azd with no hidden fallback. + + Two azd evaluation surfaces are supported. The recipe itself decides which + one is used: ``evals/azure.eval.yaml`` targets ``azd ai eval``, while + ``eval.yaml`` targets the legacy ``azd ai agent eval``. + """ started_at = datetime.now(timezone.utc) progress = options.progress or (lambda _msg: None) @@ -608,14 +613,57 @@ def _run_evaluation_azd( "Use execution: local for HTTP/JSON, model:, or custom REST targets." ) + from agentops.core.azd_eval import EvalSurface, resolve_recipe + + resolution = resolve_recipe(workspace, config.eval_recipe) + + def _display(path: Path) -> str: + try: + return path.relative_to(workspace).as_posix() + except ValueError: + return path.name + + for skipped in resolution.skipped: + progress( + f"{style('note', 'yellow')}: using the current azd evaluation surface; " + f"skipping {style(_display(skipped), 'cyan')}. Set " + f"{style('eval_recipe:', 'cyan')} in agentops.yaml to pin a different recipe." + ) + + if resolution.surface is EvalSurface.CURRENT: + return _run_evaluation_azd_current( + config, + options=options, + resolution=resolution, + workspace=workspace, + started_at=started_at, + recipe_display=_display(resolution.path), + ) + return _run_evaluation_azd_legacy( + config, + options=options, + recipe_path=resolution.path, + workspace=workspace, + started_at=started_at, + recipe_display=_display(resolution.path), + ) + + +def _run_evaluation_azd_legacy( + config: AgentOpsConfig, + *, + options: RunOptions, + recipe_path: Path, + workspace: Path, + started_at: datetime, + recipe_display: str, +) -> RunResult: + """Legacy surface: ``azd ai agent eval`` via the original adapter.""" + from agentops.pipeline import azd_runner - recipe_path = azd_runner.resolve_eval_recipe(workspace, config) + progress = options.progress or (lambda _msg: None) recipe = load_eval_recipe(recipe_path) - try: - recipe_display = recipe_path.relative_to(workspace).as_posix() - except ValueError: - recipe_display = recipe_path.name progress( f"execution: {style('azd', 'bold')} - delegating to " f"{style('azd ai agent eval', 'cyan')} (recipe " @@ -649,6 +697,80 @@ def _run_evaluation_azd( return result +def _run_evaluation_azd_current( + config: AgentOpsConfig, + *, + options: RunOptions, + resolution: Any, + workspace: Path, + started_at: datetime, + recipe_display: str, +) -> RunResult: + """Current surface: ``azd ai eval`` via the current-surface adapter.""" + + from agentops.core.azd_eval import ( + current_recipe_metric_names, + load_current_eval_recipe, + ) + from agentops.pipeline import azd_eval_runner + + progress = options.progress or (lambda _msg: None) + recipe_path = resolution.path + recipe = load_current_eval_recipe(recipe_path) + evaluation = recipe.primary_eval() + + progress( + f"execution: {style('azd', 'bold')} - delegating to " + f"{style('azd ai eval', 'cyan')} (recipe " + f"{style(recipe_display, 'cyan')}, eval " + f"{style(evaluation.name, 'cyan')})." + ) + + # Bind thresholds against what the recipe declares before anything runs, so + # a misconfigured threshold never consumes a cloud evaluation. + declared_metrics = current_recipe_metric_names(recipe, recipe_path) + metric_binding = azd_eval_runner.preflight_bind_thresholds( + config.thresholds.keys(), declared_metrics + ) + + azd_run = azd_eval_runner.run_current_eval( + recipe_path, + evaluation, + workspace=workspace, + progress=progress, + timeout_seconds=max( + options.timeout_seconds, azd_eval_runner.CURRENT_EVAL_TIMEOUT_SECONDS + ), + debug_dir=options.output_dir, + ) + + # Retain raw output before normalizing, so a normalization failure is still + # diagnosable without re-running the evaluation. + azd_eval_runner.write_raw_artifacts(azd_run, options.output_dir) + + result = azd_eval_runner.normalize_to_results( + azd_run, + config=config, + recipe=recipe, + evaluation=evaluation, + metric_binding=metric_binding, + started_at=started_at, + resolution=resolution, + ) + + if options.baseline_path is not None: + baseline = comparison_module.load_baseline(options.baseline_path) + result.comparison = comparison_module.build_comparison( + current=result, + baseline=baseline, + baseline_path=options.baseline_path, + ) + + _persist(result, options.output_dir) + return result + + + def _publish_to_foundry_safely( result: RunResult, config: AgentOpsConfig, diff --git a/src/agentops/services/azd_eval_init.py b/src/agentops/services/azd_eval_init.py index 93f2683d..4a3a0407 100644 --- a/src/agentops/services/azd_eval_init.py +++ b/src/agentops/services/azd_eval_init.py @@ -18,7 +18,13 @@ from typing import Any, Optional from agentops.core.agentops_config import classify_agent -from agentops.core.azd_eval import AzdEvalRecipeError, find_eval_yaml +from agentops.core.azd_eval import ( + AzdEvalRecipeError, + EvalSurface, + RecipeResolution, + find_eval_yaml, + resolve_recipe, +) from agentops.core.config_loader import load_agentops_config from agentops.core.evaluators import detect_dataset_shape, select_evaluators from agentops.pipeline.azd_runner import ( @@ -26,6 +32,7 @@ AZD_EXTENSION_NAME, AzdBackendError, azd_available, + probe_extension, ) from agentops.utils.yaml import load_yaml, save_yaml @@ -33,6 +40,7 @@ "builtin.coherence", "builtin.fluency", ) +CURRENT_AZD_EXTENSION_NAME = "azure.ai.evaluations" _AI_ASSISTED_AZD_EVALUATORS = { "builtin.coherence", @@ -78,6 +86,7 @@ class AzdEvalInitResult: evaluator_source: str = "unknown" evaluator_signals: tuple[str, ...] = () evaluators: tuple[str, ...] = () + surface: str = EvalSurface.LEGACY.value @dataclass(frozen=True) @@ -341,24 +350,35 @@ def run_azd_eval_init( f"config not found at {resolved_config}. Run `agentops init` first." ) - existing_recipe = _find_recipe_if_unambiguous(root) - if existing_recipe is not None and not force: + existing_resolution = _find_recipe_if_unambiguous(root) + if existing_resolution is not None and not force: return _persist_recipe( config_path=resolved_config, - recipe_path=existing_recipe, + recipe_path=existing_resolution.path, command_ran=False, + surface=existing_resolution.surface, ) _ensure_prompt_agent_azd_context(root, resolved_config) - if not azd_available(cwd=root): + target_surface = ( + EvalSurface.CURRENT + if _current_surface_extension_available(root) + else EvalSurface.LEGACY + ) + + if target_surface is EvalSurface.LEGACY and not azd_available(cwd=root): raise AzdBackendError( "azd AI agent evaluation is not available. Install azd and the " f"`{AZD_EXTENSION_NAME}` extension (`azd extension install " f"{AZD_EXTENSION_NAME}`), then rerun `agentops eval init`." ) - base_command = ["azd", "--no-prompt", "ai", "agent", "eval"] + base_command = ( + ["azd", "--no-prompt", "ai", "eval"] + if target_surface is EvalSurface.CURRENT + else ["azd", "--no-prompt", "ai", "agent", "eval"] + ) arguments: list[str] = [] project_endpoint = _project_endpoint_from_config_or_env(resolved_config) if project_endpoint: @@ -398,20 +418,44 @@ def run_azd_eval_init( for evaluator in evaluator_selection.names: arguments.extend(["--evaluator", evaluator]) - completed = _run_eval_subcommand( - base_command, - arguments, - cwd=root, - timeout_seconds=timeout_seconds, - ) - - recipe = find_eval_yaml(root) - if recipe is None: - raise AzdBackendError( - "azd ai agent eval completed, but AgentOps could not find the " - "generated eval.yaml. Move it under the workspace root or src// " - "and set `eval_recipe:` in agentops.yaml." + recipe: Path + if target_surface is EvalSurface.CURRENT: + completed = _run_current_eval_init( + base_command, + arguments, + cwd=root, + timeout_seconds=timeout_seconds, ) + try: + resolution = resolve_recipe(root) + except AzdEvalRecipeError as exc: + raise AzdBackendError( + "azd ai eval init completed, but AgentOps could not find the " + "generated evals/azure.eval.yaml. Move it under evals/ and set " + "`eval_recipe:` in agentops.yaml." + ) from exc + if resolution.surface is not EvalSurface.CURRENT: + raise AzdBackendError( + "azd ai eval init completed, but the generated recipe was not a " + "current-surface evals/azure.eval.yaml recipe." + ) + recipe = resolution.path + else: + completed = _run_eval_subcommand( + base_command, + arguments, + cwd=root, + timeout_seconds=timeout_seconds, + ) + + discovered = find_eval_yaml(root) + if discovered is None: + raise AzdBackendError( + "azd ai agent eval completed, but AgentOps could not find the " + "generated eval.yaml. Move it under the workspace root or src// " + "and set `eval_recipe:` in agentops.yaml." + ) + recipe = discovered result = _persist_recipe( config_path=resolved_config, @@ -420,17 +464,37 @@ def run_azd_eval_init( stdout=completed.stdout, stderr=completed.stderr, evaluator_selection=evaluator_selection, + surface=target_surface, ) return result -def _find_recipe_if_unambiguous(workspace: Path) -> Optional[Path]: +def _find_recipe_if_unambiguous(workspace: Path) -> Optional[RecipeResolution]: try: - return find_eval_yaml(workspace) + return resolve_recipe(workspace) except AzdEvalRecipeError: return None +def _current_surface_extension_available(workspace: Path) -> bool: + """Return whether the current-surface azd extension is installed. + + The answer is deliberately fail-safe: if availability cannot be determined + for any reason, it is reported as unavailable and initialization targets the + legacy surface, which is published and installable. Guessing "available" + here would generate an ``evals/azure.eval.yaml`` whose very next command + fails on a dependency the user cannot install (spec FR-016, SC-010). + + The broad catch is intentional and safe in this direction: a probe that + cannot answer must never be read as a green light. + """ + + try: + return bool(probe_extension(CURRENT_AZD_EXTENSION_NAME, cwd=workspace)) + except Exception: # noqa: BLE001 - any probe failure means "not available" + return False + + # azd renamed this subcommand in the ``azure.ai.agents`` extension 0.1.40: # ``init`` became ``generate``. Try the new name first and fall back to the # legacy name so AgentOps works whether the consumer has an old or new @@ -532,6 +596,43 @@ def _run_eval_subcommand( raise AzdBackendError(f"azd ai agent eval failed: {detail}") +def _run_current_eval_init( + base_command: list[str], + arguments: list[str], + *, + cwd: Path, + timeout_seconds: float, +) -> "subprocess.CompletedProcess[str]": + command = [*base_command, "init", *arguments] + try: + completed = subprocess.run( + command, + cwd=str(cwd), + text=True, + encoding="utf-8", + errors="replace", + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except FileNotFoundError as exc: + raise AzdBackendError( + "azd was not found on PATH. Install the Azure Developer CLI and " + f"the `{CURRENT_AZD_EXTENSION_NAME}` extension, then rerun " + "`agentops eval init`." + ) from exc + except subprocess.TimeoutExpired as exc: + raise AzdBackendError( + f"{' '.join(command)} timed out after {timeout_seconds:g}s." + ) from exc + + if completed.returncode != 0: + raise AzdBackendError( + f"azd ai eval init failed: {_azd_failure_detail(completed)}" + ) + return completed + + def _dataset_from_config(config_path: Path) -> Optional[Path]: data = load_yaml(config_path) raw_dataset = data.get("dataset") @@ -927,6 +1028,7 @@ def _persist_recipe( stdout: str = "", stderr: str = "", evaluator_selection: AzdEvaluatorSelection | None = None, + surface: EvalSurface = EvalSurface.LEGACY, ) -> AzdEvalInitResult: data = load_yaml(config_path) recipe_value = _relative_config_path(recipe_path, config_path.parent) @@ -948,6 +1050,7 @@ def _persist_recipe( evaluator_source=(evaluator_selection.source if evaluator_selection else "existing recipe"), evaluator_signals=(evaluator_selection.signals if evaluator_selection else ()), evaluators=(evaluator_selection.names if evaluator_selection else ()), + surface=surface.value, ) diff --git a/src/agentops/services/eval_analysis.py b/src/agentops/services/eval_analysis.py index f4b054ee..ca7b1b87 100644 --- a/src/agentops/services/eval_analysis.py +++ b/src/agentops/services/eval_analysis.py @@ -11,6 +11,12 @@ from typing import Any, Dict, Iterable, List, Optional, Sequence, Set from agentops.core.agentops_config import classify_agent +from agentops.core.azd_eval import ( + AzdEvalRecipeAmbiguous, + AzdEvalRecipeError, + RecipeResolution, + resolve_recipe, +) from agentops.services.dataset_source import diagnose_dataset_source from agentops.utils.yaml import load_yaml @@ -226,6 +232,7 @@ class _ConfigInfo: target_kind: Optional[str] dataset_exists: bool dataset_columns: Set[str] + azd_recipe_ready: bool = True signals: List[EvalSignal] = field(default_factory=list) warnings: List[str] = field(default_factory=list) @@ -312,12 +319,30 @@ def _agentops_config_info(root: Path) -> _ConfigInfo: dataset_source, ) ) + azd_recipe_ready = True + azd_recipe_warnings: list[str] = [] + if str(data.get("execution") or "").strip().lower() == "azd": + recipe_signals, azd_recipe_warnings, azd_recipe_ready = _azd_recipe_readiness( + root=path.parent, + explicit_path=( + Path(str(data["eval_recipe"])) + if data.get("eval_recipe") not in (None, "") + else None + ), + ) + signals.extend(recipe_signals) diagnosis_warnings = ( [diagnosis.message] if diagnosis is not None and diagnosis.status != "ready" else [] ) - ready = bool(agent and dataset_value and dataset_exists and "input" in dataset_columns) + ready = bool( + agent + and dataset_value + and dataset_exists + and "input" in dataset_columns + and azd_recipe_ready + ) if not agent: status = "observability_only" elif ready: @@ -332,8 +357,9 @@ def _agentops_config_info(root: Path) -> _ConfigInfo: target_kind=target_kind, dataset_exists=dataset_exists, dataset_columns=dataset_columns, + azd_recipe_ready=azd_recipe_ready, signals=signals, - warnings=diagnosis_warnings, + warnings=[*diagnosis_warnings, *azd_recipe_warnings], ) except Exception as exc: return _ConfigInfo( @@ -344,6 +370,7 @@ def _agentops_config_info(root: Path) -> _ConfigInfo: target_kind=None, dataset_exists=False, dataset_columns=set(), + azd_recipe_ready=False, signals=[ EvalSignal( "agentops_config", @@ -357,6 +384,66 @@ def _agentops_config_info(root: Path) -> _ConfigInfo: ) +def _azd_recipe_readiness( + *, + root: Path, + explicit_path: Optional[Path], +) -> tuple[list[EvalSignal], list[str], bool]: + try: + resolution = resolve_recipe(root, explicit_path) + except (AzdEvalRecipeAmbiguous, AzdEvalRecipeError) as exc: + message = ( + f"{exc} Required action: create a recipe at evals/azure.eval.yaml " + "(current surface) or eval.yaml / src//eval.yaml (legacy " + "surface), or set `eval_recipe:` to the intended file." + ) + return ( + [ + EvalSignal( + "azd_eval_recipe_gap", + "azd eval recipe", + message, + confidence="medium", + ) + ], + [message], + False, + ) + return (_azd_recipe_resolution_signals(root, resolution), [], True) + + +def _azd_recipe_resolution_signals( + root: Path, + resolution: RecipeResolution, +) -> list[EvalSignal]: + recipe_path = _rel_text(root, resolution.path) + signals = [ + EvalSignal( + "azd_eval_recipe", + "azd eval recipe", + ( + f"Resolved {recipe_path} using the {resolution.surface.value} " + f"surface ({resolution.extension})." + ), + recipe_path, + ) + ] + for skipped in resolution.skipped: + skipped_path = _rel_text(root, skipped) + signals.append( + EvalSignal( + "azd_eval_recipe_skipped", + "Skipped azd eval recipe", + ( + f"Skipped {skipped_path} because the current surface takes " + "precedence over legacy auto-discovery." + ), + skipped_path, + ) + ) + return signals + + def _resolve_dataset_path(config_dir: Path, dataset_value: Any) -> Optional[Path]: if dataset_value is None: return None @@ -795,6 +882,9 @@ def _signal_label(key: str, fallback: str) -> str: "dataset_columns": "Columns", "scenario_hint": "Scenario", "azd_project": "azd", + "azd_eval_recipe": "azd recipe", + "azd_eval_recipe_skipped": "azd skipped", + "azd_eval_recipe_gap": "azd gap", "container_or_http_app": "Host", "rag_signal": "RAG", "tool_signal": "Tools", diff --git a/src/agentops/templates/agentops.yaml b/src/agentops/templates/agentops.yaml index 55a611c8..e7c272ef 100644 --- a/src/agentops/templates/agentops.yaml +++ b/src/agentops/templates/agentops.yaml @@ -99,9 +99,26 @@ dataset: .agentops/data/smoke.jsonl # cloud — Foundry runs the agent and evaluators server-side # (only valid for 'name:version' Foundry prompt agents; # implicitly publishes to the New Foundry panel). +# azd — the Azure Developer CLI runs a Foundry-native evaluation +# from a recipe in this repository. Two azd surfaces are +# supported and the recipe decides which one is used: +# evals/azure.eval.yaml -> azd ai eval +# (azure.ai.evaluations, preview) +# eval.yaml, or src//eval.yaml +# -> azd ai agent eval +# (azure.ai.agents) +# If both are discoverable, the current surface +# (evals/azure.eval.yaml) wins and the run reports which +# recipe it skipped. Pin one with eval_recipe: below. # # execution: local # +# Optional. Path to an azd evaluation recipe, for execution: azd. When omitted, +# AgentOps auto-discovers a single recipe. Set this when you keep the recipe +# somewhere else, or to pin one surface while both are present. +# +# eval_recipe: evals/azure.eval.yaml +# # Optional. Foundry observability readiness metadata. AgentOps reads # this for Doctor/Cockpit/evidence; Foundry still owns tracing at runtime. # diff --git a/tests/fixtures/azd_stub.py b/tests/fixtures/azd_stub.py new file mode 100644 index 00000000..1d73e3a3 --- /dev/null +++ b/tests/fixtures/azd_stub.py @@ -0,0 +1,289 @@ +"""Reusable Azure Developer CLI subprocess double for tests. + +Lets a test script an ordered sequence of azd invocations, assert the exact +argv of each one, and assert that no unexpected command was run. Install it +with :meth:`AzdStub.install`, which patches ``subprocess.run`` for the duration +of the test via ``monkeypatch``. + +Both azd adapters call ``subprocess.run`` through +``agentops.pipeline.azd_runner._run_command`` when no progress callback is +supplied, so patching ``subprocess.run`` covers the whole azd boundary without +reaching the network or requiring azd to be installed. +""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Optional, Sequence + + +@dataclass +class _Expectation: + """One scripted azd invocation.""" + + match: tuple[str, ...] + returncode: int = 0 + stdout: str = "" + stderr: str = "" + side_effect: Optional[Callable[[list[str]], None]] = None + + +class UnexpectedAzdCommand(AssertionError): + """Raised when the stub receives a command it was not scripted for.""" + + +class _FakePopen: + """Minimal ``subprocess.Popen`` stand-in for the progress-heartbeat path. + + ``_run_command_with_progress`` only needs ``args``, ``poll``, ``wait``, and + ``kill``. The process is already "finished" when constructed, so the + heartbeat loop exits on its first iteration. + """ + + def __init__(self, args: list[str], returncode: int) -> None: + self.args = args + self.returncode = returncode + + def poll(self) -> int: + return self.returncode + + def wait(self, timeout: Optional[float] = None) -> int: + return self.returncode + + def kill(self) -> None: # pragma: no cover - never reached; poll ends the loop + return None + + +class AzdStub: + """An ordered, assertion-friendly stand-in for the azd CLI.""" + + def __init__(self) -> None: + self.calls: list[list[str]] = [] + self._queue: list[_Expectation] = [] + + # ------------------------------------------------------------------ + # Scripting + # ------------------------------------------------------------------ + def expect( + self, + *match: str, + returncode: int = 0, + stdout: str = "", + stderr: str = "", + side_effect: Optional[Callable[[list[str]], None]] = None, + ) -> "AzdStub": + """Queue the next expected invocation. + + ``match`` tokens must all appear in the joined argv, in any position. + Use enough tokens to identify the command unambiguously. + """ + + self._queue.append( + _Expectation( + match=tuple(match), + returncode=returncode, + stdout=stdout, + stderr=stderr, + side_effect=side_effect, + ) + ) + return self + + def expect_json(self, *match: str, payload: Any, returncode: int = 0) -> "AzdStub": + """Queue an invocation whose stdout is ``payload`` serialized as JSON.""" + + return self.expect( + *match, + returncode=returncode, + stdout=json.dumps(payload), + ) + + def expect_output_file( + self, + *match: str, + payload: Any, + flag: str = "--output-file", + returncode: int = 0, + stdout: str = "", + ) -> "AzdStub": + """Queue an invocation that writes ``payload`` to the path after ``flag``.""" + + return self.expect( + *match, + returncode=returncode, + stdout=stdout, + side_effect=write_json_to_flag(flag, payload), + ) + + # ------------------------------------------------------------------ + # Installation + # ------------------------------------------------------------------ + def install(self, monkeypatch: Any) -> "AzdStub": + """Patch both ``subprocess.run`` and ``subprocess.Popen``. + + ``azd_runner._run_command`` uses ``subprocess.run`` when no progress + callback is supplied, and ``subprocess.Popen`` when one is (the + heartbeat path). The legacy adapter is driven with a progress callback + by the orchestrator, so patching only ``run`` would let the real azd + binary be invoked from an integration test. + """ + + monkeypatch.setattr(subprocess, "run", self) + monkeypatch.setattr(subprocess, "Popen", self._popen) + return self + + def _popen(self, command: Sequence[str], **kwargs: Any) -> "_FakePopen": + completed = self(command, **kwargs) + for stream_name in ("stdout", "stderr"): + target = kwargs.get(stream_name) + payload = getattr(completed, stream_name) or "" + if payload and hasattr(target, "write"): + target.write(payload) + return _FakePopen(list(completed.args), completed.returncode) + + # ------------------------------------------------------------------ + # Invocation + # ------------------------------------------------------------------ + def __call__(self, command: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess: + argv = [str(part) for part in command] + self.calls.append(argv) + joined = " ".join(argv) + + if not self._queue: + raise UnexpectedAzdCommand( + f"azd was invoked more times than scripted.\n" + f"Unexpected: {joined}\n" + f"Calls so far:\n" + self.describe_calls() + ) + + expectation = self._queue.pop(0) + missing = [token for token in expectation.match if token not in argv] + if missing: + raise UnexpectedAzdCommand( + f"azd invocation did not match the next scripted command.\n" + f"Expected tokens: {expectation.match}\n" + f"Missing: {missing}\n" + f"Actual: {joined}" + ) + + if expectation.side_effect is not None: + expectation.side_effect(argv) + + return subprocess.CompletedProcess( + argv, + expectation.returncode, + stdout=expectation.stdout, + stderr=expectation.stderr, + ) + + # ------------------------------------------------------------------ + # Assertions and inspection + # ------------------------------------------------------------------ + @property + def commands(self) -> list[str]: + """Every invocation so far, as joined command strings.""" + + return [" ".join(argv) for argv in self.calls] + + def describe_calls(self) -> str: + if not self.calls: + return " (none)" + return "\n".join(f" {index}: {line}" for index, line in enumerate(self.commands)) + + def assert_exhausted(self) -> None: + """Assert every scripted invocation actually happened.""" + + if self._queue: + remaining = [expectation.match for expectation in self._queue] + raise AssertionError( + f"azd was invoked fewer times than scripted. Remaining: {remaining}\n" + f"Calls:\n" + self.describe_calls() + ) + + def assert_never_invoked(self) -> None: + """Assert azd was never called at all.""" + + if self.calls: + raise AssertionError( + "expected azd to never be invoked, but it was:\n" + self.describe_calls() + ) + + def find(self, *tokens: str) -> list[list[str]]: + """Return every recorded invocation containing all ``tokens`` as argv entries. + + Matching is on exact argv tokens, never on a substring of the joined + command. Paths passed to azd routinely contain words like ``run`` or + ``list`` (pytest tmp dirs are named after the test), and substring + matching would silently select the wrong invocation. + """ + + return [argv for argv in self.calls if all(token in argv for token in tokens)] + + def assert_no_call_containing(self, *tokens: str) -> None: + matches = self.find(*tokens) + if matches: + raise AssertionError( + f"expected no azd invocation containing {tokens}, found:\n" + + "\n".join(f" {' '.join(argv)}" for argv in matches) + ) + + +def arg_after(argv: Sequence[str], flag: str) -> str: + """Return the value following ``flag`` in ``argv``.""" + + parts = [str(part) for part in argv] + if flag not in parts: + raise AssertionError(f"{flag} not present in: {' '.join(parts)}") + index = parts.index(flag) + if index + 1 >= len(parts): + raise AssertionError(f"{flag} has no value in: {' '.join(parts)}") + return parts[index + 1] + + +def write_json_to_flag(flag: str, payload: Any) -> Callable[[list[str]], None]: + """Build a side effect that writes ``payload`` to the path following ``flag``.""" + + def _write(argv: list[str]) -> None: + target = Path(arg_after(argv, flag)) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(payload), encoding="utf-8") + + return _write + + +def extension_list_json(*extension_ids: str) -> str: + """Build a structured ``azd extension list --installed -o json`` payload.""" + + return json.dumps( + [ + { + "id": extension_id, + "name": f"{extension_id} (Beta)", + "version": "1.0.0-beta.1", + "installedVersion": "1.0.0-beta.1", + "updateAvailable": False, + "source": "azd", + } + for extension_id in extension_ids + ] + ) + + +def extension_list_table(*rows: tuple[str, str]) -> str: + """Build a human-readable ``azd extension list`` table. + + Each row is ``(extension_id, status)``. Statuses observed in the wild + include ``Up to date``, ``Update available``, ``Not installed``, and + ``Incompatible``. A ``Not installed`` row still contains the extension id, + which is exactly why substring scanning is unsafe. + """ + + header = "ID NAME STATUS INSTALLED" + lines = [header, "-" * len(header)] + for extension_id, status in rows: + installed = "-" if status == "Not installed" else "1.0.0-bet…" + lines.append(f"{extension_id:<26} {extension_id:<19} {status:<15} {installed}") + return "\n".join(lines) + "\n" diff --git a/tests/integration/test_azd_current_surface_cli.py b/tests/integration/test_azd_current_surface_cli.py new file mode 100644 index 00000000..63150917 --- /dev/null +++ b/tests/integration/test_azd_current_surface_cli.py @@ -0,0 +1,290 @@ +"""End-to-end CLI coverage for the current azd evaluation surface. + +Covers quickstart scenarios T1.5 (exit-code matrix) and T2-equivalents run +against a faked azd. Exit codes are a public contract, so every outcome is +asserted through the real CLI rather than by calling the adapter directly. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from agentops.cli.app import app # noqa: E402 +from fixtures.azd_stub import AzdStub, extension_list_json # noqa: E402 + + +runner = CliRunner() + +CURRENT = "azure.ai.evaluations" + +RECIPE = """ +datasets: + - name: smoke + file: ./datasets/smoke.jsonl + +evaluators: + - name: travel-quality + definition: + type: rubric + dimensions: + - id: accuracy + weight: 5 + description: Factually correct. + +evals: + - name: travel-regression + dataset: smoke + evaluators: + - evaluator: builtin.task_adherence + - evaluator: travel-quality + target: + type: agent + name: travel-agent +""".lstrip() + + +def _workspace(tmp_path: Path, thresholds: str = "") -> Path: + (tmp_path / "evals").mkdir(parents=True, exist_ok=True) + (tmp_path / "evals" / "azure.eval.yaml").write_text(RECIPE, encoding="utf-8") + (tmp_path / "dataset.jsonl").write_text( + json.dumps({"input": "hi", "expected": "hello"}) + "\n", encoding="utf-8" + ) + config_path = tmp_path / "agentops.yaml" + config_path.write_text( + "version: 1\n" + "agent: travel-agent:1\n" + "dataset: dataset.jsonl\n" + "execution: azd\n" + thresholds, + encoding="utf-8", + ) + return config_path + + +def _run_payload(status: str = "completed", *, total: int = 2, passed: int = 2, **extra): + payload = { + "id": "run-1", + "eval_id": "eval-1", + "status": status, + "result_counts": { + "total": total, + "passed": passed, + "failed": total - passed, + "errored": 0, + "skipped": 0, + }, + "error": {"code": None, "message": None}, + "report_url": "https://ai.azure.com/report", + } + payload.update(extra) + return payload + + +def _items(*scores: tuple[float, float]): + return [ + { + "id": f"item-{index}", + "run_id": "run-1", + "status": "passed", + "datasource_item": {"input": f"q{index}", "expected": "e", "response": "r"}, + "results": [ + {"metric": "builtin.task_adherence", "score": adherence, "passed": True}, + {"metric": "accuracy", "score": accuracy, "passed": True}, + ], + } + for index, (adherence, accuracy) in enumerate(scores) + ] + + +def _script(stub: AzdStub, *, statuses=("completed",), items=None, run_payload=None): + stub.expect("azd", "version") + stub.expect("extension", "list", "--installed", stdout=extension_list_json(CURRENT)) + stub.expect("ai", "eval", "create") + stub.expect( + "ai", "eval", "run", "start", + stdout=json.dumps({"run_id": "run-1", "eval_id": "eval-1", "status": "queued"}), + ) + for status in statuses: + payload = run_payload if status == statuses[-1] and run_payload else _run_payload(status) + stub.expect("ai", "eval", "run", "show", stdout=json.dumps(payload)) + if items is not None: + stub.expect_output_file("ai", "eval", "run", "output", "list", payload=items) + return stub + + +def _invoke(config_path: Path, out: Path, *extra: str): + return runner.invoke( + app, + ["eval", "run", "--config", str(config_path), "--output", str(out), *extra], + ) + + +# --------------------------------------------------------------------------- +# T021 - happy path +# --------------------------------------------------------------------------- + + +def test_current_surface_run_produces_normalized_artifacts( + tmp_path: Path, monkeypatch +) -> None: + config_path = _workspace(tmp_path, "thresholds:\n accuracy: '>=4'\n") + out = tmp_path / "out" + stub = _script(AzdStub(), items=_items((4.0, 5.0), (4.0, 5.0))).install(monkeypatch) + + result = _invoke(config_path, out) + + assert result.exit_code == 0, result.output + stub.assert_exhausted() + payload = json.loads((out / "results.json").read_text(encoding="utf-8")) + assert payload["summary"]["overall_passed"] is True + assert payload["aggregate_metrics"]["accuracy"] == 5.0 + assert len(payload["rows"]) == 2 + assert payload["config"]["azd_evaluation"]["surface"] == "current" + assert (out / "report.md").exists() + assert (out / "azd_evaluation.json").exists() + assert (out / "azd_eval_output_items.json").exists() + + +def test_the_evaluation_is_executed_exactly_once(tmp_path: Path, monkeypatch) -> None: + config_path = _workspace(tmp_path) + out = tmp_path / "out" + stub = _script(AzdStub(), items=_items((4.0, 5.0))).install(monkeypatch) + + result = _invoke(config_path, out) + + assert result.exit_code == 0, result.output + starts = [argv for argv in stub.calls if argv[2:6] == ["ai", "eval", "run", "start"]] + assert len(starts) == 1 + stub.assert_exhausted() + + +# --------------------------------------------------------------------------- +# T027 - exit-code matrix +# --------------------------------------------------------------------------- + + +def test_unsatisfied_threshold_exits_two(tmp_path: Path, monkeypatch) -> None: + config_path = _workspace(tmp_path, "thresholds:\n accuracy: '>=5.5'\n") + out = tmp_path / "out" + _script(AzdStub(), items=_items((4.0, 5.0))).install(monkeypatch) + + result = _invoke(config_path, out) + + assert result.exit_code == 2, result.output + + +def test_declared_metric_never_emitted_exits_two(tmp_path: Path, monkeypatch) -> None: + config_path = _workspace(tmp_path, "thresholds:\n accuracy: '>=1'\n") + out = tmp_path / "out" + items = [ + { + "id": "item-0", + "run_id": "run-1", + "status": "passed", + "datasource_item": {"input": "q"}, + "results": [ + {"metric": "builtin.task_adherence", "score": 5, "passed": True} + ], + } + ] + _script(AzdStub(), items=items).install(monkeypatch) + + result = _invoke(config_path, out) + + assert result.exit_code == 2, result.output + payload = json.loads((out / "results.json").read_text(encoding="utf-8")) + assert payload["config"]["azd_evaluation"]["missing_metrics"] == ["accuracy"] + + +def test_unmatched_threshold_exits_one_without_invoking_azd( + tmp_path: Path, monkeypatch +) -> None: + """A typo must never consume a cloud evaluation.""" + + config_path = _workspace(tmp_path, "thresholds:\n not_a_metric: '>=3'\n") + out = tmp_path / "out" + stub = AzdStub().install(monkeypatch) + + result = _invoke(config_path, out) + + assert result.exit_code == 1, result.output + assert "not_a_metric" in result.output + assert "Traceback" not in result.output + stub.assert_never_invoked() + + +@pytest.mark.parametrize("status", ["failed", "error", "canceled", "cancelled"]) +def test_terminal_failure_status_exits_one( + tmp_path: Path, monkeypatch, status: str +) -> None: + config_path = _workspace(tmp_path) + out = tmp_path / "out" + _script(AzdStub(), statuses=(status,)).install(monkeypatch) + + result = _invoke(config_path, out) + + assert result.exit_code == 1, result.output + assert "Traceback" not in result.output + + +def test_missing_extension_exits_one_with_an_actionable_message( + tmp_path: Path, monkeypatch +) -> None: + config_path = _workspace(tmp_path) + out = tmp_path / "out" + stub = AzdStub() + stub.expect("azd", "version") + stub.expect("extension", "list", "--installed", stdout=extension_list_json("azure.ai.agents")) + stub.install(monkeypatch) + + result = _invoke(config_path, out) + + assert result.exit_code == 1, result.output + assert CURRENT in result.output + assert "Traceback" not in result.output + + +def test_zero_samples_does_not_report_a_pass(tmp_path: Path, monkeypatch) -> None: + config_path = _workspace(tmp_path) + out = tmp_path / "out" + payload = _run_payload("completed", total=0, passed=0) + _script(AzdStub(), items=[], run_payload=payload).install(monkeypatch) + + result = _invoke(config_path, out) + + assert result.exit_code != 0, result.output + + +# --------------------------------------------------------------------------- +# T028 - baseline comparison +# --------------------------------------------------------------------------- + + +def test_baseline_comparison_works_for_a_current_surface_run( + tmp_path: Path, monkeypatch +) -> None: + config_path = _workspace(tmp_path, "thresholds:\n accuracy: '>=4'\n") + + first = tmp_path / "first" + _script(AzdStub(), items=_items((4.0, 5.0))).install(monkeypatch) + assert _invoke(config_path, first).exit_code == 0 + + second = tmp_path / "second" + _script(AzdStub(), items=_items((4.0, 4.0))).install(monkeypatch) + result = _invoke(config_path, second, "--baseline", str(first / "results.json")) + + assert result.exit_code == 0, result.output + payload = json.loads((second / "results.json").read_text(encoding="utf-8")) + assert payload["comparison"] is not None + accuracy = [ + metric + for metric in payload["comparison"]["metrics"] + if metric["metric"] == "accuracy" + ] + assert accuracy and accuracy[0]["direction"] == "regressed" diff --git a/tests/integration/test_azd_surface_compat.py b/tests/integration/test_azd_surface_compat.py new file mode 100644 index 00000000..23e455e6 --- /dev/null +++ b/tests/integration/test_azd_surface_compat.py @@ -0,0 +1,202 @@ +"""Regression guards proving legacy azd workspaces are not opted into the new surface.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from typer.testing import CliRunner + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "tests")) + +from agentops.cli.app import app # noqa: E402 +from agentops.pipeline import azd_runner # noqa: E402 +from fixtures.azd_stub import AzdStub, extension_list_json # noqa: E402 + + +runner = CliRunner() + +LEGACY_EXTENSION = "azure.ai.agents" + + +def _write_legacy_workspace(workspace: Path) -> Path: + (workspace / "dataset.jsonl").write_text( + json.dumps({"input": "hello", "expected": "hi"}) + "\n", + encoding="utf-8", + ) + (workspace / "agentops.yaml").write_text( + """ +version: 1 +agent: travel-agent:1 +dataset: dataset.jsonl +execution: azd +""".lstrip(), + encoding="utf-8", + ) + (workspace / "eval.yaml").write_text( + """ +name: travel-agent-eval +agent: + name: travel-agent + kind: prompt-agent + version: "1" +dataset_reference: + name: smoke + version: "1" + local_uri: dataset.jsonl +evaluators: + - name: builtin.coherence +""".lstrip(), + encoding="utf-8", + ) + return workspace / "agentops.yaml" + + +def _script_legacy_success(stub: AzdStub) -> AzdStub: + stub.expect("azd", "version") + stub.expect( + "extension", + "list", + "--installed", + stdout=extension_list_json(LEGACY_EXTENSION), + ) + stub.expect( + "ai", + "agent", + "eval", + "run", + stdout="Eval: eval-1\nRun: run-1\nStatus: Completed\n", + ) + stub.expect_output_file( + "ai", + "agent", + "eval", + "show", + flag="--out-file", + payload={ + "id": "run-1", + "eval_id": "eval-1", + "status": "completed", + "result_counts": {"total": 1}, + "metrics": [{"name": "builtin.coherence", "score": 4.0}], + }, + stdout="exported", + ) + return stub + + +def _install_legacy_stub(stub: AzdStub, monkeypatch) -> AzdStub: + stub.install(monkeypatch) + + def _fake_run_command( + command, + *, + cwd, + timeout_seconds, + progress=None, + progress_label="command", + ): + return stub(command) + + monkeypatch.setattr(azd_runner, "_run_command", _fake_run_command) + return stub + + +def _is_legacy_eval_command(argv: list[str]) -> bool: + return argv[:5] == ["azd", "--no-prompt", "ai", "agent", "eval"] + + +def _is_current_surface_eval_command(argv: list[str]) -> bool: + return argv[:4] == ["azd", "--no-prompt", "ai", "eval"] + + +def test_legacy_workspace_without_evals_runs_with_unchanged_normalization( + tmp_path: Path, monkeypatch +) -> None: + config_path = _write_legacy_workspace(tmp_path) + assert not (tmp_path / "evals").exists() + out = tmp_path / "out" + stub = _install_legacy_stub(_script_legacy_success(AzdStub()), monkeypatch) + + result = runner.invoke( + app, + ["eval", "run", "--config", str(config_path), "--output", str(out)], + ) + + assert result.exit_code == 0, result.output + stub.assert_exhausted() + payload = json.loads((out / "results.json").read_text(encoding="utf-8")) + assert payload["config"]["result_granularity"] == "aggregate" + assert "surface" not in payload["config"]["azd_evaluation"] + assert payload["rows"] == [] + assert payload["aggregate_metrics"] == {"builtin.coherence": 4.0} + assert "Warning" not in result.output + assert "Error" not in result.output + assert "azd ai eval" not in result.output + assert "current azd evaluation surface" not in result.output + + +def test_legacy_workspace_analyze_does_not_mention_current_surface(tmp_path: Path) -> None: + _write_legacy_workspace(tmp_path) + assert not (tmp_path / "evals").exists() + + result = runner.invoke(app, ["eval", "analyze", "--dir", str(tmp_path)]) + + assert result.exit_code == 0, result.output + assert "Warnings" not in result.output + assert "Warning" not in result.output + assert "Error" not in result.output + assert "azd ai eval" not in result.output + assert "azure.ai.evaluations" not in result.output + assert "evals/azure.eval.yaml" not in result.output + assert "current azd evaluation surface" not in result.output + + +def test_legacy_workspace_dispatches_only_to_legacy_azd_commands( + tmp_path: Path, monkeypatch +) -> None: + config_path = _write_legacy_workspace(tmp_path) + stub = _install_legacy_stub(_script_legacy_success(AzdStub()), monkeypatch) + + result = runner.invoke( + app, + ["eval", "run", "--config", str(config_path), "--output", str(tmp_path / "out")], + ) + + assert result.exit_code == 0, result.output + eval_commands = [argv for argv in stub.calls if argv and argv[0] == "azd"] + assert [argv for argv in eval_commands if _is_legacy_eval_command(argv)] == [ + [ + "azd", + "--no-prompt", + "ai", + "agent", + "eval", + "run", + "--config", + str(tmp_path / "eval.yaml"), + "--output", + "json", + ], + [ + "azd", + "--no-prompt", + "ai", + "agent", + "eval", + "show", + "eval-1", + "--eval-run-id", + "run-1", + "--out-file", + next( + argv[argv.index("--out-file") + 1] + for argv in eval_commands + if _is_legacy_eval_command(argv) and "show" in argv + ), + ], + ] + assert not [argv for argv in eval_commands if _is_current_surface_eval_command(argv)] diff --git a/tests/unit/test_azd_eval_init.py b/tests/unit/test_azd_eval_init.py index 9559c121..cd97e60c 100644 --- a/tests/unit/test_azd_eval_init.py +++ b/tests/unit/test_azd_eval_init.py @@ -701,6 +701,86 @@ def fake_run(command, **kwargs): assert subcommands == ["generate"] +def test_run_azd_eval_init_selects_legacy_when_current_extension_absent( + tmp_path: Path, + monkeypatch, +) -> None: + config_path = _setup_eval_workspace(tmp_path) + probes: list[tuple[str, Path | None]] = [] + + def fake_probe(extension_name: str, *, cwd: Path | None = None) -> bool: + probes.append((extension_name, cwd)) + return False + + monkeypatch.setattr(azd_eval_init, "probe_extension", fake_probe) + monkeypatch.setattr(azd_eval_init, "azd_available", lambda *, cwd=None: True) + + def write_legacy_recipe(command, **kwargs): + if command[:3] == ["az", "resource", "list"]: + return subprocess.CompletedProcess(command, 0, stdout="[]", stderr="") + recipe = Path(kwargs["cwd"]) / "eval.yaml" + recipe.write_text("name: travel-agent-eval\n", encoding="utf-8") + return subprocess.CompletedProcess(command, 0, stdout="created", stderr="") + + monkeypatch.setattr(subprocess, "run", write_legacy_recipe) + + result = azd_eval_init.run_azd_eval_init( + workspace=tmp_path, + config_path=config_path, + ) + + assert probes == [(azd_eval_init.CURRENT_AZD_EXTENSION_NAME, tmp_path.resolve())] + assert result.surface == "legacy" + assert result.recipe_path == (tmp_path / "eval.yaml").resolve() + assert not (tmp_path / "evals" / "azure.eval.yaml").exists() + assert "eval_recipe: eval.yaml" in config_path.read_text(encoding="utf-8") + + +def test_run_azd_eval_init_selects_current_when_current_extension_present( + tmp_path: Path, + monkeypatch, +) -> None: + config_path = _setup_eval_workspace(tmp_path) + monkeypatch.setattr( + azd_eval_init, + "probe_extension", + lambda extension_name, *, cwd=None: extension_name + == azd_eval_init.CURRENT_AZD_EXTENSION_NAME, + ) + monkeypatch.setattr( + azd_eval_init, + "azd_available", + lambda *, cwd=None: (_ for _ in ()).throw( + AssertionError("legacy availability check must not run") + ), + ) + + seen: list[list[str]] = [] + + def fake_run(command, **kwargs): + if command[:3] == ["az", "resource", "list"]: + return subprocess.CompletedProcess(command, 0, stdout="[]", stderr="") + seen.append(command) + assert command[:5] == ["azd", "--no-prompt", "ai", "eval", "init"] + recipe = Path(kwargs["cwd"]) / "evals" / "azure.eval.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text("evals: []\n", encoding="utf-8") + return subprocess.CompletedProcess(command, 0, stdout="created", stderr="") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = azd_eval_init.run_azd_eval_init( + workspace=tmp_path, + config_path=config_path, + ) + + assert result.surface == "current" + assert result.recipe_path == (tmp_path / "evals" / "azure.eval.yaml").resolve() + assert seen and seen[0][:5] == ["azd", "--no-prompt", "ai", "eval", "init"] + assert "eval_recipe: evals/azure.eval.yaml" in config_path.read_text(encoding="utf-8") + assert not (tmp_path / "eval.yaml").exists() + + def test_eval_subcommand_unsupported_matches_known_messages() -> None: assert azd_eval_init._eval_subcommand_unsupported( 'Error: unknown command "generate" for "azd ai agent eval"' diff --git a/tests/unit/test_azd_eval_runner.py b/tests/unit/test_azd_eval_runner.py new file mode 100644 index 00000000..fd2e211a --- /dev/null +++ b/tests/unit/test_azd_eval_runner.py @@ -0,0 +1,646 @@ +"""Tests for the current-surface azd evaluation adapter. + +Covers quickstart scenarios T1.3 (command sequence, polling, parsing, +aggregation, raw artifacts) and the User Story 2 gate behavior. The azd +boundary is faked throughout, so nothing here needs azd, Azure credentials, +or network access. +""" + +from __future__ import annotations + +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from agentops.core.agentops_config import AgentOpsConfig # noqa: E402 +from agentops.core.azd_eval import ( # noqa: E402 + current_recipe_metric_names, + load_current_eval_recipe, +) +from agentops.pipeline import azd_eval_runner # noqa: E402 +from agentops.pipeline.azd_runner import AzdBackendError # noqa: E402 +from fixtures.azd_stub import AzdStub, arg_after, extension_list_json # noqa: E402 + + +CURRENT = "azure.ai.evaluations" + +RECIPE = """ +datasets: + - name: smoke + file: ./datasets/smoke.jsonl + +evaluators: + - name: travel-quality + definition: + type: rubric + dimensions: + - id: accuracy + weight: 5 + description: Factually correct. + +evals: + - name: travel-regression + dataset: smoke + evaluators: + - evaluator: builtin.task_adherence + - evaluator: travel-quality + target: + type: agent + name: travel-agent +""".lstrip() + + +def _workspace(tmp_path: Path, thresholds: dict[str, str] | None = None): + recipe_path = tmp_path / "evals" / "azure.eval.yaml" + recipe_path.parent.mkdir(parents=True, exist_ok=True) + recipe_path.write_text(RECIPE, encoding="utf-8") + dataset = tmp_path / "dataset.jsonl" + dataset.write_text(json.dumps({"input": "hi", "expected": "hello"}) + "\n", encoding="utf-8") + + config = AgentOpsConfig( + version=1, + agent="travel-agent:1", + dataset=dataset, + execution="azd", + thresholds=thresholds or {}, + ) + recipe = load_current_eval_recipe(recipe_path) + return recipe_path, recipe, config + + +def _run_payload(status: str = "completed", **extra): + payload = { + "id": "run-1", + "eval_id": "eval-1", + "status": status, + "result_counts": {"total": 2, "passed": 2, "failed": 0, "errored": 0, "skipped": 0}, + "error": {"code": None, "message": None}, + "report_url": "https://ai.azure.com/report", + "portal_url": "https://ai.azure.com/portal", + } + payload.update(extra) + return payload + + +def _item(index: int, *, status="passed", results=None, data=None): + return { + "id": f"item-{index}", + "run_id": "run-1", + "status": status, + "datasource_item": data + if data is not None + else {"input": f"q{index}", "expected": f"e{index}", "response": f"r{index}"}, + "results": results + if results is not None + else [ + { + "name": "builtin.task_adherence", + "metric": "builtin.task_adherence", + "score": 4, + "passed": True, + "reason": "adhered", + }, + {"name": "accuracy", "metric": "accuracy", "score": 5, "passed": True}, + ], + } + + +def _script(stub: AzdStub, *, statuses=("completed",), items=None, run_payload=None): + stub.expect("azd", "version") + stub.expect( + "extension", "list", "--installed", stdout=extension_list_json(CURRENT) + ) + stub.expect("ai", "eval", "create") + stub.expect( + "ai", "eval", "run", "start", + stdout=json.dumps({"run_id": "run-1", "eval_id": "eval-1", "status": "queued"}), + ) + for status in statuses: + payload = run_payload if status == statuses[-1] and run_payload else _run_payload(status) + stub.expect("ai", "eval", "run", "show", stdout=json.dumps(payload)) + if items is not None: + stub.expect_output_file("ai", "eval", "run", "output", "list", payload=items) + return stub + + +def _execute(tmp_path, monkeypatch, stub, recipe_path, recipe, **kwargs): + stub.install(monkeypatch) + return azd_eval_runner.run_current_eval( + recipe_path, + recipe.primary_eval(), + workspace=tmp_path, + sleep=lambda _seconds: None, + **kwargs, + ) + + +# --------------------------------------------------------------------------- +# T010 - command sequence and flag correctness +# --------------------------------------------------------------------------- + + +def test_command_sequence_matches_the_contract(tmp_path: Path, monkeypatch) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), items=[_item(0), _item(1)]) + + run = _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + stub.assert_exhausted() + assert run.run_id == "run-1" + assert run.eval_id == "eval-1" + assert run.status == "completed" + assert len(run.output_items) == 2 + + ordered = [argv[2:5] for argv in stub.calls if argv[1] == "--no-prompt"] + assert ordered == [ + ["ai", "eval", "create"], + ["ai", "eval", "run"], + ["ai", "eval", "run"], + ["ai", "eval", "run"], + ] + + +def test_path_flag_receives_the_recipe_directory_not_the_file( + tmp_path: Path, monkeypatch +) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), items=[_item(0)]) + + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + for argv in stub.find("--path"): + assert arg_after(argv, "--path") == str(recipe_path.parent) + + +def test_run_start_names_the_eval_by_flag_and_never_positionally( + tmp_path: Path, monkeypatch +) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), items=[_item(0)]) + + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + start = stub.find("run", "start")[0] + assert arg_after(start, "--eval") == "travel-regression" + # Upstream rejects a positional here; the token right after `start` must be a flag. + assert start[start.index("start") + 1].startswith("--") + assert "--no-wait" in start + + +def test_output_list_takes_the_run_id_positionally_and_defeats_paging( + tmp_path: Path, monkeypatch +) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), items=[_item(0)]) + + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + listed = stub.find("output", "list")[0] + assert listed[listed.index("list") + 1] == "run-1" + assert "--all" in listed + assert "--output-file" in listed + # The current surface spells this --output-file; --out-file is the legacy flag. + assert "--out-file" not in listed + + +def test_no_failure_gating_flag_is_ever_passed(tmp_path: Path, monkeypatch) -> None: + """AgentOps owns the gate; delegating it would collapse exit-code meaning.""" + + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), items=[_item(0)]) + + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + stub.assert_no_call_containing("--fail-on") + + +def test_missing_extension_is_an_actionable_configuration_error( + tmp_path: Path, monkeypatch +) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = AzdStub() + stub.expect("azd", "version") + stub.expect("extension", "list", "--installed", stdout=extension_list_json("azure.ai.agents")) + + with pytest.raises(AzdBackendError) as excinfo: + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + message = str(excinfo.value) + assert CURRENT in message + assert azd_eval_runner.CURRENT_MIN_AZD_VERSION in message + assert "azure.ai.agents" not in message.replace(CURRENT, "") + # No evaluation may be created when the surface is unusable. + stub.assert_no_call_containing("ai", "eval", "create") + + +# --------------------------------------------------------------------------- +# T011 - polling and terminal statuses +# --------------------------------------------------------------------------- + + +def test_polling_continues_until_the_run_completes(tmp_path: Path, monkeypatch) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script( + AzdStub(), + statuses=("queued", "running", "completed"), + items=[_item(0)], + ) + + run = _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + assert run.status == "completed" + assert len(stub.find("run", "show")) == 3 + + +def test_unrecognized_status_is_treated_as_non_terminal(tmp_path: Path, monkeypatch) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script( + AzdStub(), + statuses=("mystery-state", "completed"), + items=[_item(0)], + ) + + run = _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + assert run.status == "completed" + assert len(stub.find("run", "show")) == 2 + + +@pytest.mark.parametrize("status", ["failed", "error", "canceled", "cancelled"]) +def test_terminal_failure_statuses_raise_a_runtime_error( + tmp_path: Path, monkeypatch, status: str +) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), statuses=(status,)) + + with pytest.raises(AzdBackendError) as excinfo: + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + message = str(excinfo.value) + assert status in message + assert "run-1" in message + assert "eval-1" in message + + +def test_timeout_preserves_the_identifiers_and_last_status( + tmp_path: Path, monkeypatch +) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), statuses=("running",)) + + with pytest.raises(AzdBackendError) as excinfo: + _execute( + tmp_path, monkeypatch, stub, recipe_path, recipe, timeout_seconds=0.0 + ) + + message = str(excinfo.value) + assert "run-1" in message + assert "eval-1" in message + assert "running" in message + assert "terminal state" in message + + +def test_start_without_a_run_identifier_fails_fast(tmp_path: Path, monkeypatch) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = AzdStub() + stub.expect("azd", "version") + stub.expect("extension", "list", "--installed", stdout=extension_list_json(CURRENT)) + stub.expect("ai", "eval", "create") + stub.expect("ai", "eval", "run", "start", stdout=json.dumps({"status": "queued"})) + + with pytest.raises(AzdBackendError, match="did not return a run identifier"): + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + +# --------------------------------------------------------------------------- +# T012 - parsing, aggregation, tolerant decoding, three-valued verdicts +# --------------------------------------------------------------------------- + + +def _normalize(tmp_path: Path, items, *, thresholds=None, status="completed", run_payload=None): + recipe_path, recipe, config = _workspace(tmp_path, thresholds) + declared = current_recipe_metric_names(recipe, recipe_path) + binding = azd_eval_runner.preflight_bind_thresholds(config.thresholds.keys(), declared) + payload = run_payload or _run_payload(status) + run = azd_eval_runner.CurrentEvalRun( + recipe_path=recipe_path, + run_id="run-1", + status=status, + run_payload=payload, + output_items=tuple(items), + eval_id="eval-1", + eval_name="travel-regression", + report_url="https://ai.azure.com/report", + error_message=azd_eval_runner._run_error_message(payload), + duration_seconds=1.0, + ) + return azd_eval_runner.normalize_to_results( + run, + config=config, + recipe=recipe, + evaluation=recipe.primary_eval(), + metric_binding=binding, + started_at=datetime.now(timezone.utc), + ) + + +def test_aggregate_metric_is_the_mean_of_present_scores(tmp_path: Path) -> None: + items = [ + _item(0, results=[{"metric": "accuracy", "score": 4, "passed": True}]), + _item(1, results=[{"metric": "accuracy", "score": 2, "passed": False}]), + ] + + result = _normalize(tmp_path, items) + + assert result.aggregate_metrics["accuracy"] == 3.0 + + +def test_absent_score_is_excluded_rather_than_counted_as_zero(tmp_path: Path) -> None: + items = [ + _item(0, results=[{"metric": "accuracy", "score": 4, "passed": True}]), + _item(1, results=[{"metric": "accuracy", "score": None, "passed": None}]), + ] + + result = _normalize(tmp_path, items) + + # A zero would drag the mean to 2.0 and fail a >=3 gate on evidence that + # was never actually produced. + assert result.aggregate_metrics["accuracy"] == 4.0 + + +def test_metric_with_no_scores_anywhere_is_absent_from_the_aggregate( + tmp_path: Path, +) -> None: + items = [_item(0, results=[{"metric": "accuracy", "score": "not-a-number"}])] + + result = _normalize(tmp_path, items) + + assert "accuracy" not in result.aggregate_metrics + + +def test_score_arriving_as_a_string_is_decoded(tmp_path: Path) -> None: + items = [_item(0, results=[{"metric": "accuracy", "score": "4", "passed": True}])] + + result = _normalize(tmp_path, items) + + assert result.aggregate_metrics["accuracy"] == 4.0 + + +def test_null_verdict_is_neither_a_pass_nor_a_judged_failure(tmp_path: Path) -> None: + unjudged = azd_eval_runner.parse_output_item( + 0, _item(0, status="", results=[{"metric": "accuracy", "score": 4, "passed": None}]) + ) + + assert unjudged.scores[0].passed is None + assert unjudged.outcome == "failed" # not an explicit pass, so it fails closed + + +def test_sample_with_an_empty_result_list_is_a_failure(tmp_path: Path) -> None: + sample = azd_eval_runner.parse_output_item(0, _item(0, status="", results=[])) + + assert sample.outcome == "failed" + + +def test_declared_status_wins_over_derivation(tmp_path: Path) -> None: + sample = azd_eval_runner.parse_output_item(0, _item(0, status="errored", results=[])) + + assert sample.outcome == "errored" + + +def test_null_run_error_members_are_not_a_failure(tmp_path: Path) -> None: + payload = _run_payload("completed", error={"code": None, "message": None}) + + assert azd_eval_runner._run_error_message(payload) is None + + +def test_non_empty_run_error_message_is_a_failure(tmp_path: Path) -> None: + payload = _run_payload("completed", error={"code": "QuotaExceeded", "message": "no capacity"}) + + detail = azd_eval_runner._run_error_message(payload) + + assert detail is not None + assert "QuotaExceeded" in detail and "no capacity" in detail + + +def test_report_url_wins_over_portal_url(tmp_path: Path, monkeypatch) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), items=[_item(0)]) + + run = _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + assert run.report_url == "https://ai.azure.com/report" + + +def test_rows_are_populated_one_per_sample(tmp_path: Path) -> None: + result = _normalize(tmp_path, [_item(0), _item(1)]) + + assert len(result.rows) == 2 + assert result.rows[0].input == "q0" + assert result.rows[0].expected == "e0" + assert result.rows[0].response == "r0" + assert {metric.name for metric in result.rows[0].metrics} == { + "builtin.task_adherence", + "accuracy", + } + assert result.config["result_granularity"] == "row" + assert result.config["azd_evaluation"]["surface"] == "current" + assert result.config["azd_evaluation"]["extension"] == CURRENT + + +def test_errored_samples_are_kept_as_rows_with_an_error(tmp_path: Path) -> None: + items = [_item(0), _item(1, status="errored", results=[])] + payload = _run_payload( + "completed", + result_counts={"total": 2, "passed": 1, "failed": 0, "errored": 1, "skipped": 0}, + ) + + result = _normalize(tmp_path, items, run_payload=payload) + + assert len(result.rows) == 2 + assert result.rows[1].error is not None + assert result.summary.items_total == 2 + assert result.config["azd_evaluation"]["result_counts"]["errored"] == 1 + + +def test_partial_retrieval_cannot_inflate_the_pass_rate(tmp_path: Path) -> None: + payload = _run_payload( + "completed", + result_counts={"total": 10, "passed": 10, "failed": 0, "errored": 0, "skipped": 0}, + ) + + result = _normalize(tmp_path, [_item(0)], run_payload=payload) + + assert result.summary.items_passed_all == 1 + assert result.config["azd_evaluation"]["retrieval_warnings"] + + +# --------------------------------------------------------------------------- +# T013 - raw artifacts +# --------------------------------------------------------------------------- + + +def test_raw_artifacts_are_written_for_a_successful_run( + tmp_path: Path, monkeypatch +) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + stub = _script(AzdStub(), items=[_item(0)]) + run = _execute(tmp_path, monkeypatch, stub, recipe_path, recipe) + + out = tmp_path / "out" + azd_eval_runner.write_raw_artifacts(run, out) + + assert json.loads((out / "azd_evaluation.json").read_text(encoding="utf-8"))["id"] == "run-1" + assert len(json.loads((out / "azd_eval_output_items.json").read_text(encoding="utf-8"))) == 1 + assert (out / "azd_stdout.log").exists() + + +def test_raw_artifacts_are_written_when_the_run_fails(tmp_path: Path, monkeypatch) -> None: + """T037: a failed run stays diagnosable without re-running the evaluation.""" + + recipe_path, recipe, _ = _workspace(tmp_path) + out = tmp_path / "out" + stub = _script(AzdStub(), statuses=("failed",)) + + with pytest.raises(AzdBackendError): + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe, debug_dir=out) + + payload = json.loads((out / "azd_evaluation.json").read_text(encoding="utf-8")) + assert payload["status"] == "failed" + assert (out / "azd_eval_output_items.json").exists() + + +def test_raw_artifacts_are_written_when_retrieval_fails(tmp_path: Path, monkeypatch) -> None: + recipe_path, recipe, _ = _workspace(tmp_path) + out = tmp_path / "out" + stub = _script(AzdStub()) + stub.expect( + "ai", "eval", "run", "output", "list", + returncode=1, + stderr="listing blew up", + ) + + with pytest.raises(AzdBackendError): + _execute(tmp_path, monkeypatch, stub, recipe_path, recipe, debug_dir=out) + + assert (out / "azd_evaluation.json").exists() + + +# --------------------------------------------------------------------------- +# T022 - pre-flight threshold binding (no azd invocation) +# --------------------------------------------------------------------------- + + +def test_unmatched_threshold_is_a_configuration_error_before_any_azd_call( + tmp_path: Path, monkeypatch +) -> None: + recipe_path, recipe, config = _workspace(tmp_path, {"not_a_metric": ">=3"}) + stub = AzdStub().install(monkeypatch) + declared = current_recipe_metric_names(recipe, recipe_path) + + with pytest.raises(AzdBackendError) as excinfo: + azd_eval_runner.preflight_bind_thresholds(config.thresholds.keys(), declared) + + message = str(excinfo.value) + assert "not_a_metric" in message + assert "no cloud run was consumed" in message.lower() + stub.assert_never_invoked() + + +def test_ambiguous_threshold_is_a_configuration_error_before_any_azd_call( + tmp_path: Path, monkeypatch +) -> None: + stub = AzdStub().install(monkeypatch) + + with pytest.raises(AzdBackendError, match="ambiguous"): + # Both metrics alias to the bare suffix "coherence", so the threshold + # key cannot be bound to exactly one of them. + azd_eval_runner.preflight_bind_thresholds( + ["coherence"], ["builtin.coherence", "rubric.coherence"] + ) + + stub.assert_never_invoked() + + +def test_threshold_binds_to_a_builtin_via_the_narrow_alias_rule(tmp_path: Path) -> None: + recipe_path, recipe, config = _workspace(tmp_path, {"task_adherence": ">=3"}) + declared = current_recipe_metric_names(recipe, recipe_path) + + binding = azd_eval_runner.preflight_bind_thresholds(config.thresholds.keys(), declared) + + assert binding.bound == {"task_adherence": "builtin.task_adherence"} + + +# --------------------------------------------------------------------------- +# T023/T024 - post-run gate behavior +# --------------------------------------------------------------------------- + + +def test_satisfied_threshold_passes(tmp_path: Path) -> None: + result = _normalize(tmp_path, [_item(0), _item(1)], thresholds={"accuracy": ">=4"}) + + assert result.summary.overall_passed is True + + +def test_unsatisfied_threshold_fails_the_gate(tmp_path: Path) -> None: + result = _normalize(tmp_path, [_item(0), _item(1)], thresholds={"accuracy": ">=5.5"}) + + assert result.summary.overall_passed is False + + +def test_declared_metric_the_run_never_emitted_fails_closed(tmp_path: Path) -> None: + """Declared but unemitted is a gate failure, not a crash and not a pass.""" + + items = [_item(0, results=[{"metric": "builtin.task_adherence", "score": 5, "passed": True}])] + + result = _normalize(tmp_path, items, thresholds={"accuracy": ">=1"}) + + assert result.summary.overall_passed is False + failed = [item for item in result.thresholds if not item.passed] + assert [item.metric for item in failed] == ["accuracy"] + assert failed[0].actual == "missing" + assert result.config["azd_evaluation"]["missing_metrics"] == ["accuracy"] + + +def test_rubric_dimension_thresholds_gate_like_builtin_metrics(tmp_path: Path) -> None: + result = _normalize( + tmp_path, + [_item(0), _item(1)], + thresholds={"accuracy": ">=5", "task_adherence": ">=4"}, + ) + + assert result.summary.overall_passed is True + assert {item.metric for item in result.thresholds} == {"accuracy", "task_adherence"} + + +def test_zero_samples_never_reports_a_pass(tmp_path: Path) -> None: + payload = _run_payload( + "completed", + result_counts={"total": 0, "passed": 0, "failed": 0, "errored": 0, "skipped": 0}, + ) + + result = _normalize(tmp_path, [], run_payload=payload) + + assert result.summary.overall_passed is False + + +def test_no_decodable_metrics_never_reports_a_pass(tmp_path: Path) -> None: + items = [_item(0, results=[{"metric": "accuracy", "score": "bogus"}])] + + result = _normalize(tmp_path, items) + + assert result.aggregate_metrics == {} + assert result.summary.overall_passed is False + + +def test_run_level_error_message_prevents_a_pass(tmp_path: Path) -> None: + payload = _run_payload("completed", error={"code": "Partial", "message": "degraded"}) + + result = _normalize(tmp_path, [_item(0)], run_payload=payload) + + assert result.summary.overall_passed is False + assert result.config["azd_evaluation"]["error_message"] is not None diff --git a/tests/unit/test_azd_eval_surface.py b/tests/unit/test_azd_eval_surface.py new file mode 100644 index 00000000..f22f4003 --- /dev/null +++ b/tests/unit/test_azd_eval_surface.py @@ -0,0 +1,402 @@ +"""Tests for cross-surface azd recipe classification, discovery, and parsing. + +Covers quickstart scenario T1.2. Kept in a separate module from +``test_azd_eval.py`` so the pre-existing legacy tests there stay untouched. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from agentops.core.azd_eval import ( + AzdEvalRecipeAmbiguous, + AzdEvalRecipeError, + EvalSurface, + classify_recipe_file, + current_recipe_dataset_path, + current_recipe_metric_names, + load_current_eval_recipe, + resolve_recipe, +) + + +LEGACY_RECIPE = """ +name: travel-agent-eval +agent: + name: travel-agent + kind: prompt-agent + version: 3 +dataset_reference: + name: smoke + version: 1 + local_uri: datasets/smoke.jsonl +evaluators: + - name: builtin.coherence +""".lstrip() + +CURRENT_RECIPE = """ +datasets: + - name: support-agent-regression + file: ./datasets/support-agent-regression.jsonl + +evaluators: + - name: support-agent-quality + source: ./evaluators/support-agent-quality.json + +evals: + - name: support-agent-regression-eval + dataset: support-agent-regression + evaluation_level: turn + max_samples: 50 + evaluators: + - evaluator: builtin.task_adherence + initialization_parameters: + model: gpt-4o + - evaluator: support-agent-quality + version: 2 + target: + type: agent + name: support-agent +x-preview-field: keep-me +""".lstrip() + + +def _write_legacy(root: Path, relative: str = "eval.yaml") -> Path: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(LEGACY_RECIPE, encoding="utf-8") + return path.resolve() + + +def _write_current(root: Path, *, with_rubric: bool = True) -> Path: + path = root / "evals" / "azure.eval.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(CURRENT_RECIPE, encoding="utf-8") + if with_rubric: + rubric = root / "evals" / "evaluators" / "support-agent-quality.json" + rubric.parent.mkdir(parents=True, exist_ok=True) + rubric.write_text( + json.dumps( + { + "dimensions": [ + {"id": "accuracy", "description": "Factually correct.", "weight": 5}, + {"id": "tone", "description": "Stays polite.", "weight": 2}, + ] + } + ), + encoding="utf-8", + ) + return path.resolve() + + +# --------------------------------------------------------------------------- +# Classification +# --------------------------------------------------------------------------- + + +def test_classify_current_recipe_by_evals_sequence(tmp_path: Path) -> None: + path = _write_current(tmp_path) + + assert classify_recipe_file(path) is EvalSurface.CURRENT + + +def test_classify_legacy_recipe_by_agent_mapping(tmp_path: Path) -> None: + path = _write_legacy(tmp_path) + + assert classify_recipe_file(path) is EvalSurface.LEGACY + + +def test_classification_prefers_content_over_filename(tmp_path: Path) -> None: + """A legacy document parked at the current-surface path is still legacy.""" + + path = tmp_path / "evals" / "azure.eval.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(LEGACY_RECIPE, encoding="utf-8") + + assert classify_recipe_file(path) is EvalSurface.LEGACY + + +def test_classification_falls_back_to_location_hint_for_minimal_legacy(tmp_path: Path) -> None: + """A minimal legacy recipe at a legacy location still classifies.""" + + path = tmp_path / "eval.yaml" + path.write_text("name: minimal\nevaluators:\n - builtin.coherence\n", encoding="utf-8") + + assert classify_recipe_file(path) is EvalSurface.LEGACY + + +def test_unclassifiable_document_is_a_configuration_error(tmp_path: Path) -> None: + path = tmp_path / "custom.yaml" + path.write_text("something: else\n", encoding="utf-8") + + with pytest.raises(AzdEvalRecipeError, match="matches neither supported schema"): + classify_recipe_file(path) + + +# --------------------------------------------------------------------------- +# Discovery precedence +# --------------------------------------------------------------------------- + + +def test_resolve_finds_current_recipe_only(tmp_path: Path) -> None: + path = _write_current(tmp_path) + + resolution = resolve_recipe(tmp_path) + + assert resolution.path == path + assert resolution.surface is EvalSurface.CURRENT + assert resolution.skipped == () + assert resolution.explicit is False + assert resolution.extension == "azure.ai.evaluations" + + +def test_resolve_finds_legacy_recipe_at_root(tmp_path: Path) -> None: + path = _write_legacy(tmp_path) + + resolution = resolve_recipe(tmp_path) + + assert resolution.path == path + assert resolution.surface is EvalSurface.LEGACY + assert resolution.extension == "azure.ai.agents" + + +def test_resolve_finds_legacy_recipe_under_src_agent(tmp_path: Path) -> None: + path = _write_legacy(tmp_path, "src/travel-agent/eval.yaml") + + resolution = resolve_recipe(tmp_path) + + assert resolution.path == path + assert resolution.surface is EvalSurface.LEGACY + + +def test_current_surface_wins_and_reports_the_skipped_legacy_recipe(tmp_path: Path) -> None: + legacy = _write_legacy(tmp_path) + current = _write_current(tmp_path) + + resolution = resolve_recipe(tmp_path) + + assert resolution.path == current + assert resolution.surface is EvalSurface.CURRENT + assert resolution.skipped == (legacy,) + + +def test_two_legacy_candidates_are_ambiguous(tmp_path: Path) -> None: + _write_legacy(tmp_path) + _write_legacy(tmp_path, "src/travel-agent/eval.yaml") + + with pytest.raises(AzdEvalRecipeAmbiguous, match="multiple"): + resolve_recipe(tmp_path) + + +def test_two_current_candidates_are_ambiguous(tmp_path: Path) -> None: + _write_current(tmp_path) + (tmp_path / "evals" / "azure.eval.yml").write_text(CURRENT_RECIPE, encoding="utf-8") + + with pytest.raises(AzdEvalRecipeAmbiguous, match="multiple"): + resolve_recipe(tmp_path) + + +def test_explicit_recipe_bypasses_discovery(tmp_path: Path) -> None: + _write_current(tmp_path) + legacy = _write_legacy(tmp_path, "config/nightly/eval.yaml") + + resolution = resolve_recipe(tmp_path, Path("config/nightly/eval.yaml")) + + assert resolution.path == legacy + assert resolution.surface is EvalSurface.LEGACY + assert resolution.explicit is True + assert resolution.skipped == () + + +def test_missing_explicit_recipe_names_the_path(tmp_path: Path) -> None: + with pytest.raises(AzdEvalRecipeError, match="azd eval recipe not found at"): + resolve_recipe(tmp_path, Path("nope/azure.eval.yaml")) + + +def test_no_recipe_names_both_supported_locations(tmp_path: Path) -> None: + with pytest.raises(AzdEvalRecipeError) as excinfo: + resolve_recipe(tmp_path) + + message = str(excinfo.value) + assert "azd eval recipe not found" in message + assert "evals/azure.eval.yaml" in message + assert "eval.yaml" in message + + +# --------------------------------------------------------------------------- +# Current recipe parsing +# --------------------------------------------------------------------------- + + +def test_current_recipe_parses_dataset_file_key(tmp_path: Path) -> None: + path = _write_current(tmp_path) + + recipe = load_current_eval_recipe(path) + + assert recipe.datasets[0].name == "support-agent-regression" + assert recipe.datasets[0].file == "./datasets/support-agent-regression.jsonl" + + +def test_current_recipe_preserves_unknown_top_level_fields(tmp_path: Path) -> None: + path = _write_current(tmp_path) + + recipe = load_current_eval_recipe(path) + + assert recipe.model_dump()["x-preview-field"] == "keep-me" + + +def test_current_recipe_coerces_versions_to_strings(tmp_path: Path) -> None: + path = _write_current(tmp_path) + + recipe = load_current_eval_recipe(path) + evaluation = recipe.primary_eval() + + assert evaluation.evaluators[1].version == "2" + + +def test_primary_eval_rejects_multiple_declared_evaluations(tmp_path: Path) -> None: + path = tmp_path / "evals" / "azure.eval.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + """ +evals: + - name: first + evaluators: + - evaluator: builtin.coherence + - name: second + evaluators: + - evaluator: builtin.fluency +""".lstrip(), + encoding="utf-8", + ) + + recipe = load_current_eval_recipe(path) + + with pytest.raises(AzdEvalRecipeAmbiguous, match="multiple evaluations"): + recipe.primary_eval() + + +def test_primary_eval_rejects_recipe_with_no_evaluations(tmp_path: Path) -> None: + path = tmp_path / "evals" / "azure.eval.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("evals: []\n", encoding="utf-8") + + recipe = load_current_eval_recipe(path) + + with pytest.raises(AzdEvalRecipeError, match="declares no evaluations"): + recipe.primary_eval() + + +# --------------------------------------------------------------------------- +# Declared metric names (pre-flight binding input) +# --------------------------------------------------------------------------- + + +def test_declared_metric_names_cover_builtins_labels_and_rubric_dimensions( + tmp_path: Path, +) -> None: + path = _write_current(tmp_path) + recipe = load_current_eval_recipe(path) + + names = current_recipe_metric_names(recipe, path) + + assert names == { + "builtin.task_adherence", + "support-agent-quality", + "accuracy", + "tone", + } + + +def test_declared_metric_names_use_the_reference_label_when_present(tmp_path: Path) -> None: + path = tmp_path / "evals" / "azure.eval.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + """ +evals: + - name: labelled + evaluators: + - evaluator: builtin.task_adherence + name: adherence_strict +""".lstrip(), + encoding="utf-8", + ) + recipe = load_current_eval_recipe(path) + + assert current_recipe_metric_names(recipe, path) == {"adherence_strict"} + + +def test_declared_metric_names_read_inline_rubric_dimensions(tmp_path: Path) -> None: + path = tmp_path / "evals" / "azure.eval.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + """ +evaluators: + - name: brevity + definition: + type: rubric + dimensions: + - id: length + weight: 1 + description: Answers without restating the question. +evals: + - name: inline-rubric + evaluators: + - evaluator: brevity +""".lstrip(), + encoding="utf-8", + ) + recipe = load_current_eval_recipe(path) + + assert current_recipe_metric_names(recipe, path) == {"brevity", "length"} + + +def test_missing_rubric_definition_is_a_configuration_error(tmp_path: Path) -> None: + path = _write_current(tmp_path, with_rubric=False) + recipe = load_current_eval_recipe(path) + + with pytest.raises(AzdEvalRecipeError, match="rubric definition that does not exist"): + current_recipe_metric_names(recipe, path) + + +# --------------------------------------------------------------------------- +# Dataset provenance +# --------------------------------------------------------------------------- + + +def test_dataset_path_resolves_the_declared_local_file(tmp_path: Path) -> None: + path = _write_current(tmp_path) + recipe = load_current_eval_recipe(path) + + resolved = current_recipe_dataset_path(recipe, recipe.primary_eval(), path) + + assert resolved.endswith("support-agent-regression.jsonl") + assert "evals" in resolved + + +def test_dataset_path_describes_a_trace_source(tmp_path: Path) -> None: + path = tmp_path / "evals" / "azure.eval.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + """ +evals: + - name: trace-eval + source: + type: traces + agent_name: support-agent + lookback_hours: 24 + max_traces: 20 + evaluators: + - evaluator: builtin.task_adherence +""".lstrip(), + encoding="utf-8", + ) + recipe = load_current_eval_recipe(path) + + resolved = current_recipe_dataset_path(recipe, recipe.primary_eval(), path) + + assert resolved.startswith("azd:traces") + assert "agent=support-agent" in resolved diff --git a/tests/unit/test_azd_extension_probe.py b/tests/unit/test_azd_extension_probe.py new file mode 100644 index 00000000..e9619614 --- /dev/null +++ b/tests/unit/test_azd_extension_probe.py @@ -0,0 +1,142 @@ +"""Tests for azd extension availability detection. + +Covers quickstart scenario T1.4. The important case is the false-positive +guard: the human-readable ``azd extension list`` table includes rows for +extensions that are in the registry but NOT installed, so a substring scan +reports availability for something the user cannot actually use. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from agentops.pipeline import azd_runner # noqa: E402 +from fixtures.azd_stub import ( # noqa: E402 + AzdStub, + extension_list_json, + extension_list_table, +) + + +CURRENT = "azure.ai.evaluations" +LEGACY = "azure.ai.agents" + + +def _stub_with_version(monkeypatch) -> AzdStub: + stub = AzdStub() + stub.expect("azd", "version") + stub.install(monkeypatch) + return stub + + +def test_structured_listing_containing_the_extension_means_available(monkeypatch) -> None: + stub = _stub_with_version(monkeypatch) + stub.expect( + "extension", "list", "--installed", "-o", "json", + stdout=extension_list_json(LEGACY, CURRENT), + ) + + assert azd_runner.probe_extension(CURRENT) is True + stub.assert_exhausted() + + +def test_structured_listing_omitting_the_extension_means_unavailable(monkeypatch) -> None: + stub = _stub_with_version(monkeypatch) + stub.expect( + "extension", "list", "--installed", "-o", "json", + stdout=extension_list_json(LEGACY), + ) + + assert azd_runner.probe_extension(CURRENT) is False + stub.assert_exhausted() + + +def test_structured_listing_is_authoritative_and_skips_the_text_fallback(monkeypatch) -> None: + """An empty installed list must not fall through to a text scan.""" + + stub = _stub_with_version(monkeypatch) + stub.expect("extension", "list", "--installed", "-o", "json", stdout="[]") + + assert azd_runner.probe_extension(CURRENT) is False + stub.assert_exhausted() + # Exactly one extension listing, and it is the structured one. A text + # fallback call here would mean the empty structured answer was ignored. + listings = stub.find("extension", "list") + assert len(listings) == 1 + assert "--installed" in listings[0] + + +def test_not_installed_row_in_the_text_table_is_not_availability(monkeypatch) -> None: + """The false-positive guard. + + The table lists every registry extension. A ``Not installed`` row still + contains the extension id, so a substring scan would wrongly report the + extension as usable. + """ + + stub = _stub_with_version(monkeypatch) + stub.expect( + "extension", "list", "--installed", "-o", "json", + stdout=extension_list_json(LEGACY), + ) + + assert azd_runner.probe_extension(CURRENT) is False + + +def test_text_fallback_is_used_when_structured_output_is_unsupported(monkeypatch) -> None: + """Older azd builds without the structured form keep the previous behavior.""" + + stub = _stub_with_version(monkeypatch) + stub.expect( + "extension", "list", "--installed", "-o", "json", + returncode=2, + stderr="unknown flag: --installed", + ) + stub.expect( + "extension", "list", + stdout=extension_list_table((LEGACY, "Up to date")), + ) + + assert azd_runner.probe_extension(LEGACY) is True + stub.assert_exhausted() + + +def test_unparseable_structured_output_falls_back_rather_than_denying(monkeypatch) -> None: + stub = _stub_with_version(monkeypatch) + stub.expect("extension", "list", "--installed", "-o", "json", stdout="not json at all") + stub.expect( + "extension", "list", + stdout=extension_list_table((LEGACY, "Up to date")), + ) + + assert azd_runner.probe_extension(LEGACY) is True + stub.assert_exhausted() + + +def test_missing_azd_binary_means_unavailable(monkeypatch) -> None: + def _raise(*args, **kwargs): + raise FileNotFoundError("azd") + + monkeypatch.setattr(subprocess, "run", _raise) + + assert azd_runner.probe_extension(CURRENT) is False + + +def test_azd_available_delegates_to_the_shared_probe(monkeypatch) -> None: + """The legacy entry point keeps its signature and answers for the legacy id.""" + + seen: list[str] = [] + + def _probe(extension_name: str, *, cwd=None) -> bool: + seen.append(extension_name) + return True + + monkeypatch.setattr(azd_runner, "probe_extension", _probe) + + assert azd_runner.azd_available() is True + assert seen == [LEGACY] diff --git a/tests/unit/test_azd_runner.py b/tests/unit/test_azd_runner.py index 6cfa5a0c..93d56a81 100644 --- a/tests/unit/test_azd_runner.py +++ b/tests/unit/test_azd_runner.py @@ -227,6 +227,40 @@ def test_normalize_to_results_binds_azd_metrics_and_thresholds(tmp_path: Path) - } +def test_legacy_normalization_remains_aggregate_without_surface_marker(tmp_path: Path) -> None: + recipe_path = tmp_path / "eval.yaml" + _write_recipe(recipe_path) + recipe = load_eval_recipe(recipe_path) + config = AgentOpsConfig( + version=1, + agent="travel-agent:1", + dataset="ignored.jsonl", + execution="azd", + ) + azd_run = azd_runner.AzdEvalRun( + recipe_path=recipe_path, + payload={ + "status": "completed", + "metrics": [{"name": "builtin.coherence", "score": 4.2}], + }, + run_id="run-1", + status="completed", + stdout="{}", + stderr="", + duration_seconds=1.0, + ) + + result = azd_runner.normalize_to_results( + azd_run, + config=config, + recipe=recipe, + started_at=datetime.now(timezone.utc), + ) + + assert result.config["result_granularity"] == "aggregate" + assert "surface" not in result.config["azd_evaluation"] + + def test_normalize_to_results_fails_closed_for_unmatched_threshold(tmp_path: Path) -> None: recipe_path = tmp_path / "eval.yaml" _write_recipe(recipe_path) diff --git a/tests/unit/test_eval_analysis.py b/tests/unit/test_eval_analysis.py index fc8e68f9..112138e6 100644 --- a/tests/unit/test_eval_analysis.py +++ b/tests/unit/test_eval_analysis.py @@ -362,3 +362,80 @@ def test_cli_eval_analyze_agentless_exits_zero(tmp_path: Path) -> None: assert result.exit_code == 0, result.stdout assert "observability" in result.stdout.lower() + + +def _write_azd_analysis_config(root: Path) -> None: + (root / "data.jsonl").write_text( + '{"input": "hello", "expected": "hi"}\n', + encoding="utf-8", + ) + (root / "agentops.yaml").write_text( + "version: 1\nagent: travel-agent:1\ndataset: data.jsonl\nexecution: azd\n", + encoding="utf-8", + ) + + +def test_eval_analysis_reports_current_azd_recipe_surface(tmp_path: Path) -> None: + _write_azd_analysis_config(tmp_path) + recipe = tmp_path / "evals" / "azure.eval.yaml" + recipe.parent.mkdir(parents=True) + recipe.write_text("evals: []\n", encoding="utf-8") + + analysis = analyze_eval_project(tmp_path) + + assert analysis.config_status == "ready" + signal = next(item for item in analysis.signals if item.key == "azd_eval_recipe") + assert signal.path == "evals/azure.eval.yaml" + assert "current surface" in signal.detail + assert "azure.ai.evaluations" in signal.detail + + +def test_eval_analysis_reports_legacy_azd_recipe_surface(tmp_path: Path) -> None: + _write_azd_analysis_config(tmp_path) + (tmp_path / "eval.yaml").write_text("name: travel-agent-eval\n", encoding="utf-8") + + analysis = analyze_eval_project(tmp_path) + + assert analysis.config_status == "ready" + signal = next(item for item in analysis.signals if item.key == "azd_eval_recipe") + assert signal.path == "eval.yaml" + assert "legacy surface" in signal.detail + assert "azure.ai.agents" in signal.detail + + +def test_eval_analysis_reports_current_selection_and_skipped_legacy_recipe( + tmp_path: Path, +) -> None: + _write_azd_analysis_config(tmp_path) + current = tmp_path / "evals" / "azure.eval.yaml" + current.parent.mkdir(parents=True) + current.write_text("evals: []\n", encoding="utf-8") + legacy = tmp_path / "eval.yaml" + legacy.write_text("name: travel-agent-eval\n", encoding="utf-8") + + analysis = analyze_eval_project(tmp_path) + + selected = next(item for item in analysis.signals if item.key == "azd_eval_recipe") + skipped = next( + item for item in analysis.signals if item.key == "azd_eval_recipe_skipped" + ) + assert selected.path == "evals/azure.eval.yaml" + assert "current surface" in selected.detail + assert skipped.path == "eval.yaml" + assert "current surface takes precedence" in skipped.detail + + +def test_eval_analysis_reports_actionable_gap_when_azd_recipe_missing( + tmp_path: Path, +) -> None: + _write_azd_analysis_config(tmp_path) + + analysis = analyze_eval_project(tmp_path) + + assert analysis.config_status == "incomplete" + signal = next(item for item in analysis.signals if item.key == "azd_eval_recipe_gap") + assert "evals/azure.eval.yaml" in signal.detail + assert "eval.yaml" in signal.detail + assert "src//eval.yaml" in signal.detail + assert "Required action" in signal.detail + assert signal.detail in analysis.warnings