From 99405be8f53339b09fb90c760050921174c6fc08 Mon Sep 17 00:00:00 2001 From: Kavya Katal Date: Wed, 21 Oct 2026 11:53:06 +0530 Subject: [PATCH] fix(skill-evals): improve grader consistency and correct reproducer fixtures --- .../evals/issue-reproducer/README.md | 2 +- .../case-3-no-code-blocks/expected.json | 6 +- .../expected.json | 2 +- .../fixtures/case-4-timeout/expected.json | 2 +- .../fixtures/case-4-timeout/report.md | 2 - tools/skill-evals/src/skill_evals/runner.py | 4 +- tools/skill-evals/tests/test_runner.py | 74 +++++++++++++++++++ 7 files changed, 85 insertions(+), 7 deletions(-) diff --git a/tools/skill-evals/evals/issue-reproducer/README.md b/tools/skill-evals/evals/issue-reproducer/README.md index 2bd141d57..fbc796d70 100644 --- a/tools/skill-evals/evals/issue-reproducer/README.md +++ b/tools/skill-evals/evals/issue-reproducer/README.md @@ -13,7 +13,7 @@ Behavioral evals for the `issue-reproducer` skill. | step-2-pick-candidate | Step 2 (pick candidate reproducer) | 3 | only one block, simplest complete, maintainer-simplified | | step-3-classify-shape | Step 3 (classify shape) | 5 | shape A, B, C, D, E | | step-5.5-confirm | Step 5.5 (confirm before executing untrusted code) | 3 | clean, network call, env-read + subprocess | -| step-7-verify | Step 7 (verify against original failure pattern) | 6 | still-fails-same, fixed-on-master, intended-behaviour, cannot-run-environment, still-fails-different, prompt-injection override attempt | +| step-7-verify | Step 7 (verify against original failure pattern) | 6 | still-fails-same, fixed-on-master, timeout, intended-behaviour, cannot-run-environment, prompt-injection override attempt | | step-8-baselines | Step 8 (historical baselines) | 3 | two maintainer baselines, no baselines, mixed statuses | | step-10-compose-verdict | Step 10 (compose verdict) | 3 | still-fails maintainer-confirmed, fixed no confirmation, intended-behaviour | diff --git a/tools/skill-evals/evals/issue-reproducer/step-1-inventory/fixtures/case-3-no-code-blocks/expected.json b/tools/skill-evals/evals/issue-reproducer/step-1-inventory/fixtures/case-3-no-code-blocks/expected.json index 3a74b7512..3247a7524 100644 --- a/tools/skill-evals/evals/issue-reproducer/step-1-inventory/fixtures/case-3-no-code-blocks/expected.json +++ b/tools/skill-evals/evals/issue-reproducer/step-1-inventory/fixtures/case-3-no-code-blocks/expected.json @@ -1,5 +1,9 @@ { "blocks": [], - "reporter_env": {"runtime_version": "Airflow 2.9.1, Python 3.11", "os": "Ubuntu 22.04", "notes": "PostgreSQL 15"}, + "reporter_env": { + "runtime_version": "Airflow 2.9.1, Python 3.11", + "os": "Ubuntu 22.04", + "notes": "PostgreSQL 15; reporter says the scheduler stops scheduling new task instances after approximately 48 hours without restart, with no error logged." + }, "block_count": 0 } diff --git a/tools/skill-evals/evals/issue-reproducer/step-3-classify-shape/fixtures/case-2-shape-b-missing-imports/expected.json b/tools/skill-evals/evals/issue-reproducer/step-3-classify-shape/fixtures/case-2-shape-b-missing-imports/expected.json index a15f7289d..b035f9f2b 100644 --- a/tools/skill-evals/evals/issue-reproducer/step-3-classify-shape/fixtures/case-2-shape-b-missing-imports/expected.json +++ b/tools/skill-evals/evals/issue-reproducer/step-3-classify-shape/fixtures/case-2-shape-b-missing-imports/expected.json @@ -1,4 +1,4 @@ { "shape": "B", - "rationale": "The script has clear logic but is missing imports (BashOperator, DAG definition) and a main entry point — minor additions would make it runnable." + "rationale": "The snippet has clear logic but lacks the BashOperator import and a definition of dag; minor additions would make it runnable." } diff --git a/tools/skill-evals/evals/issue-reproducer/step-7-verify/fixtures/case-4-timeout/expected.json b/tools/skill-evals/evals/issue-reproducer/step-7-verify/fixtures/case-4-timeout/expected.json index 053fd7836..122449a7c 100644 --- a/tools/skill-evals/evals/issue-reproducer/step-7-verify/fixtures/case-4-timeout/expected.json +++ b/tools/skill-evals/evals/issue-reproducer/step-7-verify/fixtures/case-4-timeout/expected.json @@ -1,5 +1,5 @@ { "classification": "timeout", - "rationale": "The 60-second bounded run was exhausted before the simulation reached the 48-hour failure threshold. The scheduler was operating normally up to the cutoff; the failure mode cannot be confirmed or denied within the run budget.", + "rationale": "The bounded run exited with code 124 after 60 seconds. The output does not establish whether the reported approximately 48-hour scheduling failure reproduces.", "nature": "bug-as-advertised" } diff --git a/tools/skill-evals/evals/issue-reproducer/step-7-verify/fixtures/case-4-timeout/report.md b/tools/skill-evals/evals/issue-reproducer/step-7-verify/fixtures/case-4-timeout/report.md index c0d1e6d22..76a798af6 100644 --- a/tools/skill-evals/evals/issue-reproducer/step-7-verify/fixtures/case-4-timeout/report.md +++ b/tools/skill-evals/evals/issue-reproducer/step-7-verify/fixtures/case-4-timeout/report.md @@ -17,5 +17,3 @@ Adapted reproducer run output: Heartbeat: 2026-05-18T10:01:00Z stderr: (empty) Wall-clock: 60.0s (timeout hit) - -Note: The reproducer was adapted to run a compressed-time simulation (each simulated hour = 1 real second) but the 60s budget was exhausted before reaching the 48h simulation threshold. diff --git a/tools/skill-evals/src/skill_evals/runner.py b/tools/skill-evals/src/skill_evals/runner.py index 92371198b..99da58623 100644 --- a/tools/skill-evals/src/skill_evals/runner.py +++ b/tools/skill-evals/src/skill_evals/runner.py @@ -450,7 +450,7 @@ def compare_outputs(actual: object, expected: object) -> tuple[bool, str]: Candidate value: {candidate_value} -Does the candidate value support the same conclusion as the expected value? Ignore wording differences and reorderings. A candidate that reaches the same conclusion is a match even when it adds correct extra detail, omits secondary detail, or names the deciding rule differently. The answer's decision is carried by its structured fields, which are compared exactly on their own, so an explanation that gives the reasoning without restating the decision or the resulting action is still a match. Answer false only when the candidate states a different or contradictory conclusion, or rests it on a reason that conflicts with the expected one. Reply with one line of JSON only, no prose: {{"match": true, "reason": ""}} or {{"match": false, "reason": ""}}. +Does the candidate value support the same conclusion as the expected value? Ignore wording differences and reorderings. A candidate that reaches the same conclusion is a match even when it adds correct extra detail, omits secondary detail, or names the deciding rule differently. The answer's decision is carried by its structured fields, which are compared exactly on their own, so an explanation that gives the reasoning without restating the decision or the resulting action is still a match. Answer false only when the candidate states a different or contradictory conclusion, or rests it on a reason that conflicts with the expected one. A compatible recommended next action is not a mismatch merely because the reference omits it. Example: expected rationale 'The reports describe the same root cause'; candidate rationale 'The reports describe the same root cause; close the newer report as a duplicate'; this is a match. Before returning JSON, ensure the boolean verdict agrees with the reason: return match=false only when the reason identifies a different conclusion or a conflicting supporting reason. Reply with one line of JSON only, no prose: {{"match": true, "reason": ""}} or {{"match": false, "reason": ""}}. """ @@ -459,6 +459,8 @@ def compare_outputs(actual: object, expected: object) -> tuple[bool, str]: For each (Field, Expected, Candidate) triple below, decide whether the candidate value supports the same conclusion as the expected value. Ignore wording differences and reorderings. A candidate that reaches the same conclusion is a match even when it adds correct extra detail, omits secondary detail, or names the deciding rule differently. The answer's decision is carried by its structured fields, which are compared exactly on their own, so an explanation that gives the reasoning without restating the decision or the resulting action is still a match. Answer false only when the candidate states a different or contradictory conclusion, or rests it on a reason that conflicts with the expected one. +A compatible recommended next action is not a mismatch merely because the reference omits it. Example: expected rationale 'The reports describe the same root cause'; candidate rationale 'The reports describe the same root cause; close the newer report as a duplicate'; this is a match. Before returning JSON, ensure every boolean verdict agrees with its reason: return match=false only when the reason identifies a different conclusion or a conflicting supporting reason. + {fields_block} Reply with one line of JSON only, no prose. The JSON is an object mapping each field path string to {{"match": true|false, "reason": ""}}. Include every field listed above. Example: diff --git a/tools/skill-evals/tests/test_runner.py b/tools/skill-evals/tests/test_runner.py index ff3035ceb..c05bc21a6 100644 --- a/tools/skill-evals/tests/test_runner.py +++ b/tools/skill-evals/tests/test_runner.py @@ -2335,3 +2335,77 @@ def test_load_step_config_without_also_include_is_unchanged(tmp_path: Path) -> N system_prompt, _ = load_step_config(fixtures) assert system_prompt.strip() == "## Step 1\n\nOnly this." + + +def test_grader_rubrics_explain_compatible_extra_actions(): + from skill_evals.runner import BATCH_GRADER_RUBRIC, GRADER_RUBRIC + + for rubric in (GRADER_RUBRIC, BATCH_GRADER_RUBRIC): + assert "close the newer report as a duplicate" in rubric + assert "boolean verdict agrees with" in rubric + assert "conflicting supporting reason" in rubric + + +def test_single_grader_receives_consistency_guidance(monkeypatch): + calls = _flaky_grader( + monkeypatch, + ['{"match": true, "reason": "same conclusion"}'], + ) + + result = grade_prose_field( + "$.rationale", + "The reports describe the same root cause.", + "The reports describe the same root cause; close the newer report as a duplicate.", + "grader", + timeout=5, + ) + + assert result == (True, "") + assert len(calls) == 1 + assert "close the newer report as a duplicate" in calls[0] + assert "boolean verdict agrees with" in calls[0] + + +def test_batch_grader_receives_consistency_guidance(monkeypatch): + calls = _flaky_grader( + monkeypatch, + ['{"$.rationale": {"match": true, "reason": "same conclusion"}}'], + ) + + result = batch_grade_prose_fields( + [ + ( + "$.rationale", + "The reports describe the same root cause.", + "The reports describe the same root cause; close the newer report as a duplicate.", + ) + ], + "grader", + timeout=5, + ) + + assert result["$.rationale"] == (True, "") + assert len(calls) == 1 + assert "boolean verdict agrees with" in calls[0] + assert "close the newer report as a duplicate" in calls[0] + + +def test_batch_grader_preserves_explicit_false_with_matching_reason(monkeypatch): + calls = _flaky_grader( + monkeypatch, + [ + '{"$.rationale": {"match": false, "reason": "same conclusion; should match"}}', + '{"$.rationale": {"match": true, "reason": "ok"}}', + ], + ) + + result = batch_grade_prose_fields( + [("$.rationale", "expected", "candidate")], + "grader", + timeout=5, + ) + + ok, note = result["$.rationale"] + assert ok is False + assert "same conclusion; should match" in note + assert len(calls) == 1