diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index 706d87a92..c20855604 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -98,11 +98,11 @@ The evaluator then joins trajectories and ground truth by scenario id. ### Optional evaluation metadata -Scenario folders may also include `groundtruth_eval.json`. This file provides -scorer-specific metadata while keeping the canonical answer in `groundtruth.txt`. -It is currently used by clarification-abstain-response (CAR) scenarios, where -the exact wording can vary but the response mode and key domain terms must be -correct. +Scenario folders may also include `groundtruth_eval.json`, which carries +scorer-specific metadata alongside the canonical answer in `groundtruth.txt`. +Clarification-abstain-response (CAR) scenarios ship one, but the `static_json` +scorer does not read it: a CAR answer is scored on its top-level mode key, and +the expected mode is the single key of `groundtruth.txt`. Example `groundtruth.txt`: @@ -112,30 +112,6 @@ Example `groundtruth.txt`: } ``` -Example `groundtruth_eval.json`: - -```json -{ - "mode": "clarification", - "required_terms": ["main unit"], - "optional_terms": ["asset"], - "must_have_exactly_one_mode_key": true -} -``` - -Supported CAR metadata fields: - -| Field | Meaning | -| -------------------------------- | ----------------------------------------------------------------------- | -| `mode` | Expected top-level key: `response`, `clarification`, or `abstain` | -| `required_terms` | Terms that indicate the model addressed the required ambiguity or fact | -| `optional_terms` | Informative terms reported for analysis, but not required for passing | -| `must_have_exactly_one_mode_key` | Whether the model must return exactly one top-level CAR mode key | - -If `groundtruth_eval.json` is absent, the scorer derives CAR metadata from -`groundtruth.txt` for one-key answers using `response`, `clarification`, or -`abstain`. - --- ## How Matching Works @@ -248,9 +224,9 @@ The scorer marks a scenario as passed only when the structured answer is a stric ### CAR metrics and pass criteria -For CAR scenarios, the scorer evaluates the answer as a mode-selection task plus -required-term coverage. The model answer should be a single JSON object using -exactly one of these top-level keys: +For CAR scenarios, the scorer evaluates only the mode the model chose. The +model answer should be a single JSON object using exactly one of these +top-level keys: ```text response | clarification | abstain @@ -258,32 +234,21 @@ response | clarification | abstain A CAR scenario passes when: -1. the model returns exactly one top-level CAR mode key, -2. that key matches the expected `mode`, and -3. at least one `required_terms` entry appears in the model answer value. - -The CAR soft score is reported as: - -```text -car_score = 0.6 * mode_key_match + 0.4 * mode_term_coverage -``` +1. the model returns a JSON object with exactly one top-level key, and +2. that key matches the expected mode (case-insensitive). -where `mode_term_coverage` is the fraction of required terms found in the answer -value. Optional terms are reported for diagnostics and do not affect pass/fail. +The value under the key is not scored. `score`, `f1` and +`strict_exact_match_accuracy` are all 1.0 on a pass and 0.0 otherwise. Additional CAR report fields: -| Metric | Meaning | -| ----------------------------------- | ------------------------------------------------------------------ | -| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | -| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | -| `mode_required_terms` | Required terms used for this scenario | -| `mode_matched_terms` | Required terms found in the model answer | -| `mode_term_coverage` | Fraction of required terms found | -| `mode_optional_terms` | Optional diagnostic terms | -| `mode_matched_optional_terms` | Optional diagnostic terms found | -| `mode_optional_term_coverage` | Fraction of optional terms found, or null if none were configured | -| `car_score` | Weighted CAR soft score | +| Metric | Meaning | +| ---------------------- | -------------------------------------------------------------------- | +| `mode_key_match` | 1.0 when the returned mode key matches the expected mode | +| `mode_exactly_one_key` | 1.0 when the answer has exactly one top-level key | +| `mode_required_terms` | Diagnostic terms extracted from the gold answer value | +| `mode_matched_terms` | Diagnostic terms found in the model answer value | +| `mode_term_coverage` | Fraction of diagnostic terms found; reported only, does not affect pass or score | --- diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index a70b5d65f..160d3f986 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -505,13 +505,11 @@ def _dedupe_terms(terms: list[str]) -> list[str]: def _extract_required_mode_terms(value: Any) -> list[str]: - """Extract lightweight required terms for mode-selection scenarios. + """Extract lightweight diagnostic terms for mode-selection scenarios. - The mode scenarios are not exact-string tasks: answers can be phrased - differently as long as they choose the correct mode and mention the - important ambiguity/evidence. We therefore extract only high-signal terms: - quoted phrases, asset/fault identifiers, yes/no stance, and a small - domain-term allowlist. + The terms are reported alongside the CAR score but do not affect it. We + extract only high-signal terms: quoted phrases, asset/fault identifiers, + yes/no stance, and a small domain-term allowlist. """ text = str(value) terms: list[str] = [] @@ -549,6 +547,13 @@ def _is_mode_gold_answer(value: Any) -> bool: def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: + """Score a CAR answer on its top-level mode key alone. + + The answer passes when it is a single-key object whose key is the gold + mode (``response``, ``clarification`` or ``abstain``). The value under + the key does not affect pass or score; required-term coverage is still + reported in the ``mode_*`` fields for analysis. + """ gold = parse_structured_answer(gold_answer) model = parse_structured_answer(model_answer) @@ -562,6 +567,8 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: model_value = next(iter(model.values())) if model_is_dict and model_exactly_one_key else "" key_match = model_exactly_one_key and model_key == gold_key + + # Diagnostics only: which gold terms the answer value mentions. required_terms = _extract_required_mode_terms(gold_value) model_text = _normalize_text_for_terms(model_value) matched_terms = [ @@ -583,20 +590,6 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: ) ] - for term in required_terms: - matched = term in matched_terms - details.append( - KeyComparison( - key=f"answer.required_term.{term}", - gold_value=term, - model_value=term if matched else "MISSING", - exact=matched, - match_type="term_present" if matched else "term_missing", - similarity=1.0 if matched else 0.0, - accepted=matched, - ) - ) - missing_keys = [] if key_match else [f"answer.{gold_key}"] extra_keys = [] if model_is_dict: @@ -608,36 +601,24 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: else: extra_keys = ["answer"] if model is not None else [] - total_gold_keys = 1 + len(required_terms) - total_model_keys = 1 + len(required_terms) + len(extra_keys) - exact_matches = (1 if key_match else 0) + len(matched_terms) - - precision = exact_matches / total_model_keys if total_model_keys else 0.0 - recall = exact_matches / total_gold_keys if total_gold_keys else 0.0 - f1 = ( - 2 * precision * recall / (precision + recall) - if precision + recall > 0 - else 0.0 - ) - strict_exact = 1.0 if key_match and term_coverage == 1.0 and not extra_keys else 0.0 + key_score = 1.0 if key_match else 0.0 return StaticJsonScore( - partial_match_accuracy=recall, - partial_exact_match_accuracy=recall, - strict_exact_match_accuracy=strict_exact, - partial_similarity_score=sum(item.similarity for item in details) - / total_gold_keys, + partial_match_accuracy=key_score, + partial_exact_match_accuracy=key_score, + strict_exact_match_accuracy=key_score, + partial_similarity_score=details[0].similarity, partial_numeric_match_accuracy=0.0, range_match_accuracy=0.0, delta_1_match_accuracy=0.0, - precision=precision, - recall=recall, - f1=f1, - total_gold_keys=total_gold_keys, - total_model_keys=total_model_keys, + precision=key_score, + recall=key_score, + f1=key_score, + total_gold_keys=1, + total_model_keys=len(model_keys) if model_is_dict else len(extra_keys), matched_keys=1 if key_match else 0, - accepted_value_matches=exact_matches, - exact_value_matches=exact_matches, + accepted_value_matches=1 if key_match else 0, + exact_value_matches=1 if key_match else 0, numeric_gold_keys=0, numeric_value_matches=0, range_eligible_keys=0, @@ -647,7 +628,7 @@ def _evaluate_mode_json(gold_answer: Any, model_answer: Any) -> StaticJsonScore: missing_keys=missing_keys, extra_keys=extra_keys, details=details, - mode_key_match=1.0 if key_match else 0.0, + mode_key_match=key_score, mode_exactly_one_key=1.0 if model_exactly_one_key else 0.0, mode_required_terms=required_terms, mode_matched_terms=matched_terms, diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 8a2c5bbcc..34003588d 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -328,7 +328,7 @@ def test_mode_clarification_fails_when_mode_key_is_wrong(): assert score.extra_keys == ["answer.response"] -def test_mode_abstain_requires_lately_dataset_date_time_terms(): +def test_mode_abstain_reports_lately_dataset_date_time_terms(): score = evaluate_static_json( { "abstain": ( @@ -396,95 +396,50 @@ def test_mode_requires_exactly_one_top_level_key(): assert score.extra_keys == ["answer.clarification", "answer.response"] -def test_car_metadata_passes_on_mode_and_required_terms_only(): +def test_mode_passes_on_key_alone_when_terms_are_missing(): score = evaluate_static_json( {"clarification": "Which asset do you mean by 'the main unit'?"}, - {"clarification": "Which unit are you referring to as the main unit?"}, - evaluation_metadata={ - "mode": "clarification", - "required_terms": ["main unit"], - "optional_terms": ["asset"], - }, + {"clarification": "Which asset should I analyze?"}, ) assert score.strict_exact_match_accuracy == 1.0 - assert score.car_score == 1.0 + assert score.f1 == 1.0 assert score.mode_key_match == 1.0 assert score.mode_required_terms == ["main unit"] - assert score.mode_matched_terms == ["main unit"] - assert score.mode_optional_terms == ["asset"] - assert score.mode_matched_optional_terms == [] - assert score.mode_optional_term_coverage == 0.0 + assert score.mode_matched_terms == [] + assert score.mode_term_coverage == 0.0 -def test_car_optional_terms_do_not_inflate_similarity(): +def test_mode_fails_on_wrong_key_even_when_terms_match(): score = evaluate_static_json( {"clarification": "Which asset do you mean by 'the main unit'?"}, - { - "clarification": ( - "Which asset do you mean by the main unit? Please provide the " - "asset tag." - ) - }, - evaluation_metadata={ - "mode": "clarification", - "required_terms": ["main unit"], - "optional_terms": ["asset", "asset tag"], - }, + {"response": "The main unit appears to be PMP42144."}, ) - assert score.strict_exact_match_accuracy == 1.0 - assert score.partial_similarity_score == 1.0 - assert score.mode_optional_term_coverage == 1.0 + assert score.strict_exact_match_accuracy == 0.0 + assert score.f1 == 0.0 + assert score.mode_key_match == 0.0 -def test_car_metadata_passes_when_any_required_term_matches(): +def test_mode_key_is_case_insensitive(): score = evaluate_static_json( {"abstain": "Cannot determine because date or time is missing."}, - {"abstain": "Cannot determine this from the available date fields."}, - evaluation_metadata={ - "mode": "abstain", - "required_terms": ["lately", "date", "time"], - }, + '{"Abstain": "Not enough information."}', ) assert score.strict_exact_match_accuracy == 1.0 - assert score.car_score == 0.7333333333333333 - assert score.mode_key_match == 1.0 - assert score.mode_matched_terms == ["date"] - assert score.mode_term_coverage == 1 / 3 - - -def test_car_metadata_fails_when_required_term_is_missing(): - score = evaluate_static_json( - {"clarification": "Which asset do you mean by 'the main unit'?"}, - {"clarification": "Which asset should I analyze?"}, - evaluation_metadata={ - "mode": "clarification", - "required_terms": ["main unit"], - }, - ) - - assert score.strict_exact_match_accuracy == 0.0 - assert score.car_score == 0.6 assert score.mode_key_match == 1.0 - assert score.mode_term_coverage == 0.0 -def test_car_metadata_fails_when_mode_key_is_wrong_even_with_required_term(): +def test_mode_fails_when_answer_is_not_an_object(): score = evaluate_static_json( - {"clarification": "Which asset do you mean by 'the main unit'?"}, - {"response": "The main unit appears to be PMP42144."}, - evaluation_metadata={ - "mode": "clarification", - "required_terms": ["main unit"], - }, + {"abstain": "Cannot determine because date or time is missing."}, + "I cannot answer this question.", ) assert score.strict_exact_match_accuracy == 0.0 - assert score.car_score == 0.4 assert score.mode_key_match == 0.0 - assert score.mode_term_coverage == 1.0 + assert score.mode_exactly_one_key == 0.0 def test_count_only_exact_match(): @@ -534,30 +489,24 @@ def test_static_json_scorer_wrapper_exact_match(): assert result.details["strict_exact_match_accuracy"] == 1.0 -def test_static_json_scorer_uses_car_metadata_score(): +def test_static_json_scorer_scores_car_on_mode_key_only(): scenario = Scenario.from_raw( { "id": "151", "text": "Clarify main unit.", "expected_answer": '{"clarification": "Which asset do you mean by the main unit?"}', - "evaluation_metadata": { - "mode": "clarification", - "required_terms": ["main unit"], - }, "scoring_method": "static_json", } ) scorer = StaticJsonScorer() - result = scorer( - scenario, - '{"clarification": "Which asset is the main unit?"}', - "", - ) + right_key = scorer(scenario, '{"clarification": "Which pump?"}', "") + wrong_key = scorer(scenario, '{"response": "The main unit is PMP42144."}', "") - assert result.passed is True - assert result.score == 1.0 - assert result.details["car_score"] == 1.0 + assert right_key.passed is True + assert right_key.score == 1.0 + assert wrong_key.passed is False + assert wrong_key.score == 0.0 def test_identifier_with_a_separator_is_not_read_as_a_count():