diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index ae979a177..bdaa2b92b 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -279,7 +279,12 @@ def test_no_graded_rows_is_not_an_error(self) -> None: ) self.assertEqual(status, 0) - self.assertIn("no rows with an expected decision", out.getvalue()) + report = out.getvalue() + self.assertIn("1 of 1 rows with an expected decision had no audit", report) + self.assertIn( + "none of the 1 rows with an expected decision could be graded", report + ) + self.assertNotIn("no rows with an expected decision", report) class BoundaryGateTest(unittest.TestCase): @@ -342,3 +347,156 @@ def test_total_boundary_rejection_fails_the_run(self) -> None: if __name__ == "__main__": unittest.main() + + +class CalibrationCoverageTest(unittest.TestCase): + """#1166: rows that never reach the C.3 fit must count against the gate.""" + + def _rows(self) -> list: + return [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "alpha2", ["616-305"]), + _golden_row("p3", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + _golden_row("n2", "hard_negative", "beta2", []), + _golden_row("e1", "explicit", "delta", ["616-305"]), + ] + + def _pipe(self) -> "CountingPipeline": + return CountingPipeline( + { + "alpha": [("616-305", 5.0), ("999-999", 0.1)], + "alpha2": [("616-305", 4.0), ("999-999", 0.2)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + "beta2": [("111-111", 2.0), ("222-222", 1.9)], + "delta": [("616-305", 1.0)], + # "empty" yields no candidates on purpose + } + ) + + def test_eligible_counts_only_the_two_calibration_slices(self) -> None: + self.assertEqual(harness.calibration_eligible(self._rows()), 5) + + def test_partial_drop_fails_the_gate_and_says_how_many(self) -> None: + rows, pipe = self._rows(), self._pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1) + self.assertIsNotNone(scaler, "T is still fitted so C.4 can report") + self.assertIn("reached calibration: 4/5", report) + self.assertIn("1 of 5 eligible rows did not reach the fit", report) + self.assertIn("FAILED (gate denominator incomplete)", report) + + def test_full_coverage_gates_on_ece_only(self) -> None: + rows = [r for r in self._rows() if r.id != "p3"] + pipe = self._pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertIsNotNone(scaler) + self.assertIn("reached calibration: 4/4", report) + self.assertNotIn("denominator incomplete", report) + self.assertIn("(calibrated, in-sample)", report) + # Expected outcome computed independently of the report text: refit T on + # the same calibration set and apply the 0.10 bar ourselves. + from application.utils.librarian.calibration.temperature import ( + expected_calibration_error, + fit_temperature, + ) + + logit_sets, labels = harness.calibration_set(rows, audits) + scaler_check = fit_temperature(logit_sets, labels) + ece_check = expected_calibration_error( + [scaler_check.confidence(s) for s in logit_sets], labels + ) + self.assertAlmostEqual(scaler.temperature, scaler_check.temperature, places=6) + self.assertEqual(status, 0 if ece_check < 0.10 else 1) + + def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row( + "d2", "hard_negative", "beta", [], reason_code="BELOW_THRESHOLD" + ), + _golden_row("d3", "positive", "never-audited", ["616-305"]), + ] + pipe = CountingPipeline( + { + "alpha": [("616-305", 20.0), ("999-999", 0.0)], + "beta": [("111-111", 1.0), ("222-222", 0.99)], + } + ) + audits = harness.live_audits(rows[:2], pipe, pipe) + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, audits, TemperatureScaler(1.0), 0.80 + ) + report = out.getvalue() + self.assertEqual(status, 0, "still informational") + self.assertIn("1 of 3 rows with an expected decision had no audit", report) + self.assertIn("decision (C.4, 2 rows @ tau=0.80): overall 2/2", report) + + def test_degenerate_set_still_reports_coverage(self) -> None: + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "empty2", []), + ] + pipe = CountingPipeline( + {"alpha": [("616-305", 5.0)]} + ) # the other two are empty + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1) + self.assertIsNone(scaler) + self.assertIn("reached calibration: 1/3", report) + self.assertIn("2 of 3 eligible rows did not reach the fit", report) + self.assertIn("FAILED (gate did not run)", report) + + def test_decision_report_names_missing_audits_when_nothing_graded(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row("d2", "positive", "alpha2", ["616-305"]), + ] + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, {}, TemperatureScaler(1.0), 0.80 + ) + report = out.getvalue() + self.assertEqual(status, 0) + self.assertIn("2 of 2 rows with an expected decision had no audit", report) + self.assertIn( + "none of the 2 rows with an expected decision could be graded", report + ) + self.assertNotIn("no rows with an expected decision", report) + + def test_selection_without_expected_decisions_keeps_the_original_line(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [_golden_row("e1", "explicit", "delta", ["616-305"])] + rows[0] = rows[0].model_copy( + update={"expected": rows[0].expected.model_copy(update={"decision": None})} + ) + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, {}, TemperatureScaler(1.0), 0.80 + ) + self.assertEqual(status, 0) + self.assertIn( + "no rows with an expected decision in this selection", out.getvalue() + ) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 34b945ebc..159b845f9 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -327,6 +327,14 @@ def calibration_set( return logit_sets, labels +def calibration_eligible(rows: List[GoldenDatasetRow]) -> int: + """Rows the C.3 fit is *supposed* to draw on: the positive + hard_negative + slices of the selection. ``calibration_set`` may return fewer (no audit, or an + empty reranked shortlist); the gate counts against this number so a drop is + never silent (#1166, same class as the explicit-gate fix in #1140).""" + return sum(1 for r in rows if r.slice.value in ("positive", "hard_negative")) + + def report_calibration( rows: List[GoldenDatasetRow], audits: Dict[str, Any], @@ -352,6 +360,16 @@ def report_calibration( ) logit_sets, labels = calibration_set(rows, audits) + # Coverage first, so the degenerate path below also says how many of the + # eligible rows reached the fit instead of only "need both outcomes". + eligible = calibration_eligible(rows) + used = len(labels) + print(f"calibration (C.3): reached calibration: {used}/{eligible}") + if used < eligible: + print( + f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not " + "reach the fit (no audit or empty reranked shortlist)" + ) if len(set(labels)) < 2: # Degenerate calibration set: single-class labels, or nothing left after @@ -371,12 +389,20 @@ def report_calibration( conf_cal = [scaler.confidence(s) for s in logit_sets] ece_raw = expected_calibration_error(conf_raw, labels) ece_cal = expected_calibration_error(conf_cal, labels) - gate_ok = ece_cal < 0.10 + # The gate counts against every eligible row in the selection. Rows that + # never reached the fit (no audit, empty shortlist) shrink the denominator + # silently otherwise, and a hollowed-out set can still print PASS (#1166). + coverage_ok = used == eligible + ece_ok = ece_cal < 0.10 + gate_ok = ece_ok and coverage_ok print( - f"calibration (C.3, {len(labels)} rows): fitted T={scaler.temperature:.3f}; " - f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated); " - f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}" + f"calibration (C.3, {used} rows): fitted T={scaler.temperature:.3f}; " + f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated, in-sample); " + f"gate ECE<0.10: {'PASS' if ece_ok else 'FAIL'}; " + f"reached calibration: {used}/{eligible}" ) + if not coverage_ok: + print("calibration (C.3): FAILED (gate denominator incomplete)") return (0 if gate_ok else 1), scaler @@ -406,9 +432,23 @@ def report_decision_accuracy( from application.utils.librarian.decision_engine import decide from application.utils.librarian.schemas import Decision - graded = [r for r in rows if r.expected.decision is not None and r.id in audits] + selected = [r for r in rows if r.expected.decision is not None] + graded = [r for r in selected if r.id in audits] + if len(graded) < len(selected): + # Informational report, but a silent drop still hides a broken pipeline; + # say so even when nothing at all could be graded. + print( + f"decision (C.4): {len(selected) - len(graded)} of {len(selected)} rows " + "with an expected decision had no audit and were not graded" + ) if not graded: - print("decision (C.4): no rows with an expected decision in this selection") + if selected: + print( + f"decision (C.4): none of the {len(selected)} rows with an expected " + "decision could be graded (no audits)" + ) + else: + print("decision (C.4): no rows with an expected decision in this selection") return 0 dec_match = reason_match = 0