From 46c2efd4c166a5f9706910dbdbde19c79141dd8d Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Wed, 21 Oct 2026 19:10:16 +0530 Subject: [PATCH 1/3] librarian: count rows that never reached the C.3 fit against the ECE gate (#1166) calibration_set() skipped rows with no audit or an empty reranked shortlist, and report_calibration() printed the survivors as if they were the selection, so a hollowed-out calibration set could print PASS and exit 0. Same class as the explicit-gate fix in #1140. - calibration_eligible(): positive + hard_negative rows in the selection - report_calibration(): prints "reached calibration: used/eligible", labels the ECE as in-sample, fails the gate when any eligible row was dropped and says how many; T is still returned so C.4 can report - report_decision_accuracy(): prints how many expected-decision rows had no audit instead of silently grading fewer - tests for all three --- .../tests/librarian/evaluate_harness_test.py | 82 +++++++++++++++++++ scripts/evaluate_librarian.py | 39 +++++++-- 2 files changed, 116 insertions(+), 5 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index ae979a177..ea5f5185a 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -342,3 +342,85 @@ def test_total_boundary_rejection_fails_the_run(self) -> None: if __name__ == "__main__": unittest.main() + + +class CalibrationCoverageTest(unittest.TestCase): + """#1166: rows that never reach the C.3 fit must count against the gate.""" + + def _rows(self) -> list: + return [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "alpha2", ["616-305"]), + _golden_row("p3", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "beta", []), + _golden_row("n2", "hard_negative", "beta2", []), + _golden_row("e1", "explicit", "delta", ["616-305"]), + ] + + def _pipe(self) -> "CountingPipeline": + return CountingPipeline( + { + "alpha": [("616-305", 5.0), ("999-999", 0.1)], + "alpha2": [("616-305", 4.0), ("999-999", 0.2)], + "beta": [("111-111", 3.0), ("222-222", 2.9)], + "beta2": [("111-111", 2.0), ("222-222", 1.9)], + "delta": [("616-305", 1.0)], + # "empty" yields no candidates on purpose + } + ) + + def test_eligible_counts_only_the_two_calibration_slices(self) -> None: + self.assertEqual(harness.calibration_eligible(self._rows()), 5) + + def test_partial_drop_fails_the_gate_and_says_how_many(self) -> None: + rows, pipe = self._rows(), self._pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1) + self.assertIsNotNone(scaler, "T is still fitted so C.4 can report") + self.assertIn("reached calibration: 4/5", report) + self.assertIn("1 of 5 eligible rows did not reach the fit", report) + self.assertIn("FAILED (gate denominator incomplete)", report) + + def test_full_coverage_gates_on_ece_only(self) -> None: + rows = [r for r in self._rows() if r.id != "p3"] + pipe = self._pipe() + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertIsNotNone(scaler) + self.assertIn("reached calibration: 4/4", report) + self.assertNotIn("denominator incomplete", report) + self.assertIn("(calibrated, in-sample)", report) + self.assertEqual(status, 0 if "gate ECE<0.10: PASS" in report else 1) + + def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row( + "d2", "hard_negative", "beta", [], reason_code="BELOW_THRESHOLD" + ), + _golden_row("d3", "positive", "never-audited", ["616-305"]), + ] + pipe = CountingPipeline( + { + "alpha": [("616-305", 20.0), ("999-999", 0.0)], + "beta": [("111-111", 1.0), ("222-222", 0.99)], + } + ) + audits = harness.live_audits(rows[:2], pipe, pipe) + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, audits, TemperatureScaler(1.0), 0.80 + ) + report = out.getvalue() + self.assertEqual(status, 0, "still informational") + self.assertIn("1 of 3 rows with an expected decision had no audit", report) + self.assertIn("decision (C.4, 2 rows @ tau=0.80): overall 2/2", report) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 34b945ebc..18d785567 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -327,6 +327,14 @@ def calibration_set( return logit_sets, labels +def calibration_eligible(rows: List[GoldenDatasetRow]) -> int: + """Rows the C.3 fit is *supposed* to draw on: the positive + hard_negative + slices of the selection. ``calibration_set`` may return fewer (no audit, or an + empty reranked shortlist); the gate counts against this number so a drop is + never silent (#1166, same class as the explicit-gate fix in #1140).""" + return sum(1 for r in rows if r.slice.value in ("positive", "hard_negative")) + + def report_calibration( rows: List[GoldenDatasetRow], audits: Dict[str, Any], @@ -371,12 +379,26 @@ def report_calibration( conf_cal = [scaler.confidence(s) for s in logit_sets] ece_raw = expected_calibration_error(conf_raw, labels) ece_cal = expected_calibration_error(conf_cal, labels) - gate_ok = ece_cal < 0.10 + # The gate counts against every eligible row in the selection. Rows that + # never reached the fit (no audit, empty shortlist) shrink the denominator + # silently otherwise, and a hollowed-out set can still print PASS (#1166). + eligible = calibration_eligible(rows) + used = len(labels) + coverage_ok = used == eligible + ece_ok = ece_cal < 0.10 + gate_ok = ece_ok and coverage_ok print( - f"calibration (C.3, {len(labels)} rows): fitted T={scaler.temperature:.3f}; " - f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated); " - f"gate ECE<0.10: {'PASS' if gate_ok else 'FAIL'}" + f"calibration (C.3, {used} rows): fitted T={scaler.temperature:.3f}; " + f"ECE {ece_raw:.3f} (raw, T=1) -> {ece_cal:.3f} (calibrated, in-sample); " + f"gate ECE<0.10: {'PASS' if ece_ok else 'FAIL'}; " + f"reached calibration: {used}/{eligible}" ) + if not coverage_ok: + print( + f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not " + "reach the fit (no audit or empty reranked shortlist); FAILED " + "(gate denominator incomplete)" + ) return (0 if gate_ok else 1), scaler @@ -406,10 +428,17 @@ def report_decision_accuracy( from application.utils.librarian.decision_engine import decide from application.utils.librarian.schemas import Decision - graded = [r for r in rows if r.expected.decision is not None and r.id in audits] + selected = [r for r in rows if r.expected.decision is not None] + graded = [r for r in selected if r.id in audits] if not graded: print("decision (C.4): no rows with an expected decision in this selection") return 0 + if len(graded) < len(selected): + # Informational report, but a silent drop still hides a broken pipeline. + print( + f"decision (C.4): {len(selected) - len(graded)} of {len(selected)} rows " + "with an expected decision had no audit and were not graded" + ) dec_match = reason_match = 0 link_total = link_correct = review_total = review_correct = 0 From 87353a07a0d00cd3d6fcf967ac1a445a814002af Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Wed, 21 Oct 2026 19:16:54 +0530 Subject: [PATCH 2/3] librarian: report C.3/C.4 coverage before the early returns; assert the ECE gate independently Review follow-ups on #1167: - report_calibration(): print "reached calibration: used/eligible" and the dropped-row count before the degenerate-set return, so a selection where most rows never reached the fit says so instead of only "need both outcomes". - report_decision_accuracy(): print the missing-audit count before the nothing-graded return, so "no rows with an expected decision" is never the only line when every selected row simply lacked an audit. - test: the full-coverage case now computes the expected gate outcome from the fixture logits and a refitted T instead of reading it back from the report. - tests for both early-return paths. --- .../tests/librarian/evaluate_harness_test.py | 52 ++++++++++++++++++- scripts/evaluate_librarian.py | 27 ++++++---- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index ea5f5185a..9c9c645a8 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -395,7 +395,20 @@ def test_full_coverage_gates_on_ece_only(self) -> None: self.assertIn("reached calibration: 4/4", report) self.assertNotIn("denominator incomplete", report) self.assertIn("(calibrated, in-sample)", report) - self.assertEqual(status, 0 if "gate ECE<0.10: PASS" in report else 1) + # Expected outcome computed independently of the report text: refit T on + # the same calibration set and apply the 0.10 bar ourselves. + from application.utils.librarian.calibration.temperature import ( + expected_calibration_error, + fit_temperature, + ) + + logit_sets, labels = harness.calibration_set(rows, audits) + scaler_check = fit_temperature(logit_sets, labels) + ece_check = expected_calibration_error( + [scaler_check.confidence(s) for s in logit_sets], labels + ) + self.assertAlmostEqual(scaler.temperature, scaler_check.temperature, places=6) + self.assertEqual(status, 0 if ece_check < 0.10 else 1) def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: from application.utils.librarian.calibration.temperature import ( @@ -424,3 +437,40 @@ def test_missing_audit_rows_are_counted_in_the_decision_report(self) -> None: self.assertEqual(status, 0, "still informational") self.assertIn("1 of 3 rows with an expected decision had no audit", report) self.assertIn("decision (C.4, 2 rows @ tau=0.80): overall 2/2", report) + + def test_degenerate_set_still_reports_coverage(self) -> None: + rows = [ + _golden_row("p1", "positive", "alpha", ["616-305"]), + _golden_row("p2", "positive", "empty", ["616-305"]), + _golden_row("n1", "hard_negative", "empty2", []), + ] + pipe = CountingPipeline( + {"alpha": [("616-305", 5.0)]} + ) # the other two are empty + audits = harness.live_audits(rows, pipe, pipe) + with _captured_stdout() as out: + status, scaler = harness.report_calibration(rows, audits) + report = out.getvalue() + self.assertEqual(status, 1) + self.assertIsNone(scaler) + self.assertIn("reached calibration: 1/3", report) + self.assertIn("2 of 3 eligible rows did not reach the fit", report) + self.assertIn("FAILED (gate did not run)", report) + + def test_decision_report_names_missing_audits_when_nothing_graded(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [ + _golden_row("d1", "positive", "alpha", ["616-305"]), + _golden_row("d2", "positive", "alpha2", ["616-305"]), + ] + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, {}, TemperatureScaler(1.0), 0.80 + ) + report = out.getvalue() + self.assertEqual(status, 0) + self.assertIn("2 of 2 rows with an expected decision had no audit", report) + self.assertIn("no rows with an expected decision", report) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 18d785567..ce5009f83 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -360,6 +360,16 @@ def report_calibration( ) logit_sets, labels = calibration_set(rows, audits) + # Coverage first, so the degenerate path below also says how many of the + # eligible rows reached the fit instead of only "need both outcomes". + eligible = calibration_eligible(rows) + used = len(labels) + print(f"calibration (C.3): reached calibration: {used}/{eligible}") + if used < eligible: + print( + f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not " + "reach the fit (no audit or empty reranked shortlist)" + ) if len(set(labels)) < 2: # Degenerate calibration set: single-class labels, or nothing left after @@ -382,8 +392,6 @@ def report_calibration( # The gate counts against every eligible row in the selection. Rows that # never reached the fit (no audit, empty shortlist) shrink the denominator # silently otherwise, and a hollowed-out set can still print PASS (#1166). - eligible = calibration_eligible(rows) - used = len(labels) coverage_ok = used == eligible ece_ok = ece_cal < 0.10 gate_ok = ece_ok and coverage_ok @@ -394,11 +402,7 @@ def report_calibration( f"reached calibration: {used}/{eligible}" ) if not coverage_ok: - print( - f"calibration (C.3): {eligible - used} of {eligible} eligible rows did not " - "reach the fit (no audit or empty reranked shortlist); FAILED " - "(gate denominator incomplete)" - ) + print("calibration (C.3): FAILED (gate denominator incomplete)") return (0 if gate_ok else 1), scaler @@ -430,15 +434,16 @@ def report_decision_accuracy( selected = [r for r in rows if r.expected.decision is not None] graded = [r for r in selected if r.id in audits] - if not graded: - print("decision (C.4): no rows with an expected decision in this selection") - return 0 if len(graded) < len(selected): - # Informational report, but a silent drop still hides a broken pipeline. + # Informational report, but a silent drop still hides a broken pipeline; + # say so even when nothing at all could be graded. print( f"decision (C.4): {len(selected) - len(graded)} of {len(selected)} rows " "with an expected decision had no audit and were not graded" ) + if not graded: + print("decision (C.4): no rows with an expected decision in this selection") + return 0 dec_match = reason_match = 0 link_total = link_correct = review_total = review_correct = 0 From 68080f894c7b119a63d236e1c72ede206a5360bf Mon Sep 17 00:00:00 2001 From: PRAteek-singHWY Date: Wed, 21 Oct 2026 19:25:54 +0530 Subject: [PATCH 3/3] librarian: say that selected rows had no audit instead of 'no rows with an expected decision' when nothing could be graded (#1167 review) --- .../tests/librarian/evaluate_harness_test.py | 30 +++++++++++++++++-- scripts/evaluate_librarian.py | 8 ++++- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index 9c9c645a8..bdaa2b92b 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -279,7 +279,12 @@ def test_no_graded_rows_is_not_an_error(self) -> None: ) self.assertEqual(status, 0) - self.assertIn("no rows with an expected decision", out.getvalue()) + report = out.getvalue() + self.assertIn("1 of 1 rows with an expected decision had no audit", report) + self.assertIn( + "none of the 1 rows with an expected decision could be graded", report + ) + self.assertNotIn("no rows with an expected decision", report) class BoundaryGateTest(unittest.TestCase): @@ -473,4 +478,25 @@ def test_decision_report_names_missing_audits_when_nothing_graded(self) -> None: report = out.getvalue() self.assertEqual(status, 0) self.assertIn("2 of 2 rows with an expected decision had no audit", report) - self.assertIn("no rows with an expected decision", report) + self.assertIn( + "none of the 2 rows with an expected decision could be graded", report + ) + self.assertNotIn("no rows with an expected decision", report) + + def test_selection_without_expected_decisions_keeps_the_original_line(self) -> None: + from application.utils.librarian.calibration.temperature import ( + TemperatureScaler, + ) + + rows = [_golden_row("e1", "explicit", "delta", ["616-305"])] + rows[0] = rows[0].model_copy( + update={"expected": rows[0].expected.model_copy(update={"decision": None})} + ) + with _captured_stdout() as out: + status = harness.report_decision_accuracy( + rows, {}, TemperatureScaler(1.0), 0.80 + ) + self.assertEqual(status, 0) + self.assertIn( + "no rows with an expected decision in this selection", out.getvalue() + ) diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index ce5009f83..159b845f9 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -442,7 +442,13 @@ def report_decision_accuracy( "with an expected decision had no audit and were not graded" ) if not graded: - print("decision (C.4): no rows with an expected decision in this selection") + if selected: + print( + f"decision (C.4): none of the {len(selected)} rows with an expected " + "decision could be graded (no audits)" + ) + else: + print("decision (C.4): no rows with an expected decision in this selection") return 0 dec_match = reason_match = 0