From 9bd5bb827f8b343eead31c2e291548d6332e1ee3 Mon Sep 17 00:00:00 2001 From: MCERQUA <55824402+MCERQUA@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:38:11 -0400 Subject: [PATCH] Let a fitted temperature and fitted cutoffs reach `.confidence` Two places where calibration and the runtime disagreed about what `.confidence` is: - `calibrated()` reshaped a choice's or a score's probabilities but left its `confidence` as the judge first reported it, and the `confidence` node reads exactly that value. So for choice questions a fitted temperature could never change what a `.confidence` rule decides, while docs/rules.md presents calibration as what makes such a threshold meaningful. It now works the confidence out again from the tempered probabilities: for a score with `score_confidence`, for a choice with the `choice_confidence` formula applied to the option it chose (the same number whenever a judge chose its likeliest option; lower when it chose another, down to 0 once the chosen option falls below 1/K). At temperature 1 nothing changes. - `observed()` collected the chosen option's probability (`sureness`) as the values a cutoff on a confidence node could be drawn between, while the runtime compares the reported confidence. On two options p = 0.68 reads 0.36, so for choice questions every candidate line was in the wrong number space. It now collects the reported confidence. Regression tests: a property test that, after tempering, `confidence` is recomputed from the tempered probabilities; an end-to-end `threat.confidence >= 0.7` rule that `calibrated 5` now flattens; and a 41-example cutoff fit where the old code moved the line 0.95 -> 0.555 (33/41 right) and the fixed code finds 0.39 (41/41). Co-Authored-By: Claude Opus 5.5 --- docs/rules.md | 3 +- src/rulesmith/calibrate.py | 7 ++- src/rulesmith/runtime.py | 14 ++++- tests/test_calibrate.py | 122 +++++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 5 deletions(-) diff --git a/docs/rules.md b/docs/rules.md index ac98072..6246eef 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -114,7 +114,8 @@ small model that says 0.97 and is right three times in five is overconfident. `rulesmith calibrate task.json plan.rules --output calibrated.rules` fits one temperature per choice question over the task's labels, from its answers on the training examples, and writes it into the question (`... [ATTACK: "..."] -calibrated 1.8`). The answers stay the same; how sure they read changes. +calibrated 1.8`). The answers stay the same; how sure they read changes, +`threat.confidence` included. The line a rule draws on a judgment, the 0.7 in `threat.confidence >= 0.7`, is a number the proposer wrote, and search only moves it by rewriting the rule and diff --git a/src/rulesmith/calibrate.py b/src/rulesmith/calibrate.py index 4f23f8c..d6526f9 100644 --- a/src/rulesmith/calibrate.py +++ b/src/rulesmith/calibrate.py @@ -36,7 +36,6 @@ Models, Prices, calibrated, - sureness, ) # Temperatures tried, evenly spaced in log from a fifth to five: calibration corrects a judge's @@ -263,8 +262,10 @@ def observed( and isinstance(node.question, (Noul, Score)) ): seen.setdefault(name, []).append(answers[name]["value"]) - if isinstance(node, Confidence) and node.source in answers: - seen.setdefault(name, []).append(sureness(answers[node.source])) + # The confidence the judge reported, which is what a rule compares, not the + # probability of its answer: on two options, 0.68 reads 0.36. + if isinstance(node, Confidence) and "confidence" in answers.get(node.source, {}): + seen.setdefault(name, []).append(answers[node.source]["confidence"]) return seen diff --git a/src/rulesmith/runtime.py b/src/rulesmith/runtime.py index 06d0862..a556254 100644 --- a/src/rulesmith/runtime.py +++ b/src/rulesmith/runtime.py @@ -23,6 +23,7 @@ Usage, ) +from rulesmith.chat_judge import score_confidence from rulesmith.graph import ( All, Call, @@ -112,7 +113,9 @@ def read_answer(question: Question, answer: NoulAnswer | ChoiceAnswer | ScoreAns def calibrated(answer: dict, temperature: float) -> dict: """An answer with its log-odds divided by `temperature`. The choice itself never changes, - only how sure it reads; a score is the expected level of its tempered probabilities.""" + only how sure it reads; a score is the expected level of its tempered probabilities, and the + confidence a rule tests is worked out again from them by the formula the judge reports it + with.""" if temperature == 1: return answer if answer["type"] == "noul": @@ -129,6 +132,15 @@ def calibrated(answer: dict, temperature: float) -> dict: tempered = answer | {"probabilities": probabilities} if answer["type"] == "score": tempered["value"] = sum(int(level) * p for level, p in probabilities.items()) + levels = sorted(probabilities, key=int) + tempered["confidence"] = score_confidence([probabilities[level] for level in levels]) + else: + # How far the chosen option stands above even odds, as `choice_confidence` reports it + # for the likeliest option: the same number whenever a judge chose its likeliest option, + # and less, down to none, for an answer its own probabilities argue against. + uniform = 1 / len(probabilities) + chosen = probabilities[answer["value"]] + tempered["confidence"] = max(0.0, (chosen - uniform) / (1 - uniform)) return tempered diff --git a/tests/test_calibrate.py b/tests/test_calibrate.py index d5a2f3f..cb5db7b 100644 --- a/tests/test_calibrate.py +++ b/tests/test_calibrate.py @@ -11,6 +11,7 @@ from test_graph import task_data from rulesmith.calibrate import cuts, fit_cutoffs, fit_temperature, measure +from rulesmith.chat_judge import choice_confidence, score_confidence from rulesmith.graph import Plan, Task from rulesmith.optimize import Labels from rulesmith.rules import compile_rules, render_rules @@ -290,6 +291,127 @@ def answer(request): assert 0.58 < fitted.nodes["settled"].cutoff < 0.6 and report[0]["tests"] == "sure" +@given(st.floats(0.2, 5)) +def test_a_temperature_reshapes_how_sure_a_choice_or_a_score_reads(temperature): + # A rule tests `.confidence`, so a fitted temperature has to reach it, by the formula the + # judge reported it with, or calibrating a choice could never change what a rule decides. + choice = calibrated(CHOICE, temperature) + score = calibrated( + { + "type": "score", + "value": 1.737, + "probabilities": {"0": 0.066, "1": 0.131, "2": 0.803}, + "confidence": 0.6, + }, + temperature, + ) + if abs(temperature - 1) < 0.001: + return + assert choice["confidence"] == pytest.approx( + choice_confidence(list(choice["probabilities"].values())) + ) + levels = sorted(score["probabilities"], key=int) + assert score["confidence"] == pytest.approx( + score_confidence([score["probabilities"][level] for level in levels]) + ) + # Above one an overconfident judge reads less sure; below one a timid one reads surer. + untempered = choice_confidence(list(CHOICE["probabilities"].values())) + assert (choice["confidence"] < untempered) == (temperature > 1) + # An answer its own probabilities argue against is not made sure by sharpening them. + assert calibrated(CHOICE | {"value": "c"}, temperature)["confidence"] == 0 + + +def test_a_fitted_temperature_reaches_the_confidence_a_rule_tests(provider, client): + provider.answers = { + "threat": { + "type": "choice", + "choice": "ATTACK", + "confidence": choice_confidence([0.9, 0.1]), + "probabilities": {"ATTACK": 0.9, "MOVE_BACKWARD": 0.1}, + } + } + rules = ( + 'ask threat = "Fight or fall back?" [ATTACK: "Ammo and health allow it."]' + ' [MOVE_BACKWARD: "They do not."]{}\n' + 'when threat.confidence >= 0.7 and threat == "ATTACK": ATTACK\n' + "always: MOVE_FORWARD" + ) + sure = DecisionProgram(compile_rules(rules.format("")), client, "fixture")(state="contact") + hedged = DecisionProgram(compile_rules(rules.format(" calibrated 5")), client, "fixture")( + state="contact" + ) + assert sure.label == "ATTACK" and sure.answers["threat"]["confidence"] == pytest.approx(0.8) + # Flattened by five, 0.9 against 0.1 is barely a lean, so the rule no longer acts on it. + assert hedged.answers["threat"]["confidence"] < 0.7 + assert hedged.label == "MOVE_FORWARD" + + +def confidence_threshold(cutoff: float) -> Plan: + """A plan accepting whatever its judge is at least `cutoff` sure of, from the confidence it + reports, and sending the rest to review.""" + return Plan.model_validate( + { + "nodes": { + "route": { + "type": "call", + "inputs": {"text": {"type": "state"}}, + "question": { + "type": "choice", + "instructions": "Accept or review?", + "criteria": {"accept": "Fine.", "review": "Check."}, + }, + }, + "sure": {"type": "confidence", "source": "route"}, + "settled": { + "type": "threshold", + "source": "sure", + "cutoff": cutoff, + "above": "accept", + "below": "review", + }, + }, + "outputs": {"label": {"type": "node", "node": "settled"}}, + } + ) + + +def test_a_confidence_cutoff_is_fitted_on_the_confidence_the_rule_compares(provider, client): + # The judge reports confidence the way TypeSafe and the chat judge do, how far its leading + # option stands above even odds, not the leading option's probability, so on two options + # p = 0.68 reads 0.36 and p = 0.71 reads 0.42. + def answer(request): + p = stated(request) + provider.body["answers"] = { + "route": { + "type": "choice", + "choice": "accept", + "confidence": choice_confidence([p, 1 - p]), + "probabilities": {"accept": p, "review": 1 - p}, + } + } + + provider.on_request = answer + review = [0.55, 0.56, 0.57, 0.575, 0.58, 0.585, 0.59, 0.595, 0.6, 0.605] + review += [0.61, 0.615, 0.62, 0.625, 0.63, 0.64, 0.65, 0.66, 0.67, 0.68] + accept = [0.71, 0.72, 0.73, 0.74, 0.75, 0.76, 0.765, 0.77, 0.78, 0.8, 0.82] + accept += [0.84, 0.85, 0.86, 0.88, 0.9, 0.91, 0.92, 0.93, 0.94, 0.95] + data = task_data() | { + "train": [ + *[{"id": f"a{i}", "state": str(p), "label": "accept"} for i, p in enumerate(accept)], + *[{"id": f"r{i}", "state": str(p), "label": "review"} for i, p in enumerate(review)], + ] + } + task = Task.model_validate(data) + fitted, report = fit_cutoffs(confidence_threshold(0.95), task, client, "fixture") + # Every reviewed request reads at most 0.36 and every accepted one at least 0.42, so the line + # belongs between them and gets all 41 right. Lines drawn between the leading probabilities + # instead start at 0.555, which sends the eight accepted requests reading 0.42 to 0.54 to + # review: 33 of 41. + assert 0.36 < fitted.nodes["settled"].cutoff < 0.42 + assert report[0]["train_before"] == pytest.approx(20 / 41) + assert report[0]["train_after"] == 1 + + def test_the_cli_writes_fitted_cutoffs_and_reports_the_validation_change(tmp_path, provider): noul_by_state(provider) task = scam_task(