feat(evaluations)!: remove default pass rate threshold

packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt

@@ -56,12 +56,12 @@ GenerationOverrides(TypedDict, total=False)  [class]
   provider: str
 init_evaluations(project_key: str | None = None, api_key: str | None = None, sdk_key: str | None = None, api_base_uri: str | None = None, ui_base_uri: str | None = None, transport: Transport = urllib_transport) -> EvaluationsModule
 Judge  [class, @dataclass(frozen=True)]
-  __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD, ground_truth_context: str | None = None)
+  __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = None, ground_truth_context: str | None = None)
   __post_init__() -> None
   criterion_type: str  [property]
   ground_truth_context: str | None = None
   key: str
-  pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD
+  pass_rate_threshold: float | None = None
   threshold: float | None = DEFAULT_JUDGE_THRESHOLD
   to_criteria_wire() -> dict[str, Any]
 RunSummary  [class, @dataclass]
@@ -73,12 +73,12 @@ RunSummary  [class, @dataclass]
   pending_rows: int = 0
   total_rows: int = 0
 Scorer  [class, @dataclass(frozen=True)]
-  __init__(name: str, fn: ScorerFn, threshold: float | None = 1.0, pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD, success_direction: SuccessDirection = 'higher_is_better')
+  __init__(name: str, fn: ScorerFn, threshold: float | None = 1.0, pass_rate_threshold: float | None = None, success_direction: SuccessDirection = 'higher_is_better')
   __post_init__() -> None
   criterion_type: str  [property]
   fn: ScorerFn
   name: str
-  pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD
+  pass_rate_threshold: float | None = None
   success_direction: SuccessDirection = 'higher_is_better'
   threshold: float | None = 1.0
   to_criteria_wire() -> dict[str, Any]

packages/client/src/launchdarkly_ai_server/evaluations/criteria.py

@@ -1,3 +1,3 @@
 from __future__ import annotations
 
 from collections.abc import Awaitable, Callable
@@ -16,7 +16,6 @@
 #: it would be recorded and never ruled on; defaulting is what keeps a judge
 #: usable as a gate without every caller restating the obvious.
 DEFAULT_JUDGE_THRESHOLD = 0.5
-DEFAULT_PASS_RATE_THRESHOLD = 0.95
 
 
 @dataclass(frozen=True)
@@ -35,7 +34,7 @@
 
     key: str
     threshold: float | None = DEFAULT_JUDGE_THRESHOLD
-    pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD
+    pass_rate_threshold: float | None = None
     ground_truth_context: str | None = None
 
     def __post_init__(self) -> None:
@@ -89,7 +88,7 @@
     name: str
     fn: ScorerFn
     threshold: float | None = 1.0
-    pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD
+    pass_rate_threshold: float | None = None
     success_direction: SuccessDirection = "higher_is_better"
 
     def __post_init__(self) -> None:
@@ -149,11 +148,8 @@
     options: dict[str, Any] = {}
     if threshold is not None:
         options["threshold"] = threshold
-    options["passRateThreshold"] = (
-        DEFAULT_PASS_RATE_THRESHOLD
-        if pass_rate_threshold is None
-        else pass_rate_threshold
-    )
+    if pass_rate_threshold is not None:
+        options["passRateThreshold"] = pass_rate_threshold
     if ground_truth_context is not None:
         options["groundTruthContext"] = ground_truth_context
     return options

packages/client/tests/test_evaluations_run.py

@@ -1784,7 +1784,7 @@ async def handler(
             "criterionType": "$ld:ai:judge:accuracy",
             "kind": "judge",
             "judgeKey": "$ld:ai:judge:accuracy",
-            "options": {"threshold": 0.5, "passRateThreshold": 0.95},
+            "options": {"threshold": 0.5},
         }
     ]
     assert transport.requests[3]["body"] == {"source": "api", "datasetId": "dataset-id"}
@@ -2026,7 +2026,7 @@ def count_violations(row: DatasetRow, output: Any) -> float:
         "criterionType": "policy-violations",
         "kind": "scorer",
         "successDirection": "lower_is_better",
-        "options": {"threshold": 0.0, "passRateThreshold": 0.95},
+        "options": {"threshold": 0.0},
     }
 
 
@@ -2037,23 +2037,20 @@ def test_judge_threshold_defaults_so_a_criterion_is_always_rulable() -> None:
         "criterionType": "$ld:ai:judge:accuracy",
         "kind": "judge",
         "judgeKey": "$ld:ai:judge:accuracy",
-        "options": {"threshold": 0.5, "passRateThreshold": 0.95},
+        "options": {"threshold": 0.5},
     }
 
 
-def test_explicit_none_pass_rate_threshold_falls_back_to_default() -> None:
-    """An explicit None must not drop passRateThreshold from the wire, or the
-    run would have no pass rate to gate on."""
-    judge = Judge(key="$ld:ai:judge:accuracy", pass_rate_threshold=None)
-    scorer = Scorer(name="exact", fn=lambda row, output: True, pass_rate_threshold=None)
+def test_pass_rate_threshold_is_only_sent_when_set() -> None:
+    """Pass rate has no client default yet, since LaunchDarkly does not act on
+    it; an unset one stays off the wire rather than claiming a gate."""
+    judge = Judge(key="$ld:ai:judge:accuracy")
+    scorer = Scorer(name="exact", fn=lambda row, output: True, pass_rate_threshold=0.8)
 
-    assert judge.to_criteria_wire()["options"] == {
-        "threshold": 0.5,
-        "passRateThreshold": 0.95,
-    }
+    assert judge.to_criteria_wire()["options"] == {"threshold": 0.5}
     assert scorer.to_criteria_wire()["options"] == {
         "threshold": 1.0,
-        "passRateThreshold": 0.95,
+        "passRateThreshold": 0.8,
     }
 
 
@@ -2158,7 +2155,7 @@ def check_refund(row: DatasetRow, output: Any) -> bool:
             "criterionType": "refund-exists",
             "kind": "scorer",
             "successDirection": "higher_is_better",
-            "options": {"threshold": 1.0, "passRateThreshold": 0.95},
+            "options": {"threshold": 1.0},
         }
     ]
     assert transport.requests[3]["body"] == {"source": "api", "datasetId": "dataset-id"}