diff --git a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt index 3ecd56ce..ceb113f4 100644 --- a/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt +++ b/packages/client/api-report/launchdarkly_ai_server.experimental.evaluations.txt @@ -56,12 +56,12 @@ GenerationOverrides(TypedDict, total=False) [class] provider: str init_evaluations(project_key: str | None = None, api_key: str | None = None, sdk_key: str | None = None, api_base_uri: str | None = None, ui_base_uri: str | None = None, transport: Transport = urllib_transport) -> EvaluationsModule Judge [class, @dataclass(frozen=True)] - __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD, ground_truth_context: str | None = None) + __init__(key: str, threshold: float | None = DEFAULT_JUDGE_THRESHOLD, pass_rate_threshold: float | None = None, ground_truth_context: str | None = None) __post_init__() -> None criterion_type: str [property] ground_truth_context: str | None = None key: str - pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD + pass_rate_threshold: float | None = None threshold: float | None = DEFAULT_JUDGE_THRESHOLD to_criteria_wire() -> dict[str, Any] RunSummary [class, @dataclass] @@ -73,12 +73,12 @@ RunSummary [class, @dataclass] pending_rows: int = 0 total_rows: int = 0 Scorer [class, @dataclass(frozen=True)] - __init__(name: str, fn: ScorerFn, threshold: float | None = 1.0, pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD, success_direction: SuccessDirection = 'higher_is_better') + __init__(name: str, fn: ScorerFn, threshold: float | None = 1.0, pass_rate_threshold: float | None = None, success_direction: SuccessDirection = 'higher_is_better') __post_init__() -> None criterion_type: str [property] fn: ScorerFn name: str - pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD + pass_rate_threshold: float | None = None success_direction: SuccessDirection = 'higher_is_better' threshold: float | None = 1.0 to_criteria_wire() -> dict[str, Any] diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py index e27365c6..ee0962fd 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/criteria.py @@ -16,7 +16,6 @@ #: it would be recorded and never ruled on; defaulting is what keeps a judge #: usable as a gate without every caller restating the obvious. DEFAULT_JUDGE_THRESHOLD = 0.5 -DEFAULT_PASS_RATE_THRESHOLD = 0.95 @dataclass(frozen=True) @@ -35,7 +34,7 @@ class Judge: key: str threshold: float | None = DEFAULT_JUDGE_THRESHOLD - pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD + pass_rate_threshold: float | None = None ground_truth_context: str | None = None def __post_init__(self) -> None: @@ -89,7 +88,7 @@ class Scorer: name: str fn: ScorerFn threshold: float | None = 1.0 - pass_rate_threshold: float | None = DEFAULT_PASS_RATE_THRESHOLD + pass_rate_threshold: float | None = None success_direction: SuccessDirection = "higher_is_better" def __post_init__(self) -> None: @@ -149,11 +148,8 @@ def _criteria_options( options: dict[str, Any] = {} if threshold is not None: options["threshold"] = threshold - options["passRateThreshold"] = ( - DEFAULT_PASS_RATE_THRESHOLD - if pass_rate_threshold is None - else pass_rate_threshold - ) + if pass_rate_threshold is not None: + options["passRateThreshold"] = pass_rate_threshold if ground_truth_context is not None: options["groundTruthContext"] = ground_truth_context return options diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 87390977..1b3d3cdb 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1784,7 +1784,7 @@ async def handler( "criterionType": "$ld:ai:judge:accuracy", "kind": "judge", "judgeKey": "$ld:ai:judge:accuracy", - "options": {"threshold": 0.5, "passRateThreshold": 0.95}, + "options": {"threshold": 0.5}, } ] assert transport.requests[3]["body"] == {"source": "api", "datasetId": "dataset-id"} @@ -2026,7 +2026,7 @@ def count_violations(row: DatasetRow, output: Any) -> float: "criterionType": "policy-violations", "kind": "scorer", "successDirection": "lower_is_better", - "options": {"threshold": 0.0, "passRateThreshold": 0.95}, + "options": {"threshold": 0.0}, } @@ -2037,23 +2037,20 @@ def test_judge_threshold_defaults_so_a_criterion_is_always_rulable() -> None: "criterionType": "$ld:ai:judge:accuracy", "kind": "judge", "judgeKey": "$ld:ai:judge:accuracy", - "options": {"threshold": 0.5, "passRateThreshold": 0.95}, + "options": {"threshold": 0.5}, } -def test_explicit_none_pass_rate_threshold_falls_back_to_default() -> None: - """An explicit None must not drop passRateThreshold from the wire, or the - run would have no pass rate to gate on.""" - judge = Judge(key="$ld:ai:judge:accuracy", pass_rate_threshold=None) - scorer = Scorer(name="exact", fn=lambda row, output: True, pass_rate_threshold=None) +def test_pass_rate_threshold_is_only_sent_when_set() -> None: + """Pass rate has no client default yet, since LaunchDarkly does not act on + it; an unset one stays off the wire rather than claiming a gate.""" + judge = Judge(key="$ld:ai:judge:accuracy") + scorer = Scorer(name="exact", fn=lambda row, output: True, pass_rate_threshold=0.8) - assert judge.to_criteria_wire()["options"] == { - "threshold": 0.5, - "passRateThreshold": 0.95, - } + assert judge.to_criteria_wire()["options"] == {"threshold": 0.5} assert scorer.to_criteria_wire()["options"] == { "threshold": 1.0, - "passRateThreshold": 0.95, + "passRateThreshold": 0.8, } @@ -2158,7 +2155,7 @@ def check_refund(row: DatasetRow, output: Any) -> bool: "criterionType": "refund-exists", "kind": "scorer", "successDirection": "higher_is_better", - "options": {"threshold": 1.0, "passRateThreshold": 0.95}, + "options": {"threshold": 1.0}, } ] assert transport.requests[3]["body"] == {"source": "api", "datasetId": "dataset-id"}