CXF-233: publish the Tier-1 E2E control-group baseline.json (6-run matrix) #18
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,182 @@ | ||
| { | ||
| "schema_version": 1, | ||
|
Contributor
There was a problem hiding this comment. 🟡 Suggestion: This file becomes a committed contract artifact that a future CI gate will consume, but nothing validates it — |
||
| "generated_at": "2026-09-05T01:50:55.103Z", | ||
| "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731", | ||
| "reasoning_effort": "high", | ||
| "scenarios": [ | ||
| "tier1-directory", | ||
| "tier1-directory-guide-only" | ||
| ], | ||
|
Comment on lines
+6
to
+9
Contributor
There was a problem hiding this comment. 🟡 Suggestion: |
||
| "modes": { | ||
| "guide-only": { | ||
| "run_ids": [ | ||
| "evals-tier1-directory-guide-only-20260905-000901-207", | ||
| "evals-tier1-directory-guide-only-20260905-012128-950", | ||
| "evals-tier1-directory-guide-only-20260905-015055-103" | ||
| ], | ||
| "runs": 3, | ||
| "passes": 0, | ||
| "pass_rate": 0, | ||
| "pass_at_3": 0, | ||
| "first_pass_rate_mean": 0.6111111111111112, | ||
| "per_stage": { | ||
| "S0": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 1 | ||
| }, | ||
| "S1": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S2": { | ||
| "runs": 3, | ||
| "failures": 3, | ||
| "first_pass_failures": 3 | ||
| }, | ||
| "S3": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S4": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 1 | ||
| }, | ||
| "S5": { | ||
| "runs": 3, | ||
| "failures": 2, | ||
| "first_pass_failures": 2 | ||
| }, | ||
| "S6": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S7": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S8": { | ||
| "runs": 3, | ||
| "failures": 3, | ||
| "first_pass_failures": 3 | ||
| }, | ||
| "S9": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S10": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 1 | ||
| }, | ||
| "S11": { | ||
| "runs": 3, | ||
| "failures": 3, | ||
| "first_pass_failures": 3 | ||
| } | ||
| } | ||
| }, | ||
| "none": { | ||
| "run_ids": [ | ||
| "evals-tier1-directory-20260904-204206-352", | ||
| "evals-tier1-directory-20260904-221013-308", | ||
| "evals-tier1-directory-20260904-231553-272" | ||
| ], | ||
| "runs": 3, | ||
| "passes": 0, | ||
| "pass_rate": 0, | ||
| "pass_at_3": 0, | ||
| "first_pass_rate_mean": 0.47222222222222227, | ||
| "per_stage": { | ||
| "S0": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 1 | ||
| }, | ||
| "S1": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S2": { | ||
| "runs": 3, | ||
| "failures": 3, | ||
| "first_pass_failures": 3 | ||
| }, | ||
| "S3": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 2 | ||
| }, | ||
| "S4": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 3 | ||
| }, | ||
| "S5": { | ||
| "runs": 3, | ||
| "failures": 2, | ||
| "first_pass_failures": 3 | ||
| }, | ||
| "S6": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S7": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S8": { | ||
| "runs": 3, | ||
| "failures": 3, | ||
| "first_pass_failures": 3 | ||
| }, | ||
| "S9": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 0 | ||
| }, | ||
| "S10": { | ||
| "runs": 3, | ||
| "failures": 0, | ||
| "first_pass_failures": 2 | ||
| }, | ||
| "S11": { | ||
| "runs": 3, | ||
| "failures": 2, | ||
| "first_pass_failures": 2 | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "pareto": [ | ||
| { | ||
| "stage": "S2", | ||
| "failures": 6, | ||
| "share": 0.2857142857142857 | ||
| }, | ||
| { | ||
| "stage": "S8", | ||
| "failures": 6, | ||
| "share": 0.2857142857142857 | ||
| }, | ||
| { | ||
| "stage": "S11", | ||
| "failures": 5, | ||
| "share": 0.23809523809523808 | ||
| }, | ||
| { | ||
| "stage": "S5", | ||
| "failures": 4, | ||
| "share": 0.19047619047619047 | ||
| } | ||
| ] | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🟡 Suggestion: The regression-gate contract above (lines 231–235) still says the gate "fails if the measured pass rate drops below
modes.<mode>.pass_rate", but the published reference haspass_rate: 0for both arms — that rule can never fire, so the committed contract is a no-op gate. This paragraph itself explains why the full-funnel floor is partly structural (S2's PUT leg is unobservable, S8 reads secret-masked); the consumption rule should be updated in the same PR to key onfirst_pass_rate_meanand/orper_stage.failures, or to state explicitly thatpass_rategating is inert until S2/S8 are instrumented. (confidence: high)