From 5487b681e54a729965962f80ceeb001964825abd Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Thu, 9 Jul 2026 02:20:39 +0300 Subject: [PATCH] fix(harbor): access-tier integrity (free-baseline privilege, k-anonymity floor, trusted nested CLI) Four related fixes to keep hidden-split information and scoring authority where they belong: 1. The free baseline eval no longer rides the admin flag. engine.evaluate gains a distinct `free` parameter that waives only the budget debit; admin=True also bypassed the no_access tier gate, so the agent's one free eval could target the held-out test split and read its aggregate score off the response. The freebie is also now consumed only after a successful eval, so an infra failure no longer burns it. 2. serve.py now passes split_accesses into the EvaluationEngine. Without it the engine-side no_access gate was dormant and the budget ledger (no_access splits are unbudgeted) was the only gate, which is exactly what every unmetered path skipped. 3. k-anonymity floor on subset evals of non_viewable splits (default 5, configurable via build.yaml / serve.json). EvalSummary.mean_score over an agent-chosen singleton subset is that sample's label-derived score verbatim, so n singleton evals reconstructed a hidden split's labels wholesale. Full-split evals always pass (their aggregate is the intended surface), so splits smaller than the floor stay evaluable. The floor is advertised in status() as min_subset_samples. 4. HarborConfig.harbor_requirement: when set, the nested `harbor run` is layered over the candidate env with `uv run --with `, so the orchestrator that produces trial result.json resolves from the trusted spec, not from the candidate's own pyproject/uv.lock (one edited line there could point at a fork that fabricates results). Verified that uv's ephemeral overlay takes precedence over a conflicting project pin for both the console script and sys.path. This raises the bar, not a full boundary: agent code still imports into the nested harbor process; out-of-process verification is tracked separately. Co-Authored-By: Claude Opus 4.8 (1M context) --- vero/src/vero/evaluation/engine.py | 16 +++- vero/src/vero/harbor/app.py | 6 +- vero/src/vero/harbor/build/compiler.py | 1 + vero/src/vero/harbor/build/config.py | 5 + vero/src/vero/harbor/config.py | 10 ++ vero/src/vero/harbor/protocol.py | 10 ++ vero/src/vero/harbor/runner.py | 14 ++- vero/src/vero/harbor/serve.py | 20 +++- vero/src/vero/harbor/server.py | 47 +++++++++- vero/tests/test_engine.py | 32 +++++++ vero/tests/test_harbor_protocol.py | 19 ++++ vero/tests/test_harbor_runner.py | 18 ++++ vero/tests/test_harbor_server.py | 123 +++++++++++++++++++++++-- 13 files changed, 299 insertions(+), 22 deletions(-) diff --git a/vero/src/vero/evaluation/engine.py b/vero/src/vero/evaluation/engine.py index 69b9483..201f51c 100644 --- a/vero/src/vero/evaluation/engine.py +++ b/vero/src/vero/evaluation/engine.py @@ -146,8 +146,18 @@ def resolve_samples(self, req: EvalRequest) -> tuple[list[int] | None, int]: # Evaluation # ------------------------------------------------------------------ - async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> Experiment: - """Meter (unless admin) and run one evaluation; return the full Experiment. + async def evaluate( + self, req: EvalRequest, *, admin: bool = False, free: bool = False + ) -> Experiment: + """Meter (unless admin or free) and run one evaluation; return the full + Experiment. + + ``admin`` and ``free`` are distinct authorities and must stay separate: + ``admin`` grants ACCESS (bypasses the tier gate and the ledger), while + ``free`` only waives the BUDGET debit for an otherwise ordinary agent + eval (the sidecar's one free baseline eval). A free eval still runs as + the agent: conflating the two let the free-baseline path evaluate + ``no_access`` splits and return their aggregate score to the agent. ``no_access`` gating is EXPLICIT and fail-closed: when ``split_accesses`` is configured, the split's tier is resolved (an unlisted split defaults @@ -163,7 +173,7 @@ async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> Experiment f"and cannot be evaluated by the agent." ) sample_ids, n = self.resolve_samples(req) - if not admin: + if not admin and not free: await self.budget.reserve(req.dataset_id, req.split, n) return await self.evaluator.evaluate( commit=req.commit, diff --git a/vero/src/vero/harbor/app.py b/vero/src/vero/harbor/app.py index 16a53ec..f8e2432 100644 --- a/vero/src/vero/harbor/app.py +++ b/vero/src/vero/harbor/app.py @@ -17,7 +17,7 @@ from vero.evaluation.engine import EvalRequest from vero.exceptions import ExperimentBudgetExceeded, InvalidSplitError from vero.harbor.auth import check_admin -from vero.harbor.server import SubmitDisabledError +from vero.harbor.server import KAnonymityError, SubmitDisabledError from vero.harbor.verifier import NoCandidateError if TYPE_CHECKING: @@ -58,6 +58,10 @@ def create_app( SubmitDisabledError, lambda r, e: JSONResponse(status_code=409, content={"error": str(e)}), ) + app.add_exception_handler( + KAnonymityError, + lambda r, e: JSONResponse(status_code=400, content={"error": str(e)}), + ) app.add_exception_handler( NoCandidateError, lambda r, e: JSONResponse(status_code=409, content={"error": str(e)}), diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index 98f2073..2021399 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -242,6 +242,7 @@ def _serve_config( "base_commit": base_commit, "submit_enabled": config.submit_enabled, "score_baseline": config.score_baseline, + "k_anonymity_floor": config.k_anonymity_floor, "agent_volume": AGENT_VOLUME, "admin_volume": ADMIN_VOLUME, "admin_token_path": TOKEN_PATH, diff --git a/vero/src/vero/harbor/build/config.py b/vero/src/vero/harbor/build/config.py index 4cb4a54..8cf6c9a 100644 --- a/vero/src/vero/harbor/build/config.py +++ b/vero/src/vero/harbor/build/config.py @@ -67,6 +67,11 @@ class _BuildConfigBase(BaseModel): # WORSE than the untouched repo is visible as a regression. score_baseline: bool = False + # Minimum sample count for agent-chosen subset evals of non_viewable splits + # (full-split evals always pass; <=1 disables). Aggregate responses carry + # mean_score, so singleton subsets would hand back per-sample labels. + k_anonymity_floor: int = 5 + # write-access: paths in the target repo the optimizer may NOT edit # (the scorer, by default). Applied as unix perms in main before the agent runs. read_only_paths: list[str] = Field(default_factory=list) diff --git a/vero/src/vero/harbor/config.py b/vero/src/vero/harbor/config.py index 387c132..dea857f 100644 --- a/vero/src/vero/harbor/config.py +++ b/vero/src/vero/harbor/config.py @@ -30,6 +30,16 @@ class HarborConfig: # estimates pass probability instead of pass@k (which "best" # inflates toward). aggregate_attempts: str = "best" + # Trusted source for the nested `harbor` CLI, as a uv requirement spec + # (e.g. "harbor==0.1.17" or a pinned git URL). When set, the runner layers + # it over the candidate env with `uv run --with`, whose ephemeral overlay + # takes precedence for both the console script and sys.path — so the + # orchestrator that scores the candidate resolves from THIS spec, not from + # whatever the candidate's own pyproject/uv.lock pin (which the agent + # controls, and could point at a fork that fabricates trial results + # without running anything). None keeps the current behavior: the + # candidate env supplies harbor, and is trusted to. + harbor_requirement: str | None = None extra_args: list[str] = field(default_factory=list) # passthrough harbor run flags def __post_init__(self) -> None: diff --git a/vero/src/vero/harbor/protocol.py b/vero/src/vero/harbor/protocol.py index 20c0500..c43f101 100644 --- a/vero/src/vero/harbor/protocol.py +++ b/vero/src/vero/harbor/protocol.py @@ -104,6 +104,7 @@ def build_status( split_accesses: list[SplitAccess], base_commit: str | None = None, free_baseline_available: bool = False, + k_anonymity_floor: int = 1, ) -> StatusSummary: """Build the agent-facing status from the budget ledger + split tiers. @@ -119,6 +120,15 @@ def build_status( "dataset_id": dataset_id, "tier": str(tier), "agent_evaluable": tier != SplitAccessLevel.no_access, + # Subset evals below this sample count are rejected on + # non_viewable splits (full-split evals always pass). Advertised + # so an agent plans subset probes instead of burning budget on + # requests the sidecar will refuse. + "min_subset_samples": ( + k_anonymity_floor + if tier == SplitAccessLevel.non_viewable + else 1 + ), "remaining_sample_budget": b.remaining_sample_budget, "remaining_run_budget": b.remaining_run_budget, } diff --git a/vero/src/vero/harbor/runner.py b/vero/src/vero/harbor/runner.py index 7b6eede..5bd3754 100644 --- a/vero/src/vero/harbor/runner.py +++ b/vero/src/vero/harbor/runner.py @@ -109,8 +109,18 @@ def _build_command( jobs_dir: Path, ) -> list[str]: c = self.config - cmd = [ - "uv", "run", "--project", project_path, + cmd = ["uv", "run", "--project", project_path] + # Resolve the nested harbor from the trusted spec rather than the + # candidate's lockfile (see HarborConfig.harbor_requirement). This + # raises the bar from "edit one pyproject line" to "tamper with the + # orchestrator in-process at runtime"; it is not a full boundary — + # the agent's code still imports into the nested harbor process via + # --agent-import-path, so a hostile candidate can in principle still + # forge results from inside. Full isolation needs an out-of-process + # verifier and is tracked separately. + if c.harbor_requirement: + cmd += ["--with", c.harbor_requirement] + cmd += [ "harbor", "run", *c.source_args(), "--agent-import-path", c.agent_import_path, diff --git a/vero/src/vero/harbor/serve.py b/vero/src/vero/harbor/serve.py index 603b650..00eb327 100644 --- a/vero/src/vero/harbor/serve.py +++ b/vero/src/vero/harbor/serve.py @@ -77,6 +77,11 @@ class _ServeConfigBase(BaseModel): # on the selection split; it reverts to base_commit instead (needs base_commit). auto_best_baseline_floor: bool = True + # Minimum sample count for agent-chosen subset evals of non_viewable + # splits (full-split evals always pass; <=1 disables). See + # EvaluationSidecar.k_anonymity_floor for the leak this closes. + k_anonymity_floor: int = 5 + # volumes / token agent_volume: str admin_volume: str @@ -240,6 +245,10 @@ async def build_components( eval_strategy=eval_strategy, ) + split_accesses = [ + SplitAccess(split=s.split, access=SplitAccessLevel(s.access)) + for s in config.split_accesses + ] db = ExperimentDatabase(id=config.session_id) # shared by engine (writes) + verifier (reads) engine = EvaluationEngine( evaluator=evaluator, @@ -253,12 +262,12 @@ async def build_components( ), session_id=config.session_id, vero_home=vero_home, + # The engine-side no_access gate is only armed when split_accesses is + # set. Without it the ledger was the sole gate (no_access splits are + # unbudgeted, so reserve() raised) — and every unmetered path (admin, + # the free baseline eval) walked straight past it. + split_accesses=split_accesses, ) - - split_accesses = [ - SplitAccess(split=s.split, access=SplitAccessLevel(s.access)) - for s in config.split_accesses - ] sidecar = EvaluationSidecar( engine=engine, split_accesses=split_accesses, @@ -267,6 +276,7 @@ async def build_components( admin_volume=Path(config.admin_volume), submit_enabled=config.submit_enabled, base_commit=config.base_commit, + k_anonymity_floor=config.k_anonymity_floor, ) verifier = Verifier( engine=engine, diff --git a/vero/src/vero/harbor/server.py b/vero/src/vero/harbor/server.py index 1f99f88..f74f0b3 100644 --- a/vero/src/vero/harbor/server.py +++ b/vero/src/vero/harbor/server.py @@ -38,6 +38,11 @@ class SubmitDisabledError(RuntimeError): """Raised when submit() is called but the task does not use submit selection.""" +class KAnonymityError(RuntimeError): + """Raised when an agent eval on a non_viewable split selects fewer samples + than the k-anonymity floor allows.""" + + class EvaluationSidecar: """Agent-facing handlers over the EvaluationEngine. @@ -56,6 +61,7 @@ def __init__( admin_volume: Path, submit_enabled: bool = False, base_commit: str | None = None, + k_anonymity_floor: int = 5, ): self.engine = engine self.split_accesses = split_accesses @@ -64,6 +70,17 @@ def __init__( self.admin_volume = Path(admin_volume) self.submit_enabled = submit_enabled self.base_commit = base_commit + # Minimum sample count for an agent-chosen SUBSET eval of a non_viewable + # split. The aggregate response carries mean_score, so a singleton subset + # returns the sample's label-derived score verbatim, and n singleton + # evals reconstruct the split's per-sample labels wholesale. The floor + # applies only to proper subsets (sample_ids/num_samples): a full-split + # eval reveals exactly the intended aggregate, so it always passes and a + # split smaller than the floor degrades to full-split-only rather than + # becoming unevaluable. This is a cost multiplier, not a proof: means of + # k-sized overlapping subsets still admit reconstruction by elimination, + # but at >= k times the sample budget per label. <= 1 disables the floor. + self.k_anonymity_floor = k_anonymity_floor self._free_baseline_used = False # ------------------------------------------------------------------ @@ -71,6 +88,23 @@ def __init__( # ------------------------------------------------------------------ async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> EvalSummary: + # k-anonymity floor, checked before any work (no commit transfer, no + # budget debit, no eval) so a rejected request costs the agent nothing. + # sample_ids is None exactly when the request covers the full split + # (resolve_samples collapses a covering num_samples to None too), and a + # full-split aggregate is the intended surface, so only proper subsets + # are floored. + if not admin and self.k_anonymity_floor > 1: + tier = tier_for_split(req.split, self.split_accesses) + if tier == SplitAccessLevel.non_viewable: + sample_ids, n = self.engine.resolve_samples(req) + if sample_ids is not None and n < self.k_anonymity_floor: + raise KAnonymityError( + f"Evals on non_viewable split '{req.split}' must cover " + f"at least {self.k_anonymity_floor} samples (or the " + f"whole split); got {n}. Aggregate scores over smaller " + f"subsets would reveal per-sample results." + ) sha = await self._transfer_commit(req.commit) # The agent's FIRST eval of the seeded baseline is budget-free. The # baseline is the reference every candidate is implicitly compared to, @@ -80,17 +114,23 @@ async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> EvalSummar # an optimizer that skipped the reference could not tell a no-op edit # from an improvement and quit with budget unspent). Capped at one: # later baseline evals debit normally, so free compute is bounded. + # `free` waives only the budget debit; the eval still runs as the agent + # (tier gates apply), so the free baseline cannot touch no_access + # splits — riding the admin flag here did exactly that. free_baseline = ( not admin and self.base_commit is not None and sha == self.base_commit and not self._free_baseline_used ) - if free_baseline: - self._free_baseline_used = True exp = await self.engine.evaluate( - replace(req, commit=sha), admin=admin or free_baseline + replace(req, commit=sha), admin=admin, free=free_baseline ) + # Consume the freebie only after the eval succeeded: an eval that + # raised (invalid split, infra failure) has given the agent nothing, + # so it must not burn the one free reference measurement. + if free_baseline: + self._free_baseline_used = True # Route with the agent's real tier even when the eval was unmetered. result_path = self._route_results(exp, admin=admin) budget_remaining = None @@ -125,6 +165,7 @@ def status(self) -> StatusSummary: free_baseline_available=( self.base_commit is not None and not self._free_baseline_used ), + k_anonymity_floor=self.k_anonymity_floor, ) def list_experiments(self) -> list[dict]: diff --git a/vero/tests/test_engine.py b/vero/tests/test_engine.py index 0349adf..401e6b9 100644 --- a/vero/tests/test_engine.py +++ b/vero/tests/test_engine.py @@ -162,3 +162,35 @@ async def test_non_viewable_split_still_evaluable(self, monkeypatch): ) svc.evaluator.evaluate.assert_awaited_once() assert svc.status()[("dev", "ds1")].remaining_run_budget == 2 + + +class TestFreeEval: + """`free` waives only the budget debit; every access gate still applies. + + free and admin are distinct authorities: the sidecar's free baseline eval + once rode the admin flag and could thereby evaluate no_access splits and + hand the agent their aggregate score.""" + + @pytest.mark.asyncio + async def test_free_runs_without_debiting_budget(self, monkeypatch): + svc = _make_service(monkeypatch=monkeypatch) + await svc.evaluate( + EvalRequest(dataset_id="ds1", split="dev", commit="c1", num_samples=10), + free=True, + ) + svc.evaluator.evaluate.assert_awaited_once() + assert svc.status()[("dev", "ds1")].remaining_run_budget == 3 + assert svc.status()[("dev", "ds1")].remaining_sample_budget == 100 + + @pytest.mark.asyncio + async def test_free_does_not_bypass_no_access_gate(self, monkeypatch): + from vero.core.dataset import SplitAccess + + svc = _make_service(monkeypatch=monkeypatch) + svc.split_accesses = [SplitAccess.no_access("test")] + with pytest.raises(InvalidSplitError): + await svc.evaluate( + EvalRequest(dataset_id="ds1", split="test", commit="c1", num_samples=10), + free=True, + ) + svc.evaluator.evaluate.assert_not_awaited() diff --git a/vero/tests/test_harbor_protocol.py b/vero/tests/test_harbor_protocol.py index e636f46..b7bd6f2 100644 --- a/vero/tests/test_harbor_protocol.py +++ b/vero/tests/test_harbor_protocol.py @@ -91,3 +91,22 @@ def test_lists_budgeted_splits_with_tier(self): assert by_split["validation"]["tier"] == str(SplitAccessLevel.non_viewable) assert by_split["validation"]["agent_evaluable"] is True assert by_split["validation"]["remaining_run_budget"] == 3 + + def test_advertises_subset_floor_on_non_viewable_only(self): + budget = { + ("train", "ds1"): SplitBudget(split="train", dataset_id="ds1", total_run_budget=10), + ("validation", "ds1"): SplitBudget(split="validation", dataset_id="ds1", total_run_budget=3), + } + accesses = [ + SplitAccess.viewable("train"), + SplitAccess.non_viewable("validation"), + ] + status = build_status( + submit_enabled=False, + budget=budget, + split_accesses=accesses, + k_anonymity_floor=5, + ) + by_split = {s["split"]: s for s in status.splits} + assert by_split["validation"]["min_subset_samples"] == 5 + assert by_split["train"]["min_subset_samples"] == 1 diff --git a/vero/tests/test_harbor_runner.py b/vero/tests/test_harbor_runner.py index 3b90e66..4fe93db 100644 --- a/vero/tests/test_harbor_runner.py +++ b/vero/tests/test_harbor_runner.py @@ -99,6 +99,24 @@ def test_local_source(self, tmp_path): assert "-p" in cmd and str(tmp_path) in cmd assert "-d" not in cmd + def test_harbor_requirement_layers_trusted_cli(self): + # The overlay must come before the `harbor` executable name so uv + # resolves the CLI from the trusted spec, not the candidate's lockfile. + runner = HarborRunner( + HarborConfig( + task_source="org/ds@1", + agent_import_path="pkg.mod:Agent", + harbor_requirement="harbor==0.1.17", + ) + ) + cmd = runner._build_command("/wt", _params(), ["t0"], Path("/jobs")) + assert cmd[:6] == ["uv", "run", "--project", "/wt", "--with", "harbor==0.1.17"] + assert cmd[6] == "harbor" + + def test_no_harbor_requirement_keeps_candidate_env(self): + cmd = _runner()._build_command("/wt", _params(), ["t0"], Path("/jobs")) + assert "--with" not in cmd + class TestExtractReward: def test_priority_pass_then_reward_then_sole_key(self): diff --git a/vero/tests/test_harbor_server.py b/vero/tests/test_harbor_server.py index 57f233f..741365d 100644 --- a/vero/tests/test_harbor_server.py +++ b/vero/tests/test_harbor_server.py @@ -41,9 +41,19 @@ def _experiment(split: str, commit: str = "abcdef123456") -> Experiment: ) -def _sidecar(tmp_path, *, split, submit_enabled=False, accesses=None, base_commit=None): +def _sidecar( + tmp_path, + *, + split, + submit_enabled=False, + accesses=None, + base_commit=None, + k_anonymity_floor=5, +): engine = MagicMock() engine.evaluate = AsyncMock(return_value=_experiment(split)) + # Full-split request by default (sample_ids None); floor tests override. + engine.resolve_samples = MagicMock(return_value=(None, 3)) engine.budget = BudgetLedger( [SplitBudget(split=split, dataset_id="ds1", total_run_budget=5, total_sample_budget=100)] ) @@ -56,6 +66,7 @@ def _sidecar(tmp_path, *, split, submit_enabled=False, accesses=None, base_commi admin_volume=tmp_path / "admin_vol", submit_enabled=submit_enabled, base_commit=base_commit, + k_anonymity_floor=k_anonymity_floor, ) # Stub the git transfer (integration-tested separately); pin the sha. sidecar._transfer_commit = AsyncMock(return_value="abcdef123456") @@ -218,6 +229,88 @@ def test_status_reports_submit_and_splits(self, tmp_path): assert status.splits[0]["remaining_run_budget"] == 5 +class TestKAnonymityFloor: + """Subset evals on non_viewable splits are floored: the aggregate response + carries mean_score, so a singleton subset returns that sample's score + verbatim, and n singleton evals reconstruct the split's labels wholesale.""" + + def _floored(self, tmp_path, **kw): + return _sidecar(tmp_path, split="validation", **kw) + + @pytest.mark.asyncio + async def test_subset_below_floor_rejected_before_any_work(self, tmp_path): + from vero.harbor.server import KAnonymityError + + sidecar = self._floored(tmp_path) + sidecar.engine.resolve_samples = MagicMock(return_value=([0], 1)) + with pytest.raises(KAnonymityError): + await sidecar.evaluate( + EvalRequest(dataset_id="ds1", split="validation", sample_ids=[0]) + ) + # rejected up front: no commit transfer, no eval, no budget debit + sidecar._transfer_commit.assert_not_awaited() + sidecar.engine.evaluate.assert_not_awaited() + + @pytest.mark.asyncio + async def test_subset_at_floor_allowed(self, tmp_path): + sidecar = self._floored(tmp_path) + sidecar.engine.resolve_samples = MagicMock(return_value=(list(range(5)), 5)) + await sidecar.evaluate( + EvalRequest(dataset_id="ds1", split="validation", sample_ids=list(range(5))) + ) + sidecar.engine.evaluate.assert_awaited_once() + + @pytest.mark.asyncio + async def test_full_split_always_passes(self, tmp_path): + # sample_ids None == the whole split; its aggregate is the intended + # surface, so a split smaller than the floor stays evaluable. + sidecar = self._floored(tmp_path) + sidecar.engine.resolve_samples = MagicMock(return_value=(None, 3)) + await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation")) + sidecar.engine.evaluate.assert_awaited_once() + + @pytest.mark.asyncio + async def test_viewable_split_not_floored(self, tmp_path): + # per-sample results are already exposed on viewable splits; nothing to protect + sidecar = _sidecar( + tmp_path, split="train", accesses=[SplitAccess.viewable("train")] + ) + sidecar.engine.resolve_samples = MagicMock(return_value=([0], 1)) + await sidecar.evaluate( + EvalRequest(dataset_id="ds1", split="train", sample_ids=[0]) + ) + sidecar.engine.evaluate.assert_awaited_once() + + @pytest.mark.asyncio + async def test_admin_exempt(self, tmp_path): + sidecar = self._floored(tmp_path) + sidecar.engine.resolve_samples = MagicMock(return_value=([0], 1)) + await sidecar.evaluate( + EvalRequest(dataset_id="ds1", split="validation", sample_ids=[0]), + admin=True, + ) + sidecar.engine.evaluate.assert_awaited_once() + + @pytest.mark.asyncio + async def test_floor_of_one_disables(self, tmp_path): + sidecar = self._floored(tmp_path, k_anonymity_floor=1) + sidecar.engine.resolve_samples = MagicMock(return_value=([0], 1)) + await sidecar.evaluate( + EvalRequest(dataset_id="ds1", split="validation", sample_ids=[0]) + ) + sidecar.engine.evaluate.assert_awaited_once() + + def test_status_advertises_floor(self, tmp_path): + sidecar = self._floored(tmp_path) + assert sidecar.status().splits[0]["min_subset_samples"] == 5 + + def test_status_floor_is_one_for_viewable(self, tmp_path): + sidecar = _sidecar( + tmp_path, split="train", accesses=[SplitAccess.viewable("train")] + ) + assert sidecar.status().splits[0]["min_subset_samples"] == 1 + + class TestFreeBaselineEval: """The agent's first eval of the seeded baseline is budget-free: it is the reference every candidate is compared to and can never win selection, so @@ -226,12 +319,14 @@ class TestFreeBaselineEval: no-op edit from an improvement, and quit with budget unspent).""" @pytest.mark.asyncio - async def test_first_baseline_eval_is_unmetered(self, tmp_path): + async def test_first_baseline_eval_is_unmetered_but_not_admin(self, tmp_path): sidecar = _sidecar(tmp_path, split="validation", base_commit="abcdef123456") await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation")) - # engine.evaluate was called with admin=True (bypasses the ledger) - assert sidecar.engine.evaluate.await_args.kwargs["admin"] is True - # but results were routed with the agent tier (summary written) + # free waives the ledger; admin stays False so every access gate applies + # (riding admin=True here let the free baseline evaluate no_access splits) + assert sidecar.engine.evaluate.await_args.kwargs["free"] is True + assert sidecar.engine.evaluate.await_args.kwargs["admin"] is False + # and results were routed with the agent tier (summary written) dest = tmp_path / "agent_vol" / "results" / "validation__abcdef123456" assert (dest / "summary.json").exists() @@ -240,19 +335,31 @@ async def test_second_baseline_eval_is_metered(self, tmp_path): sidecar = _sidecar(tmp_path, split="validation", base_commit="abcdef123456") await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation")) await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation")) - assert sidecar.engine.evaluate.await_args.kwargs["admin"] is False + assert sidecar.engine.evaluate.await_args.kwargs["free"] is False @pytest.mark.asyncio async def test_non_baseline_commit_always_metered(self, tmp_path): sidecar = _sidecar(tmp_path, split="validation", base_commit="other000000") await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation")) - assert sidecar.engine.evaluate.await_args.kwargs["admin"] is False + assert sidecar.engine.evaluate.await_args.kwargs["free"] is False @pytest.mark.asyncio async def test_no_base_commit_never_free(self, tmp_path): sidecar = _sidecar(tmp_path, split="validation") # base_commit=None await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation")) - assert sidecar.engine.evaluate.await_args.kwargs["admin"] is False + assert sidecar.engine.evaluate.await_args.kwargs["free"] is False + + @pytest.mark.asyncio + async def test_failed_free_eval_does_not_consume_freebie(self, tmp_path): + sidecar = _sidecar(tmp_path, split="validation", base_commit="abcdef123456") + sidecar.engine.evaluate = AsyncMock(side_effect=RuntimeError("infra")) + with pytest.raises(RuntimeError): + await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation")) + # the failed eval gave the agent nothing; the free reference remains + assert sidecar.status().free_baseline_available is True + sidecar.engine.evaluate = AsyncMock(return_value=_experiment("validation")) + await sidecar.evaluate(EvalRequest(dataset_id="ds1", split="validation")) + assert sidecar.engine.evaluate.await_args.kwargs["free"] is True def test_status_surfaces_free_baseline(self, tmp_path): sidecar = _sidecar(tmp_path, split="train", base_commit="abcdef123456")