fix: [no-ticket] harbor - access-tier integrity (free-baseline privilege, k-anonymity floor, trusted nested CLI)

vero/src/vero/evaluation/engine.py

@@ -146,8 +146,18 @@ def resolve_samples(self, req: EvalRequest) -> tuple[list[int] | None, int]:
     # Evaluation
     # ------------------------------------------------------------------
 
-    async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> Experiment:
-        """Meter (unless admin) and run one evaluation; return the full Experiment.
+    async def evaluate(
+        self, req: EvalRequest, *, admin: bool = False, free: bool = False
+    ) -> Experiment:
+        """Meter (unless admin or free) and run one evaluation; return the full
+        Experiment.
+
+        ``admin`` and ``free`` are distinct authorities and must stay separate:
+        ``admin`` grants ACCESS (bypasses the tier gate and the ledger), while
+        ``free`` only waives the BUDGET debit for an otherwise ordinary agent
+        eval (the sidecar's one free baseline eval). A free eval still runs as
+        the agent: conflating the two let the free-baseline path evaluate
+        ``no_access`` splits and return their aggregate score to the agent.
 
         ``no_access`` gating is EXPLICIT and fail-closed: when ``split_accesses``
         is configured, the split's tier is resolved (an unlisted split defaults
@@ -163,7 +173,7 @@ async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> Experiment
                     f"and cannot be evaluated by the agent."
                 )
         sample_ids, n = self.resolve_samples(req)
-        if not admin:
+        if not admin and not free:
             await self.budget.reserve(req.dataset_id, req.split, n)
         return await self.evaluator.evaluate(
             commit=req.commit,

vero/src/vero/harbor/app.py

@@ -17,7 +17,7 @@
 from vero.evaluation.engine import EvalRequest
 from vero.exceptions import ExperimentBudgetExceeded, InvalidSplitError
 from vero.harbor.auth import check_admin
-from vero.harbor.server import SubmitDisabledError
+from vero.harbor.server import KAnonymityError, SubmitDisabledError
 from vero.harbor.verifier import NoCandidateError
 
 if TYPE_CHECKING:
@@ -58,6 +58,10 @@ def create_app(
         SubmitDisabledError,
         lambda r, e: JSONResponse(status_code=409, content={"error": str(e)}),
     )
+    app.add_exception_handler(
+        KAnonymityError,
+        lambda r, e: JSONResponse(status_code=400, content={"error": str(e)}),
+    )
     app.add_exception_handler(
         NoCandidateError,
         lambda r, e: JSONResponse(status_code=409, content={"error": str(e)}),

vero/src/vero/harbor/build/compiler.py

@@ -242,6 +242,7 @@ def _serve_config(
         "base_commit": base_commit,
         "submit_enabled": config.submit_enabled,
         "score_baseline": config.score_baseline,
+        "k_anonymity_floor": config.k_anonymity_floor,
         "agent_volume": AGENT_VOLUME,
         "admin_volume": ADMIN_VOLUME,
         "admin_token_path": TOKEN_PATH,

vero/src/vero/harbor/build/config.py

@@ -67,6 +67,11 @@ class _BuildConfigBase(BaseModel):
     # WORSE than the untouched repo is visible as a regression.
     score_baseline: bool = False
 
+    # Minimum sample count for agent-chosen subset evals of non_viewable splits
+    # (full-split evals always pass; <=1 disables). Aggregate responses carry
+    # mean_score, so singleton subsets would hand back per-sample labels.
+    k_anonymity_floor: int = 5
+
     # write-access: paths in the target repo the optimizer may NOT edit
     # (the scorer, by default). Applied as unix perms in main before the agent runs.
     read_only_paths: list[str] = Field(default_factory=list)

vero/src/vero/harbor/config.py

@@ -30,6 +30,16 @@ class HarborConfig:
     #           estimates pass probability instead of pass@k (which "best"
     #           inflates toward).
     aggregate_attempts: str = "best"
+    # Trusted source for the nested `harbor` CLI, as a uv requirement spec
+    # (e.g. "harbor==0.1.17" or a pinned git URL). When set, the runner layers
+    # it over the candidate env with `uv run --with`, whose ephemeral overlay
+    # takes precedence for both the console script and sys.path — so the
+    # orchestrator that scores the candidate resolves from THIS spec, not from
+    # whatever the candidate's own pyproject/uv.lock pin (which the agent
+    # controls, and could point at a fork that fabricates trial results
+    # without running anything). None keeps the current behavior: the
+    # candidate env supplies harbor, and is trusted to.
+    harbor_requirement: str | None = None
     extra_args: list[str] = field(default_factory=list)  # passthrough harbor run flags
 
     def __post_init__(self) -> None:

vero/src/vero/harbor/protocol.py

@@ -104,6 +104,7 @@ def build_status(
     split_accesses: list[SplitAccess],
     base_commit: str | None = None,
     free_baseline_available: bool = False,
+    k_anonymity_floor: int = 1,
 ) -> StatusSummary:
     """Build the agent-facing status from the budget ledger + split tiers.
 
@@ -119,6 +120,15 @@ def build_status(
                 "dataset_id": dataset_id,
                 "tier": str(tier),
                 "agent_evaluable": tier != SplitAccessLevel.no_access,
+                # Subset evals below this sample count are rejected on
+                # non_viewable splits (full-split evals always pass). Advertised
+                # so an agent plans subset probes instead of burning budget on
+                # requests the sidecar will refuse.
+                "min_subset_samples": (
+                    k_anonymity_floor
+                    if tier == SplitAccessLevel.non_viewable
+                    else 1
+                ),
                 "remaining_sample_budget": b.remaining_sample_budget,
                 "remaining_run_budget": b.remaining_run_budget,
             }

vero/src/vero/harbor/runner.py

@@ -109,8 +109,18 @@ def _build_command(
         jobs_dir: Path,
     ) -> list[str]:
         c = self.config
-        cmd = [
-            "uv", "run", "--project", project_path,
+        cmd = ["uv", "run", "--project", project_path]
+        # Resolve the nested harbor from the trusted spec rather than the
+        # candidate's lockfile (see HarborConfig.harbor_requirement). This
+        # raises the bar from "edit one pyproject line" to "tamper with the
+        # orchestrator in-process at runtime"; it is not a full boundary —
+        # the agent's code still imports into the nested harbor process via
+        # --agent-import-path, so a hostile candidate can in principle still
+        # forge results from inside. Full isolation needs an out-of-process
+        # verifier and is tracked separately.
+        if c.harbor_requirement:
+            cmd += ["--with", c.harbor_requirement]
+        cmd += [
             "harbor", "run",
             *c.source_args(),
             "--agent-import-path", c.agent_import_path,

vero/src/vero/harbor/serve.py

@@ -77,6 +77,11 @@ class _ServeConfigBase(BaseModel):
     # on the selection split; it reverts to base_commit instead (needs base_commit).
     auto_best_baseline_floor: bool = True
 
+    # Minimum sample count for agent-chosen subset evals of non_viewable
+    # splits (full-split evals always pass; <=1 disables). See
+    # EvaluationSidecar.k_anonymity_floor for the leak this closes.
+    k_anonymity_floor: int = 5
+
     # volumes / token
     agent_volume: str
     admin_volume: str
@@ -240,6 +245,10 @@ async def build_components(
         eval_strategy=eval_strategy,
     )
 
+    split_accesses = [
+        SplitAccess(split=s.split, access=SplitAccessLevel(s.access))
+        for s in config.split_accesses
+    ]
     db = ExperimentDatabase(id=config.session_id)  # shared by engine (writes) + verifier (reads)
     engine = EvaluationEngine(
         evaluator=evaluator,
@@ -253,12 +262,12 @@ async def build_components(
         ),
         session_id=config.session_id,
         vero_home=vero_home,
+        # The engine-side no_access gate is only armed when split_accesses is
+        # set. Without it the ledger was the sole gate (no_access splits are
+        # unbudgeted, so reserve() raised) — and every unmetered path (admin,
+        # the free baseline eval) walked straight past it.
+        split_accesses=split_accesses,
     )
-
-    split_accesses = [
-        SplitAccess(split=s.split, access=SplitAccessLevel(s.access))
-        for s in config.split_accesses
-    ]
     sidecar = EvaluationSidecar(
         engine=engine,
         split_accesses=split_accesses,
@@ -267,6 +276,7 @@ async def build_components(
         admin_volume=Path(config.admin_volume),
         submit_enabled=config.submit_enabled,
         base_commit=config.base_commit,
+        k_anonymity_floor=config.k_anonymity_floor,
     )
     verifier = Verifier(
         engine=engine,

vero/src/vero/harbor/server.py

@@ -38,6 +38,11 @@ class SubmitDisabledError(RuntimeError):
     """Raised when submit() is called but the task does not use submit selection."""
 
 
+class KAnonymityError(RuntimeError):
+    """Raised when an agent eval on a non_viewable split selects fewer samples
+    than the k-anonymity floor allows."""
+
+
 class EvaluationSidecar:
     """Agent-facing handlers over the EvaluationEngine.
 
@@ -56,6 +61,7 @@ def __init__(
         admin_volume: Path,
         submit_enabled: bool = False,
         base_commit: str | None = None,
+        k_anonymity_floor: int = 5,
     ):
         self.engine = engine
         self.split_accesses = split_accesses
@@ -64,13 +70,41 @@ def __init__(
         self.admin_volume = Path(admin_volume)
         self.submit_enabled = submit_enabled
         self.base_commit = base_commit
+        # Minimum sample count for an agent-chosen SUBSET eval of a non_viewable
+        # split. The aggregate response carries mean_score, so a singleton subset
+        # returns the sample's label-derived score verbatim, and n singleton
+        # evals reconstruct the split's per-sample labels wholesale. The floor
+        # applies only to proper subsets (sample_ids/num_samples): a full-split
+        # eval reveals exactly the intended aggregate, so it always passes and a
+        # split smaller than the floor degrades to full-split-only rather than
+        # becoming unevaluable. This is a cost multiplier, not a proof: means of
+        # k-sized overlapping subsets still admit reconstruction by elimination,
+        # but at >= k times the sample budget per label. <= 1 disables the floor.
+        self.k_anonymity_floor = k_anonymity_floor
         self._free_baseline_used = False
 
     # ------------------------------------------------------------------
     # Handlers (the HTTP layer resolves `admin` from auth and calls these)
     # ------------------------------------------------------------------
 
     async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> EvalSummary:
+        # k-anonymity floor, checked before any work (no commit transfer, no
+        # budget debit, no eval) so a rejected request costs the agent nothing.
+        # sample_ids is None exactly when the request covers the full split
+        # (resolve_samples collapses a covering num_samples to None too), and a
+        # full-split aggregate is the intended surface, so only proper subsets
+        # are floored.
+        if not admin and self.k_anonymity_floor > 1:
+            tier = tier_for_split(req.split, self.split_accesses)
+            if tier == SplitAccessLevel.non_viewable:
+                sample_ids, n = self.engine.resolve_samples(req)
+                if sample_ids is not None and n < self.k_anonymity_floor:
+                    raise KAnonymityError(
+                        f"Evals on non_viewable split '{req.split}' must cover "
+                        f"at least {self.k_anonymity_floor} samples (or the "
+                        f"whole split); got {n}. Aggregate scores over smaller "
+                        f"subsets would reveal per-sample results."
+                    )
         sha = await self._transfer_commit(req.commit)
         # The agent's FIRST eval of the seeded baseline is budget-free. The
         # baseline is the reference every candidate is implicitly compared to,
@@ -80,17 +114,23 @@ async def evaluate(self, req: EvalRequest, *, admin: bool = False) -> EvalSummar
         # an optimizer that skipped the reference could not tell a no-op edit
         # from an improvement and quit with budget unspent). Capped at one:
         # later baseline evals debit normally, so free compute is bounded.
+        # `free` waives only the budget debit; the eval still runs as the agent
+        # (tier gates apply), so the free baseline cannot touch no_access
+        # splits — riding the admin flag here did exactly that.
         free_baseline = (
             not admin
             and self.base_commit is not None
             and sha == self.base_commit
             and not self._free_baseline_used
         )
-        if free_baseline:
-            self._free_baseline_used = True
         exp = await self.engine.evaluate(
-            replace(req, commit=sha), admin=admin or free_baseline
+            replace(req, commit=sha), admin=admin, free=free_baseline
         )
+        # Consume the freebie only after the eval succeeded: an eval that
+        # raised (invalid split, infra failure) has given the agent nothing,
+        # so it must not burn the one free reference measurement.
+        if free_baseline:
+            self._free_baseline_used = True
         # Route with the agent's real tier even when the eval was unmetered.
         result_path = self._route_results(exp, admin=admin)
         budget_remaining = None
@@ -125,6 +165,7 @@ def status(self) -> StatusSummary:
             free_baseline_available=(
                 self.base_commit is not None and not self._free_baseline_used
             ),
+            k_anonymity_floor=self.k_anonymity_floor,
         )
 
     def list_experiments(self) -> list[dict]:

vero/tests/test_engine.py

@@ -162,3 +162,35 @@ async def test_non_viewable_split_still_evaluable(self, monkeypatch):
         )
         svc.evaluator.evaluate.assert_awaited_once()
         assert svc.status()[("dev", "ds1")].remaining_run_budget == 2
+
+
+class TestFreeEval:
+    """`free` waives only the budget debit; every access gate still applies.
+
+    free and admin are distinct authorities: the sidecar's free baseline eval
+    once rode the admin flag and could thereby evaluate no_access splits and
+    hand the agent their aggregate score."""
+
+    @pytest.mark.asyncio
+    async def test_free_runs_without_debiting_budget(self, monkeypatch):
+        svc = _make_service(monkeypatch=monkeypatch)
+        await svc.evaluate(
+            EvalRequest(dataset_id="ds1", split="dev", commit="c1", num_samples=10),
+            free=True,
+        )
+        svc.evaluator.evaluate.assert_awaited_once()
+        assert svc.status()[("dev", "ds1")].remaining_run_budget == 3
+        assert svc.status()[("dev", "ds1")].remaining_sample_budget == 100
+
+    @pytest.mark.asyncio
+    async def test_free_does_not_bypass_no_access_gate(self, monkeypatch):
+        from vero.core.dataset import SplitAccess
+
+        svc = _make_service(monkeypatch=monkeypatch)
+        svc.split_accesses = [SplitAccess.no_access("test")]
+        with pytest.raises(InvalidSplitError):
+            await svc.evaluate(
+                EvalRequest(dataset_id="ds1", split="test", commit="c1", num_samples=10),
+                free=True,
+            )
+        svc.evaluator.evaluate.assert_not_awaited()

vero/tests/test_harbor_protocol.py

@@ -91,3 +91,22 @@ def test_lists_budgeted_splits_with_tier(self):
         assert by_split["validation"]["tier"] == str(SplitAccessLevel.non_viewable)
         assert by_split["validation"]["agent_evaluable"] is True
         assert by_split["validation"]["remaining_run_budget"] == 3
+
+    def test_advertises_subset_floor_on_non_viewable_only(self):
+        budget = {
+            ("train", "ds1"): SplitBudget(split="train", dataset_id="ds1", total_run_budget=10),
+            ("validation", "ds1"): SplitBudget(split="validation", dataset_id="ds1", total_run_budget=3),
+        }
+        accesses = [
+            SplitAccess.viewable("train"),
+            SplitAccess.non_viewable("validation"),
+        ]
+        status = build_status(
+            submit_enabled=False,
+            budget=budget,
+            split_accesses=accesses,
+            k_anonymity_floor=5,
+        )
+        by_split = {s["split"]: s for s in status.splits}
+        assert by_split["validation"]["min_subset_samples"] == 5
+        assert by_split["train"]["min_subset_samples"] == 1

vero/tests/test_harbor_runner.py

@@ -99,6 +99,24 @@ def test_local_source(self, tmp_path):
         assert "-p" in cmd and str(tmp_path) in cmd
         assert "-d" not in cmd
 
+    def test_harbor_requirement_layers_trusted_cli(self):
+        # The overlay must come before the `harbor` executable name so uv
+        # resolves the CLI from the trusted spec, not the candidate's lockfile.
+        runner = HarborRunner(
+            HarborConfig(
+                task_source="org/ds@1",
+                agent_import_path="pkg.mod:Agent",
+                harbor_requirement="harbor==0.1.17",
+            )
+        )
+        cmd = runner._build_command("/wt", _params(), ["t0"], Path("/jobs"))
+        assert cmd[:6] == ["uv", "run", "--project", "/wt", "--with", "harbor==0.1.17"]
+        assert cmd[6] == "harbor"
+
+    def test_no_harbor_requirement_keeps_candidate_env(self):
+        cmd = _runner()._build_command("/wt", _params(), ["t0"], Path("/jobs"))
+        assert "--with" not in cmd
+
 
 class TestExtractReward:
     def test_priority_pass_then_reward_then_sole_key(self):