feat: [no-ticket] harbor - transfer targets (per-target executor-model override at finalize)

vero/src/vero/evaluation/engine.py

@@ -193,22 +193,39 @@ async def evaluate_admin(
         split: str,
         commit: str,
         sample_ids: list[int] | None = None,
+        model: str | None = None,
     ) -> Experiment:
         """Admin/verifier evaluation: explicit ``task``, no budget, no allowlist.
 
         Unlike :meth:`evaluate` (which is bound to ``default_task`` and metered),
         this scores an arbitrary ``(task, dataset_id, split)`` — including held-out
         tasks/splits the agent never had access to. Used by the verifier to score
         the selected commit on its configured targets.
+
+        ``model`` overrides the executor model for this one eval (rides
+        ``task_params`` so the eval strategy can honor it; the Mode-B
+        HarborRunner does, the Mode-A vero-task path ignores it). Used for
+        transfer targets: scoring the champion under a model it was NOT
+        optimized on.
         """
+        params = self.run_constraints
+        if model is not None:
+            params = params.model_copy(
+                update={
+                    "task_params": {
+                        **(params.task_params or {}),
+                        "harbor_model_override": model,
+                    }
+                }
+            )
         return await self.evaluator.evaluate(
             commit=commit,
             dataset_id=dataset_id,
             split=split,
             task=task,
             sample_ids=sample_ids,
             db=self.db,
-            evaluation_parameters=self.run_constraints,
+            evaluation_parameters=params,
         )
 
     def status(self) -> dict[tuple[str, str], SplitBudget]:

vero/src/vero/harbor/build/compiler.py

@@ -236,6 +236,7 @@ def _serve_config(
                 "split": t.split,
                 "reward_key": t.reward_key,
                 "sample_ids": t.sample_ids,
+                "model": t.model,
             }
             for t in config.targets
         ],

vero/src/vero/harbor/build/config.py

@@ -36,6 +36,12 @@ class TargetSpec(BaseModel):
     split: str
     reward_key: str = "reward"
     sample_ids: list[int] | None = None
+    # Executor-model override for this target (transfer probe; Mode B only):
+    # score the selected commit AND the baseline under a model it was not
+    # optimized on, so model-specific couplings (measured live: hardcoded
+    # temperature=0 crashing 72/72 on an executor that rejects it) surface at
+    # finalize instead of one substrate away.
+    model: str | None = None
 
 
 class _BuildConfigBase(BaseModel):

vero/src/vero/harbor/runner.py

@@ -129,8 +129,12 @@ def _build_command(
             "--n-attempts", str(c.n_attempts),
             "--max-retries", str(c.max_retries),
         ]
-        if c.model:
-            cmd += ["-m", c.model]
+        # Per-eval executor override (transfer targets): the verifier scores
+        # the champion under a model it was not optimized on. Rides
+        # task_params so it needs no runner state.
+        model = (params.task_params or {}).get("harbor_model_override") or c.model
+        if model:
+            cmd += ["-m", str(model)]
         for task_name in task_names:
             cmd += ["-i", task_name]
         cmd += ["--jobs-dir", str(jobs_dir), *c.extra_args]

vero/src/vero/harbor/serve.py

@@ -43,6 +43,8 @@ class _TargetCfg(BaseModel):
     split: str
     reward_key: str = "reward"
     sample_ids: list[int] | None = None
+    # Executor-model override for this target (transfer probe; Mode B only).
+    model: str | None = None
 
 
 class _ServeConfigBase(BaseModel):

vero/src/vero/harbor/verifier.py

@@ -39,6 +39,15 @@ class VerificationTarget:
     split: str
     reward_key: str
     sample_ids: list[int] | None = None  # None = full split
+    # Executor-model override for this target (Mode B): score the selected
+    # commit under a DIFFERENT model than the one it was optimized on. This is
+    # the transfer probe: home-model evals cannot see model-specific couplings
+    # the optimizer bakes in (measured live: three champions independently
+    # hardcoded temperature=0 and scored 0/72 on an executor that rejects it,
+    # while looking healthy on every home-model eval). None = the task's
+    # configured model. The baseline is scored under the same override, so the
+    # comparison stays like-for-like.
+    model: str | None = None
 
 
 class Verifier:
@@ -144,6 +153,7 @@ async def _finalize(self) -> dict:
                 split=target.split,
                 commit=sha,
                 sample_ids=target.sample_ids,
+                model=target.model,
                 what=f"target '{target.reward_key}'",
             )
             if score is None:
@@ -177,6 +187,7 @@ async def _admin_eval_score(
         split: str,
         commit: str,
         sample_ids: list[int] | None = None,
+        model: str | None = None,
         what: str,
     ) -> tuple[float | None, str | None]:
         """One reward-critical admin eval with bounded retry.
@@ -201,6 +212,7 @@ async def _admin_eval_score(
                     split=split,
                     commit=commit,
                     sample_ids=sample_ids,
+                    model=model,
                 )
                 if exp.result.score(fill_score=None) is None:
                     last_error = (
@@ -293,12 +305,15 @@ async def _maybe_score_baseline(self, rewards: dict[str, float]) -> dict:
             try:
                 baselines: dict[str, float] = {}
                 for target in self.targets:
+                    # Same executor override as the candidate's target eval, or
+                    # the baseline comparison is not like-for-like.
                     exp = await self.engine.evaluate_admin(
                         task=target.task,
                         dataset_id=target.dataset_id,
                         split=target.split,
                         commit=self.base_commit,
                         sample_ids=target.sample_ids,
+                        model=target.model,
                     )
                     if exp.result.score(fill_score=None) is None:
                         # All-error/empty is an outage, not a 0.0 baseline: a

vero/tests/test_engine.py

@@ -182,6 +182,25 @@ async def test_free_runs_without_debiting_budget(self, monkeypatch):
         assert svc.status()[("dev", "ds1")].remaining_run_budget == 3
         assert svc.status()[("dev", "ds1")].remaining_sample_budget == 100
 
+    @pytest.mark.asyncio
+    async def test_admin_model_override_rides_task_params(self, monkeypatch):
+        # Transfer targets: evaluate_admin(model=...) must reach the eval
+        # strategy via task_params without mutating the shared run_constraints.
+        svc = _make_service(monkeypatch=monkeypatch)
+        await svc.evaluate_admin(
+            task="t", dataset_id="ds1", split="dev", commit="c1", model="openai/gpt-4o"
+        )
+        params = svc.evaluator.evaluate.await_args.kwargs["evaluation_parameters"]
+        assert params.task_params["harbor_model_override"] == "openai/gpt-4o"
+        assert svc.run_constraints.task_params.get("harbor_model_override") is None
+
+    @pytest.mark.asyncio
+    async def test_admin_no_override_keeps_shared_constraints(self, monkeypatch):
+        svc = _make_service(monkeypatch=monkeypatch)
+        await svc.evaluate_admin(task="t", dataset_id="ds1", split="dev", commit="c1")
+        params = svc.evaluator.evaluate.await_args.kwargs["evaluation_parameters"]
+        assert params is svc.run_constraints
+
     @pytest.mark.asyncio
     async def test_free_does_not_bypass_no_access_gate(self, monkeypatch):
         from vero.core.dataset import SplitAccess

vero/tests/test_harbor_build.py

@@ -363,6 +363,34 @@ def test_instruction_omits_free_baseline_claim_when_unsupported(built):
     assert "budget-free" not in text
 
 
+def test_target_model_reaches_serve_json(tmp_path, monkeypatch):
+    # Transfer probe: a target's executor-model override must survive the
+    # compile into serve.json and validate back through ServeConfig.
+    monkeypatch.setenv("VERO_SKIP_SECRET_CHECK", "1")
+    config = BuildConfigA(
+        name="vero/gsm8k-opt",
+        agent_repo=str(_agent_repo(tmp_path)),
+        task="gsm8k",
+        task_module="gsm8k_agent.vero_tasks",
+        dataset=str(_dataset(tmp_path)),
+        splits=[
+            {"split": "validation", "access": "non_viewable"},
+            {"split": "test", "access": "no_access"},
+        ],
+        budgets=[{"split": "validation", "total_run_budget": 5}],
+        selection_split="validation",
+        targets=[
+            {"split": "test", "reward_key": "reward"},
+            {"split": "test", "reward_key": "reward_4o", "model": "openai/gpt-4o"},
+        ],
+    )
+    out = compile_task(config, tmp_path / "task", vero_root=_stub_vero(tmp_path))
+    cfg = load_serve_config(out / "environment" / "sidecar" / "serve.json")
+    by_key = {t.reward_key: t for t in cfg.targets}
+    assert by_key["reward_4o"].model == "openai/gpt-4o"
+    assert by_key["reward"].model is None
+
+
 def test_instruction_renders_exhaust_budget_by_default(built):
     text = (built / "instruction.md").read_text()
     assert "Unspent budget is wasted" in text

vero/tests/test_harbor_runner.py

@@ -117,6 +117,19 @@ def test_no_harbor_requirement_keeps_candidate_env(self):
         cmd = _runner()._build_command("/wt", _params(), ["t0"], Path("/jobs"))
         assert "--with" not in cmd
 
+    def test_model_override_beats_configured_model(self):
+        # Transfer targets: a per-eval executor override (via task_params)
+        # wins over the task's configured model.
+        params = _params()
+        params.task_params = {"harbor_model_override": "openai/gpt-4o"}
+        cmd = _runner()._build_command("/wt", params, ["t0"], Path("/jobs"))
+        m = cmd[cmd.index("-m") + 1]
+        assert m == "openai/gpt-4o"
+
+    def test_no_override_uses_configured_model(self):
+        cmd = _runner()._build_command("/wt", _params(), ["t0"], Path("/jobs"))
+        assert cmd[cmd.index("-m") + 1] == "anthropic/x"
+
 
 class TestExtractReward:
     def test_priority_pass_then_reward_then_sole_key(self):

vero/tests/test_harbor_verifier.py

@@ -89,7 +89,7 @@ async def test_auto_best_reranks_by_admin_score(self, tmp_path):
         )
         admin_scores = {"hi": 0.1, "lo": 0.95}
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             return MagicMock(
                 result=MagicMock(score=MagicMock(return_value=admin_scores.get(commit, 0.99)))
             )
@@ -125,7 +125,7 @@ async def test_auto_best_excludes_baseline_from_ranking(self, tmp_path):
             }
         )
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             return MagicMock(result=MagicMock(score=MagicMock(return_value=0.7)))
 
         engine.evaluate_admin = AsyncMock(side_effect=_admin)
@@ -172,7 +172,7 @@ async def test_lucky_subset_eval_does_not_outrank_full_split(self, tmp_path):
             }
         )
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             # admin re-score agrees the full-split ranking is right
             score = {"solid": 0.7, "lucky": 0.3}.get(commit, 0.5)
             return MagicMock(result=MagicMock(score=MagicMock(return_value=score)))
@@ -212,7 +212,7 @@ async def test_all_subset_evals_still_rankable(self, tmp_path):
             }
         )
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             return MagicMock(result=MagicMock(score=MagicMock(return_value=0.5)))
 
         engine.evaluate_admin = AsyncMock(side_effect=_admin)
@@ -260,7 +260,7 @@ async def test_reverts_to_base_when_no_candidate_beats_baseline(self, tmp_path):
         # agent admin-scores 0.2 on the selection split; base admin-scores 0.3;
         # the reverted base scores 0.35 on the target split (distinct values so the
         # assertions can tell the target eval apart from the floor comparison).
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             if commit == "base":
                 score = 0.35 if split == "validation" else 0.3
             else:
@@ -296,7 +296,7 @@ async def test_exact_tie_reverts_to_base(self, tmp_path):
         engine = MagicMock()
         engine.db.get_experiments_df.return_value = self._df()
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             return MagicMock(result=MagicMock(score=MagicMock(return_value=0.3)))  # all equal
 
         engine.evaluate_admin = AsyncMock(side_effect=_admin)
@@ -327,7 +327,7 @@ async def test_floor_noop_without_base_commit(self, tmp_path):
             }
         )
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             return MagicMock(result=MagicMock(score=MagicMock(return_value=0.5)))
 
         engine.evaluate_admin = AsyncMock(side_effect=_admin)
@@ -349,7 +349,7 @@ async def test_keeps_candidate_that_beats_baseline(self, tmp_path):
         engine = MagicMock()
         engine.db.get_experiments_df.return_value = self._df()
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             score = 0.3 if commit == "base" else 0.6  # agent genuinely improves
             return MagicMock(result=MagicMock(score=MagicMock(return_value=score)))
 
@@ -374,7 +374,7 @@ async def test_floor_off_ships_least_bad_candidate(self, tmp_path):
         engine = MagicMock()
         engine.db.get_experiments_df.return_value = self._df()
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             return MagicMock(result=MagicMock(score=MagicMock(return_value=0.2)))
 
         engine.evaluate_admin = AsyncMock(side_effect=_admin)
@@ -476,7 +476,7 @@ async def test_candidates_present_keeps_normal_selection(self, tmp_path):
             }
         )
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             return MagicMock(result=MagicMock(score=MagicMock(return_value=0.5)))
 
         engine.evaluate_admin = AsyncMock(side_effect=_admin)
@@ -733,7 +733,7 @@ async def test_floor_fail_safe_reverts_on_unmeasurable_baseline(self, tmp_path):
             }
         )
 
-        async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
+        async def _admin(*, task, dataset_id, split, commit, sample_ids=None, model=None):
             if commit == "base" and split == "validation":
                 raise RuntimeError("nested run crashed")
             return MagicMock(result=MagicMock(score=MagicMock(return_value=0.9)))
@@ -755,6 +755,32 @@ async def _admin(*, task, dataset_id, split, commit, sample_ids=None):
         assert engine.evaluate_admin.await_args.kwargs["commit"] == "base"
         assert engine.evaluate_admin.await_args.kwargs["split"] == "test"
 
+    @pytest.mark.asyncio
+    async def test_transfer_target_model_reaches_champion_and_baseline_evals(self, tmp_path):
+        # A target's executor-model override must apply to BOTH the champion
+        # eval and the baseline eval, or the comparison is not like-for-like.
+        self._submit(tmp_path)
+        engine = MagicMock()
+        engine.evaluate_admin = AsyncMock(
+            return_value=MagicMock(result=MagicMock(score=MagicMock(return_value=0.5)))
+        )
+        v = Verifier(
+            engine=engine,
+            admin_volume=tmp_path,
+            reward_mode="submit",
+            score_baseline=True,
+            base_commit="base",
+            targets=[VerificationTarget(
+                task="t", dataset_id="ds", split="test",
+                reward_key="reward_4o", model="openai/gpt-4o",
+            )],
+        )
+        await v.finalize()
+        models = [c.kwargs["model"] for c in engine.evaluate_admin.await_args_list]
+        commits = [c.kwargs["commit"] for c in engine.evaluate_admin.await_args_list]
+        assert models == ["openai/gpt-4o", "openai/gpt-4o"]
+        assert set(commits) == {"cand", "base"}
+
     @pytest.mark.asyncio
     async def test_floored_target_records_dominant_crash_cause(self, tmp_path):
         # An all-errored target eval floors the reward AND names the dominant