MVP: task and verification schemas for outcome-graded evals

deployers/base.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Dict, Any
 
+
 class Deployer(ABC):
     """
     Abstract base class for cloud-specific infra deployers.

deployers/factory.py

@@ -21,7 +21,9 @@ def __init__(self, cluster_name: str, project_id: str):
         self._project_id = project_id
 
     def up(self) -> None:
-        print("[NoOpDeployer] Skipping infrastructure provisioning (deployer: noop / BENCH_NO_INFRA)")
+        print(
+            "[NoOpDeployer] Skipping infrastructure provisioning (deployer: noop / BENCH_NO_INFRA)"
+        )
 
     def down(self) -> None:
         print("[NoOpDeployer] Skipping infrastructure teardown (deployer: noop / BENCH_NO_INFRA)")
@@ -39,7 +41,7 @@ def get_deployer(
     infra_config: Dict[str, Any],
     global_project_id: str,
     global_cluster_name: str,
-    global_location: str = None
+    global_location: str = None,
 ) -> Deployer:
     """
     Factory to instantiate the appropriate infrastructure deployer.
@@ -82,7 +84,5 @@ def get_deployer(
 
     # Fallback to legacy GCPDeployer (kubetest2)
     return GCPDeployer(
-        project=global_project_id,
-        location=location,
-        cluster_name=global_cluster_name
+        project=global_project_id, location=location, cluster_name=global_cluster_name
     )

deployers/gcp/gcp_deployer.py

@@ -4,27 +4,36 @@
 from typing import Dict, Any
 from deployers.base import Deployer
 
+
 class GCPDeployer(Deployer):
     """
     GCP implementation of the Deployer interface using kubetest2 gke.
-    
+
     Migrated to pathlib for better cross-platform compatibility and readability.
     Supports both 'location' and 'zone' in initialization for robust API compatibility.
     """
-    def __init__(self, project: str, location: str = None, cluster_name: str = None, zone: str = None, **config):
+
+    def __init__(
+        self,
+        project: str,
+        location: str = None,
+        cluster_name: str = None,
+        zone: str = None,
+        **config,
+    ):
         self.project = project
         self.cluster_name = cluster_name
         self.config = config
-        
+
         # Robust location resolution for backward compatibility
         self.location = location or zone or os.environ.get("GCP_LOCATION", "us-central1-a")
-        self.zone = self.location 
-        
+        self.zone = self.location
+
         # This file is at deployers/gcp/gcp_deployer.py
         # Project root is 2 levels up.
         current_dir = Path(__file__).resolve().parent
         self.bin_dir = str((current_dir.parents[1] / "third_party" / "kubetest2" / "bin").resolve())
-        
+
     def _get_env(self) -> Dict[str, str]:
         env = os.environ.copy()
         # Ensure kubetest2 and its plugins are in PATH
@@ -44,8 +53,15 @@ def _state_marker_path(self) -> Path:
     def up(self) -> None:
         # Check if cluster exists
         check_cmd = [
-            "gcloud", "container", "clusters", "describe", self.cluster_name,
-            "--project", self.project, "--location", self.location
+            "gcloud",
+            "container",
+            "clusters",
+            "describe",
+            self.cluster_name,
+            "--project",
+            self.project,
+            "--location",
+            self.location,
         ]
         print(f"Checking if cluster exists: {' '.join(check_cmd)}")
         result = subprocess.run(check_cmd, capture_output=True, text=True)
@@ -56,28 +72,36 @@ def up(self) -> None:
         if result.returncode == 0:
             print(f"Cluster {self.cluster_name} already exists. Getting credentials.")
             get_cred_cmd = [
-                "gcloud", "container", "clusters", "get-credentials", self.cluster_name, 
-                "--project", self.project, "--location", self.location
+                "gcloud",
+                "container",
+                "clusters",
+                "get-credentials",
+                self.cluster_name,
+                "--project",
+                self.project,
+                "--location",
+                self.location,
             ]
             subprocess.run(get_cred_cmd, check=True)
             # Record that we didn't create it
             state_file.write_text("false")
         else:
-            print(
-                f"Cluster {self.cluster_name} does not exist or error checking. "
-                "Creating it."
-            )
+            print(f"Cluster {self.cluster_name} does not exist or error checking. Creating it.")
             cmd = [
-                "kubetest2", "gke",
-                "--project", self.project,
-                "--zone", self.location, # Passing resolved location to --zone flag in kubetest2
-                "--cluster-name", self.cluster_name,
+                "kubetest2",
+                "gke",
+                "--project",
+                self.project,
+                "--zone",
+                self.location,  # Passing resolved location to --zone flag in kubetest2
+                "--cluster-name",
+                self.cluster_name,
             ]
             for key, value in self.config.items():
                 if value is not None:
                     flag_name = f"--{key.replace('_', '-')}"
                     cmd.extend([flag_name, str(value)])
-                
+
             cmd.append("--up")
             print(f"Running: {' '.join(cmd)}")
             subprocess.run(cmd, env=self._get_env(), check=True)
@@ -89,31 +113,31 @@ def down(self) -> None:
         created_by_us = True
         if state_file.exists():
             created_by_us = state_file.read_text().strip() == "true"
-                
+
         if not created_by_us:
-            print(
-                f"Skipping teardown for pre-existing cluster {self.cluster_name}"
-            )
+            print(f"Skipping teardown for pre-existing cluster {self.cluster_name}")
             return
-            
+
         cmd = [
-            "kubetest2", "gke",
-            "--project", self.project,
-            "--zone", self.location,
-            "--cluster-name", self.cluster_name,
-            "--down"
+            "kubetest2",
+            "gke",
+            "--project",
+            self.project,
+            "--zone",
+            self.location,
+            "--cluster-name",
+            self.cluster_name,
+            "--down",
         ]
         print(f"Running: {' '.join(cmd)}")
         subprocess.run(cmd, env=self._get_env(), check=True)
 
     def get_cluster_info(self) -> Dict[str, Any]:
-        kubeconfig_path = os.environ.get(
-            "KUBECONFIG", str(Path.home() / ".kube" / "config")
-        )
+        kubeconfig_path = os.environ.get("KUBECONFIG", str(Path.home() / ".kube" / "config"))
         return {
             "name": self.cluster_name,
             "location": self.location,
-            "zone": self.location, 
+            "zone": self.location,
             "project": self.project,
-            "kubeconfig_path": kubeconfig_path
+            "kubeconfig_path": kubeconfig_path,
         }

deployers/gcp/variables.py

@@ -1,12 +1,13 @@
 import os
 from typing import Any, Dict
 
+
 def resolve_variables(
     stack: str,
     custom_variables: Dict[str, Any],
     global_project_id: str,
     global_cluster_name: str,
-    global_location: str
+    global_location: str,
 ) -> Dict[str, Any]:
     """Resolves default variables for GCP-based TF stacks."""
     variables = custom_variables.copy()

deployers/kind/variables.py

@@ -2,12 +2,13 @@
 import pathlib
 from typing import Any, Dict
 
+
 def resolve_variables(
     stack: str,
     custom_variables: Dict[str, Any],
     global_project_id: str,
     global_cluster_name: str,
-    global_location: str
+    global_location: str,
 ) -> Dict[str, Any]:
     """Resolves default variables for local KinD-based stacks."""
     variables = custom_variables.copy()

deployers/tf/tf_deployer.py

@@ -44,6 +44,7 @@ class TFDeployer(Deployer):
     Supports the standard TF_DATA_DIR environment variable for controlling
     where OpenTofu stores its state, enabling idempotent runs.
     """
+
     def __init__(self, tf_dir: str, variables: Dict[str, Any] = None):
         # Locate project root (3 levels up from this file)
         repo_root = Path(__file__).resolve().parents[2]
@@ -59,10 +60,7 @@ def __init__(self, tf_dir: str, variables: Dict[str, Any] = None):
             if repo_tf_path.exists():
                 self.tf_dir = str(repo_tf_path)
             else:
-                raise ValueError(
-                    f"TF stack not found in repo: {tf_dir} "
-                    f"(checked {repo_tf_path})"
-                )
+                raise ValueError(f"TF stack not found in repo: {tf_dir} (checked {repo_tf_path})")
 
         # Per-run private working directory (a copy of the tf/ tree under the
         # run's scratch dir) when isolated; otherwise the shared stack dir.
@@ -87,18 +85,14 @@ def _state_flags() -> list:
             return []
         return ["-state", str(Path(tf_data_dir).resolve().parent / "terraform.tfstate")]
 
-    def _run_cmd(
-        self, cmd: list, cwd: str, capture: bool = False
-    ) -> subprocess.CompletedProcess:
+    def _run_cmd(self, cmd: list, cwd: str, capture: bool = False) -> subprocess.CompletedProcess:
         print(f"Executing: {' '.join(cmd)} in {cwd}")
         env = os.environ.copy()
         if "TF_DATA_DIR" in env:
-             print(f"Using TF_DATA_DIR: {env['TF_DATA_DIR']}")
+            print(f"Using TF_DATA_DIR: {env['TF_DATA_DIR']}")
 
         if capture:
-            return subprocess.run(
-                cmd, cwd=cwd, check=True, capture_output=True, text=True, env=env
-            )
+            return subprocess.run(cmd, cwd=cwd, check=True, capture_output=True, text=True, env=env)
         else:
             return subprocess.run(cmd, cwd=cwd, check=True, env=env)
 
@@ -115,14 +109,10 @@ def up(self) -> None:
 
         self._run_cmd(cmd, cwd=self.work_dir)
 
-
     def down(self) -> None:
         tf_path = Path(self.work_dir)
         if not tf_path.exists():
-            print(
-                f"Warning: TF directory {self.work_dir} not found. "
-                "Skipping teardown."
-            )
+            print(f"Warning: TF directory {self.work_dir} not found. Skipping teardown.")
             return
 
         self._run_cmd(["tofu", "init", "-input=false"], cwd=self.work_dir)
@@ -149,37 +139,44 @@ def get_cluster_info(self) -> Dict[str, Any]:
 
         location = outputs.get("cluster_location", {}).get("value")
         if not location:
-            raise ValueError(
-                "Failed to retrieve 'cluster_location' from TF outputs."
-            )
+            raise ValueError("Failed to retrieve 'cluster_location' from TF outputs.")
 
-        kubeconfig_path = os.environ.get(
-            "KUBECONFIG", str(Path.home() / ".kube" / "config")
-        )
+        kubeconfig_path = os.environ.get("KUBECONFIG", str(Path.home() / ".kube" / "config"))
 
         if location == "local":
-            project = self.variables.get("project_id") or os.environ.get("GCP_PROJECT_ID") or "local-kind"
+            project = (
+                self.variables.get("project_id") or os.environ.get("GCP_PROJECT_ID") or "local-kind"
+            )
             return {
                 "name": cluster_name,
                 "location": location,
                 "project": project,
-                "kubeconfig_path": kubeconfig_path
+                "kubeconfig_path": kubeconfig_path,
             }
 
         project = self.variables.get("project_id") or os.environ.get("GCP_PROJECT_ID")
         if not project:
-             raise ValueError("Project ID not found in variables or environment (GCP_PROJECT_ID).")
+            raise ValueError("Project ID not found in variables or environment (GCP_PROJECT_ID).")
 
         print(f"Configuring kubectl for cluster: {cluster_name} in {location}...")
-        subprocess.run([
-            "gcloud", "container", "clusters", "get-credentials", cluster_name,
-            "--location", location, "--project", project
-        ], check=True)
+        subprocess.run(
+            [
+                "gcloud",
+                "container",
+                "clusters",
+                "get-credentials",
+                cluster_name,
+                "--location",
+                location,
+                "--project",
+                project,
+            ],
+            check=True,
+        )
 
         return {
             "name": cluster_name,
             "location": location,
             "project": project,
-            "kubeconfig_path": kubeconfig_path
+            "kubeconfig_path": kubeconfig_path,
         }
-