From 3322663fbdc171963e6c1cb5bfbf9d4591e41a45 Mon Sep 17 00:00:00 2001 From: Jegath S Date: Wed, 15 Jul 2026 15:56:08 +0530 Subject: [PATCH 1/4] feat: add deterministic verifiers (resource_exists, resource_field) to replace LLM judge for fix-config and deploy-config --- pkg/agents/verifier/resource_exists.py | 57 +++++++ pkg/agents/verifier/resource_field.py | 86 ++++++++++ pkg/agents/verifier/spec.py | 9 +- pkg/agents/verifier/test_verifier.py | 149 ++++++++++++++++++ pkg/evaluator/evaluate.py | 208 +++++++++++++++++++++---- tasks/gcp/deploy-config/task.yaml | 39 ++++- tasks/gcp/fix-config/task.yaml | 20 ++- 7 files changed, 536 insertions(+), 32 deletions(-) create mode 100644 pkg/agents/verifier/resource_exists.py create mode 100644 pkg/agents/verifier/resource_field.py diff --git a/pkg/agents/verifier/resource_exists.py b/pkg/agents/verifier/resource_exists.py new file mode 100644 index 00000000..69ad17cd --- /dev/null +++ b/pkg/agents/verifier/resource_exists.py @@ -0,0 +1,57 @@ +import subprocess +import time +from typing import Literal, Optional +from pkg.agents.verifier.base import BaseVerifier, VerificationResult + + +class ResourceExistsVerifier(BaseVerifier): + """Verifies that a named Kubernetes resource exists in the cluster. + + Generic primitive over `kubectl get `. Because it reads live + cluster state, a successful check also proves the resource was actually + applied (not merely printed by the agent). + """ + + type: Literal["resource_exists"] = "resource_exists" + kind: str + name: str + namespace: Optional[str] = None + + def verify(self, timeout_sec: int) -> VerificationResult: + start_time = time.time() + delay = 1 + max_delay = 10 + + while True: + success, details = self._check_exists() + if success: + return VerificationResult( + success=True, + elapsed_time=time.time() - start_time, + reason=details.get("reason"), + details=details, + ) + if time.time() - start_time >= timeout_sec: + return VerificationResult( + success=False, + elapsed_time=time.time() - start_time, + reason=details.get("reason"), + details=details, + ) + time.sleep(delay) + delay = min(delay * 2, max_delay) + + def _check_exists(self) -> (bool, dict): + cmd = ["kubectl", "get", self.kind, self.name] + if self.namespace: + cmd.extend(["-n", self.namespace]) + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + return True, { + "reason": f"{self.kind}/{self.name} exists", + "output": result.stdout.strip(), + } + except subprocess.CalledProcessError as e: + return False, { + "reason": f"{self.kind}/{self.name} not found: {e.stderr.strip()}" + } diff --git a/pkg/agents/verifier/resource_field.py b/pkg/agents/verifier/resource_field.py new file mode 100644 index 00000000..5d4bef48 --- /dev/null +++ b/pkg/agents/verifier/resource_field.py @@ -0,0 +1,86 @@ +import subprocess +import time +from typing import Literal, Optional +from pkg.agents.verifier.base import BaseVerifier, VerificationResult + + +class ResourceFieldVerifier(BaseVerifier): + """Verifies that a specific field on a Kubernetes resource equals an + expected value. + + Generic primitive over `kubectl get -o jsonpath=`. + kubectl performs the extraction, so no extra Python dependency is required. + Reading the live field value also proves the resource was actually applied + with the correct configuration. + + Example (container env var): + json_path = "{.spec.template.spec.containers[?(@.name=='frontend')]" + ".env[?(@.name=='USE_GEMINI_API')].value}" + expected = "false" + """ + + type: Literal["resource_field"] = "resource_field" + kind: str + name: str + json_path: str + expected: str + namespace: Optional[str] = None + + def verify(self, timeout_sec: int) -> VerificationResult: + start_time = time.time() + delay = 1 + max_delay = 10 + + while True: + success, details = self._check_field() + if success: + return VerificationResult( + success=True, + elapsed_time=time.time() - start_time, + reason=details.get("reason"), + details=details, + ) + if time.time() - start_time >= timeout_sec: + return VerificationResult( + success=False, + elapsed_time=time.time() - start_time, + reason=details.get("reason"), + details=details, + ) + time.sleep(delay) + delay = min(delay * 2, max_delay) + + def _check_field(self) -> (bool, dict): + cmd = [ + "kubectl", + "get", + self.kind, + self.name, + "-o", + f"jsonpath={self.json_path}", + ] + if self.namespace: + cmd.extend(["-n", self.namespace]) + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + return False, { + "reason": f"Failed to read {self.kind}/{self.name}: {e.stderr.strip()}" + } + + actual = result.stdout.strip() + success = actual == self.expected + reason = ( + f"{self.kind}/{self.name} field matched expected value '{self.expected}'" + if success + else ( + f"{self.kind}/{self.name} field mismatch: expected " + f"'{self.expected}', got '{actual}'" + ) + ) + return success, { + "reason": reason, + "json_path": self.json_path, + "expected": self.expected, + "actual": actual, + } diff --git a/pkg/agents/verifier/spec.py b/pkg/agents/verifier/spec.py index 3591b0cf..03db46d2 100644 --- a/pkg/agents/verifier/spec.py +++ b/pkg/agents/verifier/spec.py @@ -2,9 +2,16 @@ from pydantic import RootModel from pkg.agents.verifier.pod_healthy import PodHealthyVerifier from pkg.agents.verifier.scaling_complete import ScalingCompleteVerifier +from pkg.agents.verifier.resource_exists import ResourceExistsVerifier +from pkg.agents.verifier.resource_field import ResourceFieldVerifier # SingleVerificationSpec is a discriminated union of all supported checker types -SingleVerificationSpec = Union[PodHealthyVerifier, ScalingCompleteVerifier] +SingleVerificationSpec = Union[ + PodHealthyVerifier, + ScalingCompleteVerifier, + ResourceExistsVerifier, + ResourceFieldVerifier, +] # Top-level VerificationSpec which can parse a dict, a list, or a single checker spec. class VerificationSpec(RootModel[Union[Dict[str, SingleVerificationSpec], List[SingleVerificationSpec], SingleVerificationSpec]]): diff --git a/pkg/agents/verifier/test_verifier.py b/pkg/agents/verifier/test_verifier.py index e04f4979..194da594 100644 --- a/pkg/agents/verifier/test_verifier.py +++ b/pkg/agents/verifier/test_verifier.py @@ -6,6 +6,8 @@ from pkg.agents.verifier.verifier import VerifierAgent from pkg.agents.verifier.pod_healthy import PodHealthyVerifier from pkg.agents.verifier.scaling_complete import ScalingCompleteVerifier +from pkg.agents.verifier.resource_exists import ResourceExistsVerifier +from pkg.agents.verifier.resource_field import ResourceFieldVerifier class TestVerifierAgent(unittest.TestCase): @@ -110,6 +112,153 @@ def test_scaling_complete_verifier_verify_polling_success(self, mock_sleep, mock self.assertEqual(result.reason, "Scaling complete: done") self.assertEqual(mock_sleep.call_count, 1) + # --- ResourceExistsVerifier --- + + @patch("subprocess.run") + def test_resource_exists_verifier_success(self, mock_run): + mock_run.return_value = MagicMock( + stdout="deployment.apps/my-dep", returncode=0 + ) + + r_verifier = ResourceExistsVerifier( + kind="deployment", name="my-dep", namespace="default" + ) + result = r_verifier.verify(timeout_sec=5) + + self.assertTrue(result.success) + self.assertIn("exists", result.reason) + + @patch("time.sleep") + @patch("subprocess.run") + def test_resource_exists_verifier_not_found(self, mock_run, mock_sleep): + mock_run.side_effect = subprocess.CalledProcessError( + 1, "kubectl get", stderr='Error from server (NotFound)' + ) + + r_verifier = ResourceExistsVerifier( + kind="deployment", name="does-not-exist", namespace="default" + ) + result = r_verifier.verify(timeout_sec=1) + + self.assertFalse(result.success) + self.assertIn("not found", result.reason) + + @patch("subprocess.run") + def test_resource_exists_verifier_omits_namespace_when_none(self, mock_run): + mock_run.return_value = MagicMock(stdout="namespace/foo", returncode=0) + + r_verifier = ResourceExistsVerifier(kind="namespace", name="foo") + result = r_verifier.verify(timeout_sec=5) + + self.assertTrue(result.success) + called_cmd = mock_run.call_args[0][0] + self.assertNotIn("-n", called_cmd) + + # --- ResourceFieldVerifier --- + + @patch("subprocess.run") + def test_resource_field_verifier_match(self, mock_run): + mock_run.return_value = MagicMock(stdout="false", returncode=0) + + f_verifier = ResourceFieldVerifier( + kind="deployment", + name="hypercomputer-d1-frontend", + namespace="default", + json_path="{.spec.template.spec.containers[?(@.name=='frontend')].env[?(@.name=='USE_GEMINI_API')].value}", + expected="false", + ) + result = f_verifier.verify(timeout_sec=5) + + self.assertTrue(result.success) + self.assertIn("matched expected value", result.reason) + self.assertEqual(result.details["actual"], "false") + + @patch("time.sleep") + @patch("subprocess.run") + def test_resource_field_verifier_mismatch(self, mock_run, mock_sleep): + mock_run.return_value = MagicMock(stdout="true", returncode=0) + + f_verifier = ResourceFieldVerifier( + kind="deployment", + name="hypercomputer-d1-frontend", + namespace="default", + json_path="{.spec.template.spec.containers[?(@.name=='frontend')].env[?(@.name=='USE_GEMINI_API')].value}", + expected="false", + ) + result = f_verifier.verify(timeout_sec=1) + + self.assertFalse(result.success) + self.assertIn("mismatch", result.reason) + self.assertEqual(result.details["expected"], "false") + self.assertEqual(result.details["actual"], "true") + + @patch("time.sleep") + @patch("subprocess.run") + def test_resource_field_verifier_get_fails(self, mock_run, mock_sleep): + mock_run.side_effect = subprocess.CalledProcessError( + 1, "kubectl get", stderr="Error from server (NotFound)" + ) + + f_verifier = ResourceFieldVerifier( + kind="deployment", + name="missing", + namespace="default", + json_path="{.spec.replicas}", + expected="1", + ) + result = f_verifier.verify(timeout_sec=1) + + self.assertFalse(result.success) + self.assertIn("Failed to read", result.reason) + + @patch("subprocess.run") + def test_resource_field_verifier_uses_jsonpath_output(self, mock_run): + mock_run.return_value = MagicMock(stdout="1", returncode=0) + + f_verifier = ResourceFieldVerifier( + kind="hpa", + name="my-hpa", + namespace="default", + json_path="{.spec.minReplicas}", + expected="1", + ) + result = f_verifier.verify(timeout_sec=5) + + self.assertTrue(result.success) + called_cmd = mock_run.call_args[0][0] + self.assertIn("jsonpath={.spec.minReplicas}", called_cmd) + + # --- Compound spec that mixes the new verifiers --- + + @patch("subprocess.run") + def test_wait_for_condition_compound_new_verifiers(self, mock_run): + def run_side_effect(cmd, *args, **kwargs): + if "jsonpath" in " ".join(cmd): + return MagicMock(stdout="false", returncode=0) + return MagicMock(stdout="service/my-svc", returncode=0) + + mock_run.side_effect = run_side_effect + + spec = { + "cfg": { + "type": "resource_field", + "kind": "deployment", + "name": "my-dep", + "json_path": "{.spec.replicas}", + "expected": "false", + }, + "svc": { + "type": "resource_exists", + "kind": "service", + "name": "my-svc", + }, + } + result = self.verifier.wait_for_condition(spec, timeout_sec=30) + + self.assertTrue(result.success) + self.assertIn("cfg succeeded", result.reason) + self.assertIn("svc succeeded", result.reason) + @patch("subprocess.run") def test_wait_for_condition_compound_success(self, mock_run): def run_side_effect(cmd, *args, **kwargs): diff --git a/pkg/evaluator/evaluate.py b/pkg/evaluator/evaluate.py index 79c8f48c..142136f4 100644 --- a/pkg/evaluator/evaluate.py +++ b/pkg/evaluator/evaluate.py @@ -40,6 +40,7 @@ ) import threading from pkg.manager.manager import ScenarioManager +from pkg.agents.verifier.verifier import VerifierAgent from deployers.factory import get_deployer @@ -161,6 +162,98 @@ def replace_placeholders(text, project_id, cluster_name): ) +def run_deterministic_verification( + verification_spec, project_id, cluster_name, timeout_sec=180 +): + """Runs a task's verification_spec against the live cluster. + + Executed while the cluster is still up (before teardown), so kubectl reads + reflect real applied state. Returns a plain dict describing the outcome, or + None if there is no verification_spec to run. + + Supports both the plain verifier form (single spec / list / dict of specs) + and the named-bundle form used by chaos tasks (a list of objects each with a + "name" key plus verifier sub-specs); in the latter case the verifier + sub-specs of every bundle are evaluated. + """ + if not verification_spec: + return None + + processed = replace_placeholders( + json.dumps(verification_spec) + if isinstance(verification_spec, (dict, list)) + else str(verification_spec), + project_id, + cluster_name, + ) + try: + spec = json.loads(processed) + except (json.JSONDecodeError, TypeError) as e: + return { + "success": False, + "reason": f"Failed to parse verification_spec: {e}", + } + + # Normalize the chaos "named bundle" form into plain verifier specs. + spec = _strip_named_bundles(spec) + + try: + result = VerifierAgent().wait_for_condition(spec, timeout_sec=timeout_sec) + return result.model_dump() + except Exception as e: + return { + "success": False, + "reason": f"Deterministic verification raised: {e}", + } + + +def _strip_named_bundles(spec): + """Converts chaos-style named verification bundles into plain verifier specs. + + A named bundle (the format used by chaos tasks, e.g. optimize-scale) looks + like: + [{ "name": "...", "pod_spec": {..verifier..}, "scaling_spec": {..} }] + The "name" exists so a chaos_spec can reference the bundle by name; it is NOT + a verifier. We drop the "name" key and keep only the verifier sub-specs. + Plain specs (already valid SingleVerificationSpec shapes) pass through + unchanged. + + WHY THIS SHIM EXISTS + -------------------- + A bundle dict is not a valid VerificationSpec: it carries a top-level "name" + string, and if that dict is handed straight to VerifierAgent.wait_for_condition + the dict branch (see pkg/agents/verifier/verifier.py) tries to validate the + "name" *string* as a verifier and raises a ValidationError (verified against + the current code). To reuse the exact same task format for non-chaos + deterministic grading, we normalize the bundle here (drop "name", keep the + verifier sub-specs) before validation. + + Note: the existing chaos path (ScenarioManager -> wait_for_condition) passes + the bundle through WITHOUT this normalization, so it is subject to the same + "name"-key validation issue. The proper long-term fix is at the root (see + below), which would cover both paths. + + HOW TO REMOVE THIS SHIM + ----------------------- + Move the fix to the single source of truth instead: in + VerifierAgent.wait_for_condition's dict branch, skip the reserved "name" key + while iterating (e.g. `if key == "name": continue`). That fixes both the + chaos path and this path at the root, after which this function can be + deleted and callers can pass the spec through directly. + """ + if isinstance(spec, list): + converted = [] + for entry in spec: + if isinstance(entry, dict) and "type" not in entry and "name" in entry: + converted.extend( + v for k, v in entry.items() if k != "name" and isinstance(v, dict) + ) + else: + converted.append(entry) + return converted + return spec + + def print_configuration_context( cloud_provider, project_id, @@ -489,6 +582,13 @@ def evaluate_metrics_batch(detailed_results, judge_model): outcome_criteria = metrics[0].criteria tool_criteria = metrics[1].criteria + # Deterministic gate: if this task has a verification_spec, its + # (already-computed, pre-teardown) result is authoritative for + # correctness. We record it as ChecklistScore and SKIP the slow, + # reward-hackable LLM critical-requirements checks entirely. + det_result = res.get("deterministic_verification") + use_deterministic_gate = res.get("verification_spec") is not None + # Extract checklist items ONLY from the critical requirements section to avoid parsing YAML lists reqs_section = expected_output if "critical requirements:" in reqs_section.lower(): @@ -509,24 +609,30 @@ def evaluate_metrics_batch(detailed_results, judge_model): if line.strip().startswith("-") ] checklist_items = [] - for item in raw_checklist_items: - if not bench_use_mcp and "expected tool call" in item.lower(): - print(f"Skipping Expected Tool Call criteria: '{item}'") - continue - checklist_items.append(item) dynamic_metrics = [] - for item in checklist_items: - dynamic_metrics.append( - GEval( - name=f"Check: {item}", - criteria=( - "Verify that the actual output fulfills this specific" - f" requirement: {item}" - ), - evaluation_params=[SingleTurnParams.ACTUAL_OUTPUT], - model=judge_model, - ) + if use_deterministic_gate: + print( + f"Deterministic gate active for '{name}': skipping LLM" + " critical-requirements checks." ) + else: + for item in raw_checklist_items: + if not bench_use_mcp and "expected tool call" in item.lower(): + print(f"Skipping Expected Tool Call criteria: '{item}'") + continue + checklist_items.append(item) + for item in checklist_items: + dynamic_metrics.append( + GEval( + name=f"Check: {item}", + criteria=( + "Verify that the actual output fulfills this specific" + f" requirement: {item}" + ), + evaluation_params=[SingleTurnParams.ACTUAL_OUTPUT], + model=judge_model, + ) + ) outcome_validity = GEval( name="OutcomeValidity", @@ -580,21 +686,37 @@ def evaluate_metrics_batch(detailed_results, judge_model): latency=latency, ) - print(f"Evaluating metrics for: {name}...") - outcome_result = evaluate([outcome_test_case], metrics=[outcome_validity]) - scores = {} - for test_result in outcome_result.test_results: - for metric_data in test_result.metrics_data: - scores[metric_data.name] = { - "score": metric_data.score, - "success": metric_data.success, - "reason": getattr(metric_data, "reason", None), - } - if os.environ.get("BENCH_USE_MCP", "true").lower() == "true": - tool_result = evaluate([tool_test_case], metrics=[tool_invocation]) - for test_result in tool_result.test_results: + if use_deterministic_gate: + # Correctness comes entirely from the pre-computed deterministic + # verification result. No LLM judge calls are made for this task. + det = det_result or { + "success": False, + "reason": "verification_spec present but no deterministic result was recorded", + } + det_success = bool(det.get("success")) + scores["ChecklistScore"] = { + "score": 1.0 if det_success else 0.0, + "success": det_success, + "reason": det.get("reason", "deterministic verification"), + } + scores["DeterministicVerification"] = { + "score": 1.0 if det_success else 0.0, + "success": det_success, + "reason": det.get("reason"), + "elapsed_time": det.get("elapsed_time"), + "details": det.get("details"), + } + print( + f"Deterministic ChecklistScore for '{name}': " + f"success={det_success} reason={det.get('reason')}" + ) + else: + print(f"Evaluating metrics for: {name}...") + outcome_result = evaluate([outcome_test_case], metrics=[outcome_validity]) + + for test_result in outcome_result.test_results: for metric_data in test_result.metrics_data: scores[metric_data.name] = { "score": metric_data.score, @@ -602,6 +724,16 @@ def evaluate_metrics_batch(detailed_results, judge_model): "reason": getattr(metric_data, "reason", None), } + if os.environ.get("BENCH_USE_MCP", "true").lower() == "true": + tool_result = evaluate([tool_test_case], metrics=[tool_invocation]) + for test_result in tool_result.test_results: + for metric_data in test_result.metrics_data: + scores[metric_data.name] = { + "score": metric_data.score, + "success": metric_data.success, + "reason": getattr(metric_data, "reason", None), + } + if dynamic_metrics: print(f"Evaluating {len(dynamic_metrics)} dynamic metrics sequentially...") for m in dynamic_metrics: @@ -850,9 +982,27 @@ def main(): if scenario_manager else agent_res.get("perf_report", {}), "documentation": item.get("documentation", []), + "verification_spec": item.get("verification_spec"), } ) + # Run deterministic verification against the live cluster BEFORE + # teardown. When present, this becomes the authoritative correctness + # gate and the LLM critical-requirements judge is skipped downstream. + verification_spec = item.get("verification_spec") + if verification_spec: + print( + f"--- Running deterministic verification for: {item['name']} ---" + ) + det_result = run_deterministic_verification( + verification_spec, project_id, active_cluster_name + ) + detailed_results[-1]["deterministic_verification"] = det_result + print( + f"Deterministic verification result: " + f"success={det_result.get('success')} reason={det_result.get('reason')}" + ) + print(f"--- Agent Response ---\n{actual_output}\n----------------------") except Exception as e: diff --git a/tasks/gcp/deploy-config/task.yaml b/tasks/gcp/deploy-config/task.yaml index e9432f77..679d6008 100644 --- a/tasks/gcp/deploy-config/task.yaml +++ b/tasks/gcp/deploy-config/task.yaml @@ -125,4 +125,41 @@ expected_output: | - Execute the deployment of the provided manifests to the '{{GKE_CLUSTER_NAME}}' in project '{{GCP_PROJECT_ID}}'. - Confirm that the 'hypercomputer-d1-vllm-server' deployment was successfully applied to the cluster. - Expected Manifest Generated: N/A, not a manifest generation request. \ No newline at end of file + Expected Manifest Generated: N/A, not a manifest generation request. +# Deterministic correctness gate. When present, this replaces the LLM judge for +# grading: each object below is looked up in the live cluster with kubectl, which +# proves the agent actually applied the manifests (Deployment + Service + HPA) +# rather than just printing YAML. We assert HPA.minReplicas as a field-level check +# so config correctness is verified without depending on GPU pod scheduling. +verification_spec: | + [ + { + "name": "Manifests Applied Verification", + "deployment_applied": { + "type": "resource_exists", + "kind": "deployment", + "name": "hypercomputer-d1-vllm-server", + "namespace": "default" + }, + "service_applied": { + "type": "resource_exists", + "kind": "service", + "name": "hypercomputer-d1-vllm-service", + "namespace": "default" + }, + "hpa_applied": { + "type": "resource_exists", + "kind": "hpa", + "name": "hypercomputer-d1-vllm-hpa", + "namespace": "default" + }, + "hpa_min_replicas": { + "type": "resource_field", + "kind": "hpa", + "name": "hypercomputer-d1-vllm-hpa", + "namespace": "default", + "json_path": "{.spec.minReplicas}", + "expected": "1" + } + } + ] \ No newline at end of file diff --git a/tasks/gcp/fix-config/task.yaml b/tasks/gcp/fix-config/task.yaml index 1527176c..1e138fe5 100644 --- a/tasks/gcp/fix-config/task.yaml +++ b/tasks/gcp/fix-config/task.yaml @@ -54,4 +54,22 @@ expected_output: | - Confirm that 'USE_GEMINI_API' is set to 'false' and 'VLLM_API_URL' is correctly pointed to the local service. - Validate that the agent actually applied the manifest to the cluster using tools (e.g., kubectl) rather than just showing the YAML. - Expected Manifest Generated: N/A, not a manifest generation request. \ No newline at end of file + Expected Manifest Generated: N/A, not a manifest generation request. +# Deterministic correctness gate. When present, this replaces the LLM judge for +# grading: the checks below are run against the live cluster with kubectl. Reading +# these live field values also proves the manifest was actually applied (not just +# printed), covering the "actually applied the manifest" critical requirement. +verification_spec: | + [ + { + "name": "Config Applied Verification", + "use_gemini_api_disabled": { + "type": "resource_field", + "kind": "deployment", + "name": "hypercomputer-d1-frontend", + "namespace": "default", + "json_path": "{.spec.template.spec.containers[?(@.name=='frontend')].env[?(@.name=='USE_GEMINI_API')].value}", + "expected": "false" + } + } + ] \ No newline at end of file From 525822fcd9ce608d3fa8e5eb0f6d2c170b993456 Mon Sep 17 00:00:00 2001 From: Jegath S Date: Wed, 15 Jul 2026 16:57:24 +0530 Subject: [PATCH 2/4] Update pkg/agents/verifier/resource_exists.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- pkg/agents/verifier/resource_exists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/agents/verifier/resource_exists.py b/pkg/agents/verifier/resource_exists.py index 69ad17cd..013aa752 100644 --- a/pkg/agents/verifier/resource_exists.py +++ b/pkg/agents/verifier/resource_exists.py @@ -41,7 +41,7 @@ def verify(self, timeout_sec: int) -> VerificationResult: time.sleep(delay) delay = min(delay * 2, max_delay) - def _check_exists(self) -> (bool, dict): + def _check_exists(self) -> tuple[bool, dict]: cmd = ["kubectl", "get", self.kind, self.name] if self.namespace: cmd.extend(["-n", self.namespace]) From 349b6a3699e0caa90b840320ef283c62768210fa Mon Sep 17 00:00:00 2001 From: Jegath S Date: Wed, 15 Jul 2026 16:58:11 +0530 Subject: [PATCH 3/4] Update pkg/agents/verifier/resource_exists.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- pkg/agents/verifier/resource_exists.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/agents/verifier/resource_exists.py b/pkg/agents/verifier/resource_exists.py index 013aa752..4adb2788 100644 --- a/pkg/agents/verifier/resource_exists.py +++ b/pkg/agents/verifier/resource_exists.py @@ -55,3 +55,7 @@ def _check_exists(self) -> tuple[bool, dict]: return False, { "reason": f"{self.kind}/{self.name} not found: {e.stderr.strip()}" } + except FileNotFoundError: + return False, { + "reason": "kubectl command not found in PATH. Please ensure kubectl is installed." + } From 0f2bf3f03b079567e102ec5484e0d312fd015eb3 Mon Sep 17 00:00:00 2001 From: Jegath S Date: Wed, 15 Jul 2026 17:29:14 +0530 Subject: [PATCH 4/4] if chaos and verification is enabled just reuse chaos verification result instead of running it again. --- pkg/evaluator/evaluate.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/pkg/evaluator/evaluate.py b/pkg/evaluator/evaluate.py index 142136f4..3579e082 100644 --- a/pkg/evaluator/evaluate.py +++ b/pkg/evaluator/evaluate.py @@ -585,7 +585,10 @@ def evaluate_metrics_batch(detailed_results, judge_model): # Deterministic gate: if this task has a verification_spec, its # (already-computed, pre-teardown) result is authoritative for # correctness. We record it as ChecklistScore and SKIP the slow, - # reward-hackable LLM critical-requirements checks entirely. + # reward-hackable LLM critical-requirements checks entirely. This holds + # for both non-chaos and chaos tasks; for chaos tasks the result was + # produced by the ScenarioManager during the fault window and reused + # (see the execution loop) rather than re-run. det_result = res.get("deterministic_verification") use_deterministic_gate = res.get("verification_spec") is not None @@ -986,17 +989,30 @@ def main(): } ) - # Run deterministic verification against the live cluster BEFORE - # teardown. When present, this becomes the authoritative correctness - # gate and the LLM critical-requirements judge is skipped downstream. + # Deterministic verification. When a verification_spec is present it + # becomes the authoritative correctness gate and the LLM + # critical-requirements judge is skipped downstream. + # + # For chaos tasks the ScenarioManager already ran the same + # verification_spec against the cluster during the fault window and + # stored the outcome in chaos_report["verification"]. We reuse that + # result instead of running the verifier a second time. For non-chaos + # tasks we run it once here, against the live cluster before teardown. verification_spec = item.get("verification_spec") if verification_spec: - print( - f"--- Running deterministic verification for: {item['name']} ---" - ) - det_result = run_deterministic_verification( - verification_spec, project_id, active_cluster_name - ) + chaos_verification = chaos_report.get("verification") + if scenario_manager and chaos_verification: + print( + f"--- Reusing chaos verification result for: {item['name']} ---" + ) + det_result = chaos_verification + else: + print( + f"--- Running deterministic verification for: {item['name']} ---" + ) + det_result = run_deterministic_verification( + verification_spec, project_id, active_cluster_name + ) detailed_results[-1]["deterministic_verification"] = det_result print( f"Deterministic verification result: "