feat(verification): type verification_spec with roles (schema foundation)

devops_bench/evalharness/default.py

@@ -54,6 +54,7 @@
 )
 from devops_bench.tasks import Task
 from devops_bench.verification import VerificationSpec
+from devops_bench.verification.entry import VerificationEntry
 
 __all__ = ["DefaultEvalHarness"]
 
@@ -470,6 +471,32 @@ def _build_verification_mapping(
             return {}, []
 
         errors: list[dict[str, str]] = []
+        mapping: dict[str, Any] = {}
+
+        # Typed path: task.verification_spec has been parsed into VerificationEntry
+        # objects at load time. Placeholders live inside entry.spec (still raw dicts).
+        if isinstance(raw, list) and raw and isinstance(raw[0], VerificationEntry):
+            for entry in raw:
+                try:
+                    spec_resolved = self._resolve_spec_placeholders(
+                        entry.spec, cluster_name, target_deployment, namespace
+                    )
+                    if entry.name in mapping:
+                        _log.warning(
+                            "duplicate verification entry name %r; overwriting", entry.name
+                        )
+                    mapping[entry.name] = VerificationSpec(spec_resolved)
+                except Exception as exc:  # noqa: BLE001 - surface every failure
+                    _log.warning(
+                        "verification entry %r failed to validate; skipping: %s",
+                        entry.name,
+                        exc,
+                    )
+                    errors.append({"name": entry.name, "reason": str(exc)})
+            return mapping, errors
+
+        # Legacy path: raw dict / list / JSON-string (pre-migration or direct calls
+        # that bypass the typed schema).
         resolved = self._resolve_spec_placeholders(raw, cluster_name, target_deployment, namespace)
         if isinstance(resolved, str):
             try:
@@ -480,7 +507,6 @@ def _build_verification_mapping(
                 return {}, errors
         entries = resolved if isinstance(resolved, list) else [resolved]
 
-        mapping: dict[str, Any] = {}
         for index, entry in enumerate(entries):
             if not isinstance(entry, dict):
                 msg = (
@@ -500,6 +526,8 @@ def _build_verification_mapping(
             # a ``spec`` key is itself treated as the typed node.
             node = entry.get("spec") if "spec" in entry else entry
             try:
+                if name in mapping:
+                    _log.warning("duplicate verification entry name %r; overwriting", name)
                 mapping[name] = VerificationSpec(node)
             except Exception as exc:  # noqa: BLE001 - surface every failure
                 _log.warning(
@@ -944,7 +972,11 @@ def _empty_record(self, task: Task) -> dict[str, Any]:
             "expected_output_raw": task.expected_output,
             "retrieval_context": list(task.retrieval_context),
             "chaos_spec": task.chaos_spec,
-            "verification_spec": task.verification_spec,
+            "verification_spec": (
+                [e.model_dump() if hasattr(e, "model_dump") else e for e in task.verification_spec]
+                if task.verification_spec is not None
+                else None
+            ),
             "chaos_report": {},
             "perf_report": {},
             "documentation": [doc.model_dump() for doc in task.documentation],

devops_bench/models/__init__.py

@@ -14,10 +14,10 @@
 
 """LLM provider clients and the provider-selection factory.
 
-Each provider has an adapter module named by model family (``gemini``,
-``claude``; ``ollama`` is the runtime exception). A provider's SDK is imported
-only when its adapter is constructed, so a missing SDK surfaces as
-:class:`MissingDependencyError` at construction time, not on import.
+Adapter modules are named by model family and register themselves in
+``MODELS`` on import. A provider's SDK is imported only when its adapter is
+constructed, so a missing SDK surfaces as :class:`MissingDependencyError` at
+construction time, not on import.
 """
 
 from __future__ import annotations

devops_bench/models/base.py

@@ -67,7 +67,7 @@ def format_tools(self, mcp_tools: Any) -> Any:
         """
 
     @abstractmethod
-    def extract_function_calls(self, response: Any) -> list[dict]:
+    def extract_function_calls(self, response: Any) -> list[dict[str, Any]]:
         """Extract function calls from the model's response.
 
         Args:

devops_bench/tasks/schema.py

@@ -19,6 +19,7 @@
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 from devops_bench.core import get_logger
+from devops_bench.verification.entry import VerificationEntry
 
 __all__ = ["Task", "DocumentationEntry", "Constraint"]
 
@@ -143,7 +144,7 @@ class Task(BaseModel):
     expected_output: str = ""
     retrieval_context: list[str] = Field(default_factory=list)
     chaos_spec: Any = None
-    verification_spec: Any = None
+    verification_spec: list[VerificationEntry] | None = None
     infrastructure: dict[str, Any] = Field(default_factory=dict)
     documentation: list[DocumentationEntry] = Field(default_factory=list)
     validated: bool = False

devops_bench/verification/__init__.py

@@ -19,11 +19,15 @@
 typed outcome (:class:`VerificationResult`), the registry-driven extension
 surface (:data:`VERIFIERS`, :func:`parse_node`), and the deadline-based
 :class:`VerifierAgent` that evaluates them. Importing the package pulls no
-heavy SDKs — concrete leaf verifiers register via this package's submodules
+heavy SDKs -- concrete leaf verifiers register via this package's submodules
 only.
+
+The entry-level schema (:class:`VerificationEntry`, :data:`Role`) is available
+via direct submodule imports or from this package.
 """
 
-from devops_bench.verification.base import VERIFIERS, BaseVerifier, VerificationResult
+from devops_bench.verification.base import VERIFIERS, BaseVerifier, Mode, VerificationResult
+from devops_bench.verification.entry import Role, VerificationEntry
 from devops_bench.verification.runner import VerifierAgent
 from devops_bench.verification.spec import (
     ParallelSpec,
@@ -36,9 +40,12 @@
 
 __all__ = [
     "BaseVerifier",
+    "Mode",
     "ParallelSpec",
+    "Role",
     "SequenceSpec",
     "VERIFIERS",
+    "VerificationEntry",
     "VerificationNode",
     "VerificationResult",
     "VerificationSpec",

devops_bench/verification/base.py

@@ -23,14 +23,23 @@
 import time
 from abc import ABC, abstractmethod
 from collections.abc import Callable
-from typing import Any
+from typing import Any, Literal
 
 from pydantic import BaseModel, Field
 
 from devops_bench.core import Registry
 from devops_bench.k8s import poll_until
 
-__all__ = ["VERIFIERS", "VerificationResult", "BaseVerifier"]
+__all__ = ["Mode", "VERIFIERS", "VerificationResult", "BaseVerifier"]
+
+# Execution mode for a leaf verifier.
+#
+# converge  -- verify(timeout_sec=N): keep polling until success or deadline.
+# assert    -- verify(timeout_sec=0): evaluate exactly once, never sleep.
+# hold      -- sample verify(0) repeatedly; every sample must pass.
+# unchanged -- NOT IMPLEMENTED: requires a pre-agent snapshot protocol.
+#              The first consumer (unchanged_outside) is not in this PR.
+Mode = Literal["converge", "assert", "unchanged", "hold"]
 
 # Registry keyed by the ``type`` discriminator literal. Entry-point discovery
 # lets external packages register a verifier without touching this tree.
@@ -53,6 +62,14 @@ class VerificationResult(BaseModel):
         name: Optional label echoed from the spec node, for result rendering.
         children: Per-member results from compound (sequence/parallel) nodes.
         raw: Leaf-only kubectl diagnostics or supporting data.
+        weight: Contribution weight used by the rollup; default 1.0. Echoed
+            from the originating leaf's :attr:`BaseVerifier.weight` so the
+            result tree is self-describing for downstream scoring. The runner
+            (:meth:`~devops_bench.verification.runner.VerifierAgent._run_leaf`)
+            stamps the leaf's configured weight onto the result after
+            evaluation, so a custom verifier that builds this result directly
+            (bypassing :meth:`BaseVerifier._poll_to_result`) does not need to
+            forward ``weight`` itself — the runner is authoritative.
     """
 
     success: bool
@@ -61,6 +78,7 @@ class VerificationResult(BaseModel):
     name: str | None = None
     children: list[VerificationResult] = Field(default_factory=list)
     raw: dict | None = None
+    weight: float = 1.0
 
 
 VerificationResult.model_rebuild()
@@ -77,10 +95,26 @@ class BaseVerifier(BaseModel, ABC):
         kubeconfig: Optional path to a kubeconfig file, forwarded to the
             ``devops_bench.k8s`` wrappers so a check can target a specific
             cluster. When ``None`` the wrappers use the ambient kubeconfig.
+        weight: Contribution weight used by the rollup when summing pass/fail
+            fractions across leaves. Default 1.0 (all leaves equal). Echoed
+            onto the :class:`VerificationResult` so the tree is self-describing.
+        mode: Explicit execution mode. When ``None`` the runner derives the
+            mode from the parent :class:`~devops_bench.verification.entry.Role`:
+            ``"objective"`` -> ``"converge"``;
+            ``"safeguard"`` -> ``"assert"``. An explicit ``mode`` always wins
+            over the role-derived default.
+        hold_window_sec: Duration of the hold-mode sampling window; meaningful
+            only when ``mode="hold"``. Defaults to 30 s.
+        hold_interval_sec: Pause between hold-mode samples; meaningful only
+            when ``mode="hold"``. Defaults to 5 s.
     """
 
     name: str | None = None
     kubeconfig: str | None = None
+    weight: float = 1.0
+    mode: Mode | None = None
+    hold_window_sec: float = 30.0
+    hold_interval_sec: float = 5.0
 
     @abstractmethod
     def verify(self, timeout_sec: float) -> VerificationResult:
@@ -133,4 +167,5 @@ def predicate() -> bool:
             reason=last["reason"],
             name=self.name,
             raw=last["raw"],
+            weight=self.weight,
         )

devops_bench/verification/entry.py

@@ -0,0 +1,102 @@
+# Copyright 2026 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Typed outer wrapper for a verification spec entry.
+
+This module is intentionally kept dependency-light: it must not import from
+``devops_bench.tasks`` (that direction would create a cycle) and it must not
+import from ``devops_bench.verification.spec`` (that triggers registry
+population and verifier imports at task-load time, before placeholder
+substitution has run).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, model_validator
+
+__all__ = ["Role", "Severity", "VerificationEntry"]
+
+# Scoring role of a verification entry.
+#
+# objective  -- something the agent must achieve; feeds the correctness score c.
+# safeguard  -- a protective invariant that must hold; a violation penalizes
+#               (recoverable) or hard-gates (catastrophic) the score.
+Role = Literal["objective", "safeguard"]
+
+# Severity of a safeguard violation. Required when role='safeguard', forbidden
+# when role='objective'.
+#
+# recoverable  -- a violation penalizes the score (feeds rec_v).
+# catastrophic -- a violation hard-gates the score to zero (feeds cat_v).
+Severity = Literal["recoverable", "catastrophic"]
+
+
+def _contains_unchanged_mode(value: Any) -> bool:
+    """Recursively walk a raw spec value looking for ``mode: unchanged``."""
+    if isinstance(value, dict):
+        if value.get("mode") == "unchanged":
+            return True
+        return any(_contains_unchanged_mode(v) for v in value.values())
+    if isinstance(value, list):
+        return any(_contains_unchanged_mode(item) for item in value)
+    return False
+
+
+class VerificationEntry(BaseModel):
+    """One named verification suite with an associated scoring role.
+
+    Attributes:
+        name: Cross-reference key resolved by the chaos ``verify:`` field and
+            used as the key in the name-keyed verification mapping.
+        role: Scoring category. Defaults to ``"objective"`` so every
+            existing spec authored without a ``role`` key is backward-compatible
+            without editing.
+        severity: Required when ``role == "safeguard"``; must be ``None`` when
+            ``role == "objective"``. ``"recoverable"`` violations penalize the
+            score (feed ``rec_v``); ``"catastrophic"`` violations hard-gate the
+            run (feed ``cat_v``).
+        spec: Raw (unparsed) verification node dict. Placeholder substitution
+            (``{{NAMESPACE}}``, etc.) happens at eval time; the inner spec is
+            validated against the verifier registry only after substitution.
+    """
+
+    name: str
+    role: Role = "objective"
+    severity: Severity | None = None
+    spec: Any
+
+    @model_validator(mode="after")
+    def _validate_severity(self) -> VerificationEntry:
+        if self.role == "safeguard" and self.severity is None:
+            raise ValueError(
+                "severity is required when role='safeguard'; "
+                "set severity='recoverable' or severity='catastrophic'"
+            )
+        if self.role == "objective" and self.severity is not None:
+            raise ValueError(
+                "severity must be None when role='objective'; "
+                "severity is only meaningful on safeguard entries"
+            )
+        return self
+
+    @model_validator(mode="after")
+    def _reject_unchanged_mode(self) -> VerificationEntry:
+        if _contains_unchanged_mode(self.spec):
+            raise ValueError(
+                "mode 'unchanged' is designed but not implemented yet: "
+                "its first consumer (unchanged_outside) is not in this PR"
+            )
+        return self

devops_bench/verification/runner.py

@@ -119,11 +119,23 @@ def _run_leaf(self, node: Any, deadline: float) -> VerificationResult:
         Short-circuits when the remaining budget is below
         :data:`_MIN_LEAF_BUDGET_SECONDS` so we never issue a useless
         sub-second ``kubectl wait`` at the tail of the deadline.
+
+        The verifier's configured ``weight`` is stamped onto the returned result
+        here, authoritatively: the leaf is the single source of truth for its
+        weight, so a custom ``verify()`` that builds a
+        :class:`VerificationResult` directly (bypassing ``_poll_to_result``) can
+        never silently drop it. When the node carries no ``weight`` attribute
+        (e.g. a compound spec reached as a leaf), the result's own weight is
+        left untouched.
         """
         remaining = deadline - time.monotonic()
         if remaining < _MIN_LEAF_BUDGET_SECONDS:
             return _timed_out(node, "deadline exhausted before evaluation")
-        return node.verify(remaining)
+        result = node.verify(remaining)
+        weight = getattr(node, "weight", None)
+        if weight is not None:
+            result.weight = weight
+        return result
 
     def _run_sequence(self, node: SequenceSpec, deadline: float) -> VerificationResult:
         """Run children in order; stop and skip the rest on the first failure."""