From 1754bc1f736d8c9dfe969a7e34c085f04c7979c6 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 5 Aug 2026 17:29:19 -0600 Subject: [PATCH 01/34] feat(examples): add Harbor Hermes Switchyard eval Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 215 ++++++++++++ .../agents/harbor_hermes_agent.py | 330 ++++++++++++++++++ .../config/otel-collector.yaml | 18 + .../config/relay.toml.in | 95 +++++ .../harbor-hermes-switchyard/requirements.txt | 7 + .../run_terminal_bench.sh | 234 +++++++++++++ .../scripts/build_switchyard_plugin.sh | 189 ++++++++++ .../scripts/fake_openai_upstream.py | 97 +++++ .../scripts/fake_otlp_collector.py | 66 ++++ .../scripts/finalize_artifacts.py | 259 ++++++++++++++ .../scripts/native_plugin_loader_smoke.py | 43 +++ .../scripts/offline_compatibility_smoke.py | 121 +++++++ .../scripts/prepare_runtime.py | 230 ++++++++++++ .../run_offline_compatibility_smoke.sh | 135 +++++++ .../scripts/run_phase1_regressions.sh | 70 ++++ .../scripts/upload_openinference.py | 149 ++++++++ .../scripts/validate_run.py | 235 +++++++++++++ .../scripts/verify_harbor_hermes_compat.py | 110 ++++++ .../tests/test_agent_result_contract.py | 99 ++++++ .../tests/test_config_contract.py | 65 ++++ .../tests/test_plugin_lifecycle.py | 59 ++++ 21 files changed, 2826 insertions(+) create mode 100644 examples/harbor-hermes-switchyard/README.md create mode 100644 examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py create mode 100644 examples/harbor-hermes-switchyard/config/otel-collector.yaml create mode 100644 examples/harbor-hermes-switchyard/config/relay.toml.in create mode 100644 examples/harbor-hermes-switchyard/requirements.txt create mode 100755 examples/harbor-hermes-switchyard/run_terminal_bench.sh create mode 100755 examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh create mode 100755 examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py create mode 100755 examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py create mode 100755 examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py create mode 100644 examples/harbor-hermes-switchyard/scripts/native_plugin_loader_smoke.py create mode 100755 examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py create mode 100755 examples/harbor-hermes-switchyard/scripts/prepare_runtime.py create mode 100755 examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh create mode 100755 examples/harbor-hermes-switchyard/scripts/run_phase1_regressions.sh create mode 100755 examples/harbor-hermes-switchyard/scripts/upload_openinference.py create mode 100755 examples/harbor-hermes-switchyard/scripts/validate_run.py create mode 100755 examples/harbor-hermes-switchyard/scripts/verify_harbor_hermes_compat.py create mode 100644 examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py create mode 100644 examples/harbor-hermes-switchyard/tests/test_config_contract.py create mode 100644 examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md new file mode 100644 index 000000000..e39579da8 --- /dev/null +++ b/examples/harbor-hermes-switchyard/README.md @@ -0,0 +1,215 @@ +# Harbor + Hermes + Switchyard evaluation + +This example runs a Terminal-Bench 2.0 task through Harbor and Hermes while +Hermes owns an in-process NeMo Relay 0.7.0 runtime. Relay initializes static +pricing and observability components and activates Switchyard as a standard +dynamic native plugin. + +The example is deliberately a one-task integration reference. It does not +replace Harbor's task lifecycle and it is not the full 89-task coordinator. +The four-task regression command below is the Phase 1 readiness gate. + +## Pinned inputs + +| Dependency | Input used by this example | +|---|---| +| NeMo Relay | Released Linux/amd64 `nemo-relay==0.7.0` wheel; that exact wheel is installed and its digest is recorded per run. | +| Hermes | `bbednarski9/hermes-agent`, branch `feat/relay-native-plugin-init`, detached commit `a07830e086b3055e313b74cc0c8fd5326a4c2c00` (PR #77915). | +| Switchyard | `bbednarski9/Switchyard`, detached commit `8293936a0f5758aa1a782639d485b8b8948cf03e` (PR #270). | +| Harbor | `harbor==0.18.0`, dataset `terminal-bench@2.0`. | + +The branch names make the development inputs discoverable; only the full +commits are authoritative. Every checkout is detached and verified before +execution. The Hermes installer is followed by a final `uv sync --frozen` +against that commit's checked-in lock because its date-relative resolution +guard can otherwise make an older checkout appear stale. The verified Relay +0.7.0 platform wheel is then force-installed by digest without dependencies. + +## Request and lifecycle ownership + +There is no Switchyard service in this topology: + +1. Harbor creates the Terminal-Bench task environment and invokes its built-in + Hermes lifecycle through the temporary subclass in + `agents/harbor_hermes_agent.py`. +2. Hermes initializes Relay and asks Relay's public dynamic-plugin loader to + activate `nvidia.switchyard` from `[[plugins.dynamic]]`. +3. Relay owns the outer managed LLM operation and invokes the native execution + intercept. +4. The Switchyard plugin selects the route and its `switchyard-llm-client` + performs the provider HTTP request. +5. Hermes waits for Relay operations, plugin cleanup, subscribers, and + exporters before returning to Harbor. + +That split is important: Relay dispatches into the plugin intercept, while the +pinned Switchyard plugin owns the provider HTTP client. The direct receipt +records both facts without claiming a second routing service exists. + +Static components and dynamic plugins are separate concepts. The pricing and +schema-v3 observability components in `config/relay.toml.in` are static Relay +components. Switchyard is a standard dynamic native Relay plugin. Hermes +`[[dynamic_plugins]]` Python workers are not used, and the bridge rejects a +configuration that mixes the two activation models before provider traffic. + +## Why the temporary Harbor agent exists + +Harbor 0.18.0's built-in Hermes agent accepts a branch-like `version` and +clones the upstream NousResearch repository. It cannot select a fork plus an +immutable arbitrary commit, nor can it project this example's Relay config and +native bundle. `HarborHermesAgent` changes only installation, configuration +projection, and additional artifact framing; the inherited Harbor setup/run, +timeout, task, session export, and ATIF conversion remain in control. + +Remove this bridge and use `--agent hermes` once +[hermes-agent#77915](https://github.com/NousResearch/hermes-agent/pull/77915) +is upstream **and** Harbor's built-in agent can install a released, pinned +compatible Hermes revision while projecting the Relay config and plugin +bundle. Merging the Hermes PR alone is not sufficient while Harbor remains +upstream-repository-only and branch-only. + +## Prerequisites + +- Docker with enough space to build one Linux/amd64 Rust plugin and task image; +- Python 3.11 or newer; +- a provider endpoint compatible with OpenAI Chat Completions; +- a Phoenix endpoint accepting OTLP/HTTP traces; and +- the provider authorization value in an environment variable. + +On macOS, place bundle and run roots below a directory shared with Docker +(normally `/Users/...`). Do not assume `$TMPDIR` or `/private/tmp` is shared by +Colima merely because the same path exists inside its VM. + +Create a host-side environment for Harbor and the validation tools: + +```bash +cd examples/harbor-hermes-switchyard +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt +export HARBOR_BIN="$PWD/.venv/bin/harbor" +export PHASE1_PYTHON="$PWD/.venv/bin/python" +``` + +The scripts never put the authorization value into TOML or command-line +configuration. They pass the selected environment variable into the task and +scan direct artifacts, Harbor logs, ATOF, ATIF, and OpenInference evidence for +the exact secret value. + +## Offline compatibility gate + +Build the pinned Linux plugin bundle, prepare a fresh run root, and run the +forked Hermes/Relay runtime against local fake provider and OTLP endpoints: + +```bash +export EXAMPLE_ROOT="$PWD" +export SPIKE_ROOT="/absolute/new/spike-root" + +"$EXAMPLE_ROOT/scripts/build_switchyard_plugin.sh" /absolute/new/switchyard-bundle +"$PHASE1_PYTHON" "$EXAMPLE_ROOT/scripts/prepare_runtime.py" \ + --run-root "$SPIKE_ROOT" \ + --switchyard-bundle /absolute/new/switchyard-bundle \ + --upstream-base-url http://127.0.0.1:8000/v1 \ + --target-model phase1/fake-model \ + --openinference-endpoint http://127.0.0.1:4318/v1/traces \ + --phoenix-project phase1-offline \ + --eval-cohort phase1-offline +"$EXAMPLE_ROOT/scripts/run_offline_compatibility_smoke.sh" "$SPIKE_ROOT" +``` + +This gate proves the exact detached Hermes checkout, released Relay wheel, +public loader path, one native Switchyard activation, a real fake-provider HTTP +request, routing marks, file sinks, mixed-mode rejection, and clean shutdown. + +The review gate targets `linux/amd64`, matching the Terminal-Bench task +environment. On an Apple Silicon Docker host, QEMU may crash while unloading a +native Rust plugin; an ARM control can distinguish that emulator failure from +an integration failure: + +```bash +SWITCHYARD_TARGET_ARCHITECTURE=aarch64 \ + "$EXAMPLE_ROOT/scripts/build_switchyard_plugin.sh" /absolute/new/arm64-bundle +"$PHASE1_PYTHON" "$EXAMPLE_ROOT/scripts/prepare_runtime.py" \ + --run-root /absolute/new/arm64-spike-root \ + --switchyard-bundle /absolute/new/arm64-bundle \ + --relay-architecture aarch64 \ + --upstream-base-url http://127.0.0.1:8000/v1 \ + --target-model phase1/fake-model \ + --openinference-endpoint http://127.0.0.1:4318/v1/traces \ + --phoenix-project phase1-offline-arm64 \ + --eval-cohort phase1-offline-arm64 +PHASE1_COMPAT_PLATFORM=linux/arm64 \ + "$EXAMPLE_ROOT/scripts/run_offline_compatibility_smoke.sh" \ + /absolute/new/arm64-spike-root +``` + +That control validates the same source commits and lifecycle on a different +released Relay wheel architecture. It does **not** replace a passing +`linux/amd64` run on native amd64 infrastructure before merge. + +## Run one Terminal-Bench task + +Use a new absolute run root on every invocation: + +```bash +export TARGET_MODEL="your-provider-model" +export UPSTREAM_BASE_URL="https://your-openai-compatible-endpoint/v1" +export UPSTREAM_AUTH_ENV="SWITCHYARD_PROVIDER_AUTHORIZATION" +export SWITCHYARD_PROVIDER_AUTHORIZATION="Bearer ..." +export PHOENIX_BASE_URL="https://your-phoenix-endpoint" +export PHOENIX_PROJECT="harbor-hermes-switchyard-phase1" +export EVAL_COHORT="harbor-hermes-switchyard-phase1" + +./run_terminal_bench.sh /absolute/new/run-root +``` + +The default task is `adaptive-rejection-sampler`. Override it with +`TASK_NAME`. To avoid rebuilding Switchyard for each task, set +`SWITCHYARD_BUNDLE` to a previously built, immutable bundle. Set `RELAY_WHEEL` +to a downloaded 0.7.0 wheel to avoid a repeated package download. + +A task is complete only if both of these files contain `"status": "passed"`: + +- `/validation.json` +- `/phoenix-upload.json` + +`reward.task_passed=false` is a valid completed benchmark observation and is +not retried when both evidence gates pass. + +## Phase 1 regression gate + +Run all historical risk cases independently: + +```bash +./scripts/run_phase1_regressions.sh /absolute/new/regression-root +``` + +| Task | Assertion | +|---|---| +| `adaptive-rejection-sampler` | Provider/config projection, routing marks, receipt, cleanup, and secret scan. | +| `circuit-fibsqrt` | A deterministic post-response test fault preserves the completed response and records the late failure separately. | +| `gpt2-codegolf` | Harbor's bounded agent timeout applies and no Hermes/plugin process survives the task container. | +| `overfull-hbox` | Streaming validation and bounded Phoenix batching preserve the completed result under a larger export load. | + +The deterministic `circuit-fibsqrt` fault is injected only after inherited +Hermes execution returns. It tests the result-framing regression without +corrupting the Relay plugin lifecycle or disabling Phoenix upload. + +## Evidence and safety properties + +Each run root is immutable and private. Preparation refuses an existing root. +The runtime snapshot contains config and dependency digests; it never contains +credential values. Direct task artifacts include: + +- `direct-hermes-result.json`; +- `direct-hermes-receipt.json`; +- `relay/trajectory.atof.jsonl`; +- `relay/atif/trajectory-.atif.json`; +- bounded Hermes diagnostics; +- `validation.json`; and +- `phoenix-upload.json`. + +Artifact validation rejects symlinks and canonical paths escaping the declared +root. Phoenix import is streaming, bounded in batches, retry-limited, and runs +only after the task has returned and OpenInference evidence exists. + +Phase 2 (a parallel 89-task cohort) and Phase 3 (multiple independent cohorts +and aggregated reporting) intentionally remain outside this first example PR. diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py new file mode 100644 index 000000000..e76e5ba93 --- /dev/null +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -0,0 +1,330 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Temporary Harbor Hermes agent for a fork-hosted, commit-pinned Hermes build. + +The class deliberately inherits Harbor's built-in Hermes lifecycle. It only +changes installation, copies an immutable Relay configuration/plugin bundle +into the task environment, and frames the additional Phase 1 artifacts around +``super().run``. +""" + +from __future__ import annotations + +import hashlib +import re +import shlex +import time +import tomllib +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +from harbor.agents.installed.hermes import Hermes +from harbor.environments.base import BaseEnvironment +from harbor.models.agent.context import AgentContext +from typing_extensions import override + +_FULL_SHA = re.compile(r"[0-9a-f]{40}") +_SHA256 = re.compile(r"[0-9a-f]{64}") +_DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" +_DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" +_DEFAULT_HERMES_COMMIT = "a07830e086b3055e313b74cc0c8fd5326a4c2c00" +_DEFAULT_SWITCHYARD_COMMIT = "8293936a0f5758aa1a782639d485b8b8948cf03e" + + +def _require_full_sha(value: str, name: str) -> str: + normalized = value.strip().lower() + if not _FULL_SHA.fullmatch(normalized): + raise ValueError(f"{name} must be a full 40-character hexadecimal commit") + return normalized + + +def _require_sha256(value: str, name: str) -> str: + normalized = value.strip().lower() + if not _SHA256.fullmatch(normalized): + raise ValueError(f"{name} must be a 64-character hexadecimal SHA-256") + return normalized + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _require_public_https_git_url(value: str) -> str: + parsed = urlsplit(value) + if ( + parsed.scheme != "https" + or parsed.hostname != "github.com" + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + or not parsed.path.endswith(".git") + ): + raise ValueError("repository_url must be a credential-free https://github.com/...git URL") + return value + + +def _find_named_component(config: dict[str, Any], kind: str) -> dict[str, Any]: + matches = [ + component + for component in config.get("components", []) + if isinstance(component, dict) and component.get("kind") == kind and component.get("enabled", True) + ] + if len(matches) != 1: + raise ValueError(f"Relay config must contain exactly one enabled {kind!r} component") + return matches[0] + + +def _validate_relay_config(path: Path) -> None: + with path.open("rb") as stream: + config = tomllib.load(stream) + + if config.get("version") != 1: + raise ValueError("Relay config must use top-level version = 1") + if config.get("dynamic_plugins"): + raise ValueError("Hermes [[dynamic_plugins]] worker records are not allowed in this example") + + plugins = config.get("plugins") + dynamic = plugins.get("dynamic") if isinstance(plugins, dict) else None + if not isinstance(dynamic, list) or len(dynamic) != 1: + raise ValueError("Relay config must contain exactly one [[plugins.dynamic]] record") + manifest = dynamic[0].get("manifest") if isinstance(dynamic[0], dict) else None + if not isinstance(manifest, str) or not manifest.endswith("/relay-plugin.toml"): + raise ValueError("the dynamic plugin must reference a Relay plugin manifest") + + _find_named_component(config, "pricing") + observability = _find_named_component(config, "observability") + observability_config = observability.get("config") + if not isinstance(observability_config, dict) or observability_config.get("version") != 3: + raise ValueError("the observability component must use schema version = 3") + + def reject_literal_headers(value: Any, location: str = "config") -> None: + if isinstance(value, dict): + for key, nested in value.items(): + if key == "headers": + raise ValueError(f"literal headers are forbidden; use header_env ({location}.headers)") + reject_literal_headers(nested, f"{location}.{key}") + elif isinstance(value, list): + for index, nested in enumerate(value): + reject_literal_headers(nested, f"{location}[{index}]") + + reject_literal_headers(config) + + +class HarborHermesAgent(Hermes): + """Hermes #77915 bridge retaining Harbor's built-in Hermes behavior.""" + + def __init__( + self, + *args: Any, + repository_url: str = _DEFAULT_HERMES_REPOSITORY, + repository_ref: str = _DEFAULT_HERMES_REF, + commit: str = _DEFAULT_HERMES_COMMIT, + relay_config_path: str, + switchyard_bundle_dir: str, + relay_wheel_path: str, + relay_wheel_sha256: str, + switchyard_commit: str = _DEFAULT_SWITCHYARD_COMMIT, + artifact_root: str = "/logs/agent/direct-hermes", + inject_post_response_failure: bool = False, + **kwargs: Any, + ) -> None: + self.repository_url = _require_public_https_git_url(repository_url) + self.repository_ref = repository_ref.strip() + if not self.repository_ref or self.repository_ref.startswith("-"): + raise ValueError("repository_ref must be a non-option branch or tag name") + self.commit = _require_full_sha(commit, "commit") + self.switchyard_commit = _require_full_sha(switchyard_commit, "switchyard_commit") + self.relay_wheel_sha256 = _require_sha256(relay_wheel_sha256, "relay_wheel_sha256") + + self.relay_config_path = Path(relay_config_path).expanduser().resolve() + self.switchyard_bundle_dir = Path(switchyard_bundle_dir).expanduser().resolve() + self.relay_wheel_path = Path(relay_wheel_path).expanduser().resolve() + self.artifact_root = artifact_root.rstrip("/") + self.inject_post_response_failure = inject_post_response_failure + if not self.artifact_root.startswith("/logs/agent/"): + raise ValueError("artifact_root must be an absolute child of /logs/agent") + if not self.relay_config_path.is_file(): + raise FileNotFoundError(self.relay_config_path) + if not self.relay_wheel_path.is_file(): + raise FileNotFoundError(self.relay_wheel_path) + if _sha256(self.relay_wheel_path) != self.relay_wheel_sha256: + raise ValueError("Relay wheel digest does not match relay_wheel_sha256") + if "manylinux" not in self.relay_wheel_path.name or "x86_64" not in self.relay_wheel_path.name: + raise ValueError("Relay wheel must target Linux x86_64") + _validate_relay_config(self.relay_config_path) + + self.switchyard_manifest = self.switchyard_bundle_dir / "relay-plugin.toml" + if not self.switchyard_manifest.is_file(): + raise FileNotFoundError(self.switchyard_manifest) + libraries = sorted( + path + for path in self.switchyard_bundle_dir.iterdir() + if path.is_file() and path.suffix in {".so", ".dylib", ".dll"} + ) + if len(libraries) != 1: + raise ValueError("Switchyard bundle must contain exactly one native library") + self.switchyard_library = libraries[0] + + self._example_root = Path(__file__).resolve().parents[1] + self._finalizer_path = self._example_root / "scripts" / "finalize_artifacts.py" + if not self._finalizer_path.is_file(): + raise FileNotFoundError(self._finalizer_path) + + extra_env = dict(kwargs.pop("extra_env", None) or {}) + extra_env["HERMES_NEMO_RELAY_PLUGINS_TOML"] = "/tmp/hermes/relay/plugins.toml" + super().__init__(*args, version=self.commit, extra_env=extra_env, **kwargs) + + @override + async def install(self, environment: BaseEnvironment) -> None: + await self.exec_as_root( + environment, + command=( + "apt-get update && " + "apt-get install -y --no-install-recommends " + "ca-certificates build-essential curl git ripgrep xz-utils" + ), + env={"DEBIAN_FRONTEND": "noninteractive"}, + ) + + repository = shlex.quote(self.repository_url) + repository_ref = shlex.quote(self.repository_ref) + commit = shlex.quote(self.commit) + install_dir = "/tmp/hermes-agent-src" + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; " + f"git clone --no-tags --branch {repository_ref} {repository} {install_dir}; " + f"git -C {install_dir} fetch --depth 1 origin {commit}; " + f"git -C {install_dir} checkout --detach {commit}; " + f'test "$(git -C {install_dir} rev-parse HEAD)" = {commit}; ' + f"HERMES_HOME=/tmp/hermes HERMES_INSTALL_DIR={install_dir} " + f"bash {install_dir}/scripts/install.sh --skip-setup --skip-browser " + f"--no-skills --dir {install_dir} --branch {repository_ref} " + f"--commit {commit} --force-commit; " + f'test "$(git -C {install_dir} rev-parse HEAD)" = {commit}; ' + f"cd {install_dir}; " + f"UV_PROJECT_ENVIRONMENT={install_dir}/venv " + "/tmp/hermes/bin/uv sync --frozen --extra all; " + 'export PATH="$HOME/.local/bin:$PATH"; ' + "hermes version; " + f'{install_dir}/venv/bin/python -c "import importlib.metadata as m; ' + "assert m.version('nemo-relay') == '0.7.0'\"" + ), + ) + + @override + async def setup(self, environment: BaseEnvironment) -> None: + await super().setup(environment) + await self.exec_as_root( + environment, + command=( + "mkdir -p /tmp/hermes/relay /opt/relay-wheels /opt/relay-plugins/nvidia.switchyard /installed-agent" + ), + ) + await environment.upload_file(self.relay_config_path, "/tmp/hermes/relay/plugins.toml") + relay_wheel = f"/opt/relay-wheels/{self.relay_wheel_path.name}" + await environment.upload_file(self.relay_wheel_path, relay_wheel) + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; " + f"test \"$(sha256sum {shlex.quote(relay_wheel)} | cut -d' ' -f1)\" = " + f"{shlex.quote(self.relay_wheel_sha256)}; " + "/tmp/hermes/bin/uv pip install " + "--python /tmp/hermes-agent-src/venv/bin/python " + f"--force-reinstall --no-deps {shlex.quote(relay_wheel)}; " + "/tmp/hermes-agent-src/venv/bin/python -c " + "\"import importlib.metadata as m; assert m.version('nemo-relay') == '0.7.0'\"" + ), + timeout_sec=120, + ) + await environment.upload_dir(self.switchyard_bundle_dir, "/opt/relay-plugins/nvidia.switchyard") + await environment.upload_file(self._finalizer_path, "/installed-agent/finalize_artifacts.py") + await self.exec_as_agent( + environment, + command=self._finalizer_command("initialize"), + env={"HERMES_HOME": "/tmp/hermes"}, + timeout_sec=30, + ) + + def _finalizer_command( + self, + mode: str, + *, + started_at: float | None = None, + error_type: str = "", + ) -> str: + arguments = [ + "/tmp/hermes-agent-src/venv/bin/python", + "/installed-agent/finalize_artifacts.py", + mode, + "--artifact-root", + self.artifact_root, + "--hermes-repository", + self.repository_url, + "--hermes-commit", + self.commit, + "--switchyard-commit", + self.switchyard_commit, + "--relay-wheel-sha256", + self.relay_wheel_sha256, + "--relay-config", + "/tmp/hermes/relay/plugins.toml", + "--switchyard-manifest", + "/opt/relay-plugins/nvidia.switchyard/relay-plugin.toml", + "--switchyard-library", + f"/opt/relay-plugins/nvidia.switchyard/{self.switchyard_library.name}", + "--session-handle", + self.session_id or "", + ] + if started_at is not None: + arguments.extend(["--started-at", str(started_at)]) + if error_type: + arguments.extend(["--error-type", error_type]) + return " ".join(shlex.quote(value) for value in arguments) + + @override + async def run( + self, + instruction: str, + environment: BaseEnvironment, + context: AgentContext, + ) -> None: + started_at = time.time() + error: BaseException | None = None + try: + await super().run(instruction, environment, context) + except BaseException as exc: + error = exc + raise + finally: + try: + await self.exec_as_agent( + environment, + command=self._finalizer_command( + "complete", + started_at=started_at, + error_type=( + type(error).__name__ + if error is not None + else ("InjectedPostResponseFailure" if self.inject_post_response_failure else "") + ), + ), + env={"HERMES_HOME": "/tmp/hermes"}, + timeout_sec=30, + ) + except Exception: + if error is None: + raise + self.logger.exception("Could not frame direct Hermes artifacts after agent failure") + + +__all__ = ["HarborHermesAgent"] diff --git a/examples/harbor-hermes-switchyard/config/otel-collector.yaml b/examples/harbor-hermes-switchyard/config/otel-collector.yaml new file mode 100644 index 000000000..68f78738d --- /dev/null +++ b/examples/harbor-hermes-switchyard/config/otel-collector.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +exporters: + file/openinference: + path: /artifacts/trajectory.openinference.json + +service: + pipelines: + traces/openinference: + receivers: [otlp] + exporters: [file/openinference] diff --git a/examples/harbor-hermes-switchyard/config/relay.toml.in b/examples/harbor-hermes-switchyard/config/relay.toml.in new file mode 100644 index 000000000..e40ea8f41 --- /dev/null +++ b/examples/harbor-hermes-switchyard/config/relay.toml.in @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[[components]] +kind = "pricing" +enabled = true + +[[components.config.sources]] +type = "inline" + +[components.config.sources.catalog] +version = 1 + +[[components.config.sources.catalog.entries]] +provider = "openai" +model_id = "@TARGET_MODEL@" +currency = "USD" +unit = "per_token" +pricing_as_of = "2026-08-05" +pricing_source = "harbor-hermes-switchyard-example" + +[components.config.sources.catalog.entries.rates] +input_per_million = 0.0 +output_per_million = 0.0 +cache_read_per_million = 0.0 + +[components.config.sources.catalog.entries.prompt_cache] +read_accounting = "included_in_prompt_tokens" + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 3 + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +mode = "append" +output_directory = "/logs/agent/direct-hermes/relay" +filename = "trajectory.atof.jsonl" + +[components.config.atif] +enabled = true +agent_name = "Hermes" +agent_version = "@HERMES_COMMIT@" +model_name = "@TARGET_MODEL@" +output_directory = "/logs/agent/direct-hermes/relay/atif" +filename_template = "trajectory-{session_id}.atif.json" + +[components.config.opentelemetry] +enabled = true + +[[components.config.opentelemetry.endpoints]] +type = "openinference" +transport = "http_binary" +endpoint = "@OPENINFERENCE_ENDPOINT@" +service_name = "harbor-hermes-switchyard" +service_namespace = "nemo-relay-examples" +instrumentation_scope = "harbor-hermes-switchyard" +timeout_millis = 5000 + +[components.config.opentelemetry.endpoints.resource_attributes] +"openinference.project.name" = "@PHOENIX_PROJECT@" +"evaluation.cohort" = "@EVAL_COHORT@" + +[[plugins.dynamic]] +manifest = "/opt/relay-plugins/nvidia.switchyard/relay-plugin.toml" + +[plugins.dynamic.config] +version = 2 +priority = 0 +max_retries = 1 + +[plugins.dynamic.config.algorithm] +kind = "random" +seed = 42 + +[plugins.dynamic.config.default_targets] +openai_chat = "primary" + +[plugins.dynamic.config.targets.primary] +model = "@TARGET_MODEL@" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "@UPSTREAM_BASE_URL@" +weight = 1 + +[plugins.dynamic.config.targets.primary.header_env] +authorization = "@UPSTREAM_AUTH_ENV@" diff --git a/examples/harbor-hermes-switchyard/requirements.txt b/examples/harbor-hermes-switchyard/requirements.txt new file mode 100644 index 000000000..5d78e7ac0 --- /dev/null +++ b/examples/harbor-hermes-switchyard/requirements.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +harbor==0.18.0 +opentelemetry-proto==1.38.0 +protobuf==6.33.5 +typing-extensions==4.15.0 diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh new file mode 100755 index 000000000..c40a92be9 --- /dev/null +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +run_root="${1:-}" +task_name="${TASK_NAME:-adaptive-rejection-sampler}" +target_model="${TARGET_MODEL:-}" +upstream_base_url="${UPSTREAM_BASE_URL:-}" +upstream_auth_env="${UPSTREAM_AUTH_ENV:-SWITCHYARD_PROVIDER_AUTHORIZATION}" +phoenix_base="${PHOENIX_BASE_URL:-}" +phoenix_project="${PHOENIX_PROJECT:-harbor-hermes-switchyard-phase1}" +eval_cohort="${EVAL_COHORT:-harbor-hermes-switchyard-phase1}" +harbor_bin="${HARBOR_BIN:-harbor}" +python_bin="${PHASE1_PYTHON:-python3}" +switchyard_bundle="${SWITCHYARD_BUNDLE:-}" +relay_wheel="${RELAY_WHEEL:-}" +agent_timeout_multiplier="${AGENT_TIMEOUT_MULTIPLIER:-3}" +agent_setup_timeout_multiplier="${AGENT_SETUP_TIMEOUT_MULTIPLIER:-6}" +environment_build_timeout_multiplier="${ENVIRONMENT_BUILD_TIMEOUT_MULTIPLIER:-6}" +collector_image="${OTEL_COLLECTOR_IMAGE:-otel/opentelemetry-collector-contrib:0.135.0}" +inject_post_response_failure="${INJECT_POST_RESPONSE_FAILURE:-false}" + +if [[ -z "$run_root" || "$run_root" != /* ]]; then + echo "usage: $0 /absolute/new-run-root" >&2 + exit 2 +fi +if [[ -e "$run_root" ]]; then + echo "run root already exists: $run_root" >&2 + exit 2 +fi +for required in "$target_model" "$upstream_base_url" "$phoenix_base"; do + [[ -n "$required" ]] || { + echo "TARGET_MODEL, UPSTREAM_BASE_URL, and PHOENIX_BASE_URL are required" >&2 + exit 2 + } +done +for dependency in curl docker "$harbor_bin" "$python_bin"; do + command -v "$dependency" >/dev/null || { + echo "missing required command: $dependency" >&2 + exit 1 + } +done +if [[ -z "${!upstream_auth_env:-}" ]]; then + echo "required provider authorization environment variable is unset: $upstream_auth_env" >&2 + exit 2 +fi + +docker info >/dev/null +curl --fail --silent --show-error --max-time 10 "$phoenix_base" >/dev/null + +temporary_build="" +collector_name="" +collector_running=0 +cleanup() { + local status=$? + if [[ "$collector_running" == 1 ]]; then + docker stop --time 10 "$collector_name" >/dev/null 2>&1 || true + fi + if [[ -n "$temporary_build" && -d "$temporary_build" ]]; then + rm -rf "$temporary_build" + fi + return "$status" +} +trap cleanup EXIT + +if [[ -z "$switchyard_bundle" ]]; then + temporary_build="$(mktemp -d "$(dirname "$run_root")/.phase1-switchyard-build.XXXXXX")" + switchyard_bundle="$temporary_build/bundle" + "$example_root/scripts/build_switchyard_plugin.sh" "$switchyard_bundle" +fi + +free_port="$($python_bin - <<'PY' +import socket +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +)" +openinference_endpoint="http://host.docker.internal:$free_port/v1/traces" +upstream_host="$($python_bin -c 'import sys; from urllib.parse import urlsplit; print(urlsplit(sys.argv[1]).hostname or "")' "$upstream_base_url")" + +prepare_args=( + "$example_root/scripts/prepare_runtime.py" + --run-root "$run_root" + --switchyard-bundle "$switchyard_bundle" + --upstream-base-url "$upstream_base_url" + --upstream-auth-env "$upstream_auth_env" + --target-model "$target_model" + --openinference-endpoint "$openinference_endpoint" + --phoenix-project "$phoenix_project" + --eval-cohort "$eval_cohort" +) +if [[ -n "$relay_wheel" ]]; then + prepare_args+=(--relay-wheel "$relay_wheel") +fi +"$python_bin" "${prepare_args[@]}" >"$run_root.prepare.log" + +relay_wheel_sha256="$($python_bin -c 'import json,sys; print(json.load(open(sys.argv[1]))["nemo_relay"]["wheel_sha256"])' "$run_root/runtime/provenance.json")" +relay_wheel_path="$($python_bin -c 'import json,pathlib,sys; p=json.load(open(sys.argv[1])); print(pathlib.Path(sys.argv[1]).parent / "wheels" / p["nemo_relay"]["wheel"])' "$run_root/runtime/provenance.json")" + +"$python_bin" "$example_root/scripts/verify_harbor_hermes_compat.py" \ + --bridge "$example_root/agents/harbor_hermes_agent.py" \ + --relay-config "$run_root/runtime/plugins.toml" \ + --output "$run_root/artifacts/harbor-hermes-compatibility.json" \ + >"$run_root/compatibility.log" + +mkdir -m 0700 "$run_root/telemetry" +collector_name="harbor-hermes-switchyard-$($python_bin -c 'import uuid; print(uuid.uuid4().hex[:12])')" +docker run --detach --rm \ + --name "$collector_name" \ + --publish "127.0.0.1:$free_port:4318" \ + --volume "$example_root/config/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro" \ + --volume "$run_root/telemetry:/artifacts" \ + "$collector_image" \ + --config=/etc/otelcol-contrib/config.yaml >"$run_root/collector.container-id" +collector_running=1 + +export PYTHONPATH="$example_root/agents${PYTHONPATH:+:$PYTHONPATH}" +export OPENAI_API_KEY="relay-managed-placeholder" +job_name="phase1-${task_name}-$(date -u +%Y%m%dT%H%M%SZ)" +agent_hosts=(--allow-agent-host host.docker.internal) +if [[ -n "$upstream_host" && "$upstream_host" != "host.docker.internal" ]]; then + agent_hosts+=(--allow-agent-host "$upstream_host") +fi +agent_kwargs=() +validation_expectations=() +if [[ "$inject_post_response_failure" == "true" ]]; then + agent_kwargs+=(--ak inject_post_response_failure=true) + validation_expectations+=(--expect-late-failure) +elif [[ "$inject_post_response_failure" != "false" ]]; then + echo "INJECT_POST_RESPONSE_FAILURE must be true or false" >&2 + exit 2 +fi +( + "$harbor_bin" run \ + --dataset terminal-bench@2.0 \ + --include-task-name "$task_name" \ + --n-tasks 1 \ + --agent harbor_hermes_agent:HarborHermesAgent \ + --model "openai/$target_model" \ + --ak "repository_url=https://github.com/bbednarski9/hermes-agent.git" \ + --ak "repository_ref=feat/relay-native-plugin-init" \ + --ak "commit=a07830e086b3055e313b74cc0c8fd5326a4c2c00" \ + --ak "relay_config_path=$run_root/runtime/plugins.toml" \ + --ak "switchyard_bundle_dir=$run_root/runtime/switchyard-plugin" \ + --ak "relay_wheel_path=$relay_wheel_path" \ + --ak "relay_wheel_sha256=$relay_wheel_sha256" \ + "${agent_kwargs[@]}" \ + --ae "$upstream_auth_env=${!upstream_auth_env}" \ + --ae OPENAI_API_KEY=relay-managed-placeholder \ + "${agent_hosts[@]}" \ + --artifact /logs/agent/direct-hermes \ + --agent-include-logs 'direct-hermes/**' \ + --agent-include-logs hermes-session.jsonl \ + --agent-include-logs hermes.txt \ + --job-name "$job_name" \ + --jobs-dir "$run_root/jobs" \ + --n-concurrent 1 \ + --n-attempts 1 \ + --agent-timeout-multiplier "$agent_timeout_multiplier" \ + --agent-setup-timeout-multiplier "$agent_setup_timeout_multiplier" \ + --environment-build-timeout-multiplier "$environment_build_timeout_multiplier" \ + --force-build \ + --yes +) >"$run_root/harbor.log" 2>&1 + +docker stop --time 10 "$collector_name" >/dev/null +collector_running=0 + +direct_result="$($python_bin - "$run_root/jobs/$job_name" <<'PY' +import pathlib +import sys + +matches = sorted(pathlib.Path(sys.argv[1]).glob("**/direct-hermes-result.json")) +if len(matches) != 1: + raise SystemExit(f"expected one direct Hermes result, found {len(matches)}") +print(matches[0]) +PY +)" +if [[ -z "$direct_result" ]]; then + echo "direct Hermes result discovery returned an empty path" >&2 + exit 1 +fi +artifact_root="$(dirname "$direct_result")" +openinference="$run_root/telemetry/trajectory.openinference.json" + +validation_args=( + "$example_root/scripts/validate_run.py" + --artifacts "$artifact_root" + --provenance "$run_root/runtime/provenance.json" + --openinference "$openinference" + --harbor-job-dir "$run_root/jobs/$job_name" + --scan-root "$run_root/jobs/$job_name" + --secret-env "$upstream_auth_env" + --output "$artifact_root/validation.json" + "${validation_expectations[@]}" +) +"$python_bin" "${validation_args[@]}" >"$run_root/validation.log" + +"$python_bin" "$example_root/scripts/upload_openinference.py" \ + --openinference "$openinference" \ + --phoenix-url "$phoenix_base" \ + --project "$phoenix_project" \ + --output "$artifact_root/phoenix-upload.json" \ + >"$run_root/phoenix-upload.log" + +"$python_bin" - "$artifact_root" "$run_root" "$job_name" "$task_name" <<'PY' +import json +import pathlib +import sys + +artifacts = pathlib.Path(sys.argv[1]) +run_root = pathlib.Path(sys.argv[2]) +summary = { + "schema_version": "harbor-hermes-switchyard.task-summary.v1", + "status": "passed", + "job_name": sys.argv[3], + "task_name": sys.argv[4], + "artifacts": str(artifacts), + "validation": json.loads((artifacts / "validation.json").read_text()), + "phoenix_upload": json.loads((artifacts / "phoenix-upload.json").read_text()), +} +if summary["validation"].get("status") != "passed" or summary["phoenix_upload"].get("status") != "passed": + raise SystemExit("task evidence gates did not pass") +(run_root / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") +print(json.dumps(summary, indent=2)) +PY + +echo "Phase 1 task passed: $task_name" +echo "Run root: $run_root" +echo "Artifacts: $artifact_root" diff --git a/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh b/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh new file mode 100755 index 000000000..b02f98a94 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +switchyard_repository="${SWITCHYARD_REPOSITORY:-https://github.com/bbednarski9/Switchyard.git}" +switchyard_commit="${SWITCHYARD_COMMIT:-8293936a0f5758aa1a782639d485b8b8948cf03e}" +target_architecture="${SWITCHYARD_TARGET_ARCHITECTURE:-x86_64}" +output_dir="${1:-}" + +if [[ -z "$output_dir" ]]; then + echo "usage: $0 /absolute/output-directory" >&2 + exit 2 +fi +if [[ "$output_dir" != /* ]]; then + echo "output directory must be absolute" >&2 + exit 2 +fi +if [[ -e "$output_dir" ]]; then + echo "refusing to overwrite existing output directory: $output_dir" >&2 + exit 2 +fi +if [[ ! "$switchyard_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "SWITCHYARD_COMMIT must be a full commit SHA" >&2 + exit 2 +fi + +for dependency in docker git python3; do + command -v "$dependency" >/dev/null || { + echo "missing required command: $dependency" >&2 + exit 1 + } +done +docker info >/dev/null + +docker_architecture="$(docker info --format '{{.Architecture}}')" +if [[ "$target_architecture" != "x86_64" && "$target_architecture" != "aarch64" ]]; then + echo "SWITCHYARD_TARGET_ARCHITECTURE must be x86_64 or aarch64" >&2 + exit 2 +fi +if [[ "$docker_architecture" == "aarch64" || "$docker_architecture" == "arm64" ]]; then + builder_image="${SWITCHYARD_BUILDER_IMAGE:-rust:1.96.1-bookworm@sha256:809725748b728a8e1f8621a3c76e49fba8780c16d99ceda20abdb44d32665c30}" + builder_platform="linux/arm64" + if [[ "$target_architecture" == "x86_64" ]]; then + cargo_target="x86_64-unknown-linux-gnu" + library_path="/tmp/target/x86_64-unknown-linux-gnu/release/libswitchyard_nemo_relay_plugin.so" + else + cargo_target="" + library_path="/tmp/target/release/libswitchyard_nemo_relay_plugin.so" + fi +else + if [[ "$target_architecture" != "x86_64" ]]; then + echo "aarch64 cross-builds from an x86_64 Docker host are not supported" >&2 + exit 2 + fi + builder_image="${SWITCHYARD_BUILDER_IMAGE:-rust:1.96.1-bookworm@sha256:d99f7b31f49909348dc59b51f3c95d1efded1701ffb222f095aaab7de3c4abd8}" + builder_platform="linux/amd64" + cargo_target="" + library_path="/tmp/target/release/libswitchyard_nemo_relay_plugin.so" +fi + +# Stage beside the requested output so source and result use the same +# Docker-shared filesystem. Colima installations often do not share $TMPDIR or +# host /private/tmp even though those paths also exist inside the VM. +output_parent="$(dirname "$output_dir")" +if [[ ! -d "$output_parent" ]]; then + echo "output parent must already exist: $output_parent" >&2 + exit 2 +fi +build_root="$(mktemp -d "$output_parent/.switchyard-relay-plugin.XXXXXX")" +source_dir="$build_root/source" +staging_dir="${output_dir}.partial.$$" +if [[ -e "$staging_dir" ]]; then + echo "refusing to overwrite existing staging directory: $staging_dir" >&2 + exit 2 +fi +mkdir -m 0700 "$staging_dir" +cleanup() { + rm -rf "$build_root" "$staging_dir" +} +trap cleanup EXIT + +git clone --filter=blob:none --no-checkout "$switchyard_repository" "$source_dir" +git -C "$source_dir" fetch --depth 1 origin "$switchyard_commit" +git -C "$source_dir" checkout --detach "$switchyard_commit" +actual_commit="$(git -C "$source_dir" rev-parse HEAD)" +if [[ "$actual_commit" != "$switchyard_commit" ]]; then + echo "Switchyard checkout mismatch: expected $switchyard_commit, got $actual_commit" >&2 + exit 1 +fi + +docker run --rm \ + --platform "$builder_platform" \ + --env PHASE1_CARGO_TARGET="$cargo_target" \ + --env PHASE1_LIBRARY_PATH="$library_path" \ + --volume "$source_dir:/src:ro" \ + --volume "$staging_dir:/out" \ + "$builder_image" \ + bash -lc ' + set -euo pipefail + test -f /src/Cargo.toml + export DEBIAN_FRONTEND=noninteractive + export PATH="/usr/local/cargo/bin:$PATH" + apt-get update + apt-get install -y --no-install-recommends ca-certificates clang cmake pkg-config python3 + if [[ -n "$PHASE1_CARGO_TARGET" ]]; then + apt-get install -y --no-install-recommends crossbuild-essential-amd64 + rustup target add "$PHASE1_CARGO_TARGET" + export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc + export CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc + export CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ + export AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar + fi + mkdir -p /tmp/switchyard + cp -a /src/. /tmp/switchyard/ + cd /tmp/switchyard + export CARGO_TARGET_DIR=/tmp/target + cargo_args=(build --locked --release -p switchyard-nemo-relay-plugin) + if [[ -n "$PHASE1_CARGO_TARGET" ]]; then + cargo_args+=(--target "$PHASE1_CARGO_TARGET") + fi + cargo "${cargo_args[@]}" + python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ + --library "$PHASE1_LIBRARY_PATH" \ + --output /out + ' + +python3 - "$staging_dir" "$switchyard_repository" "$switchyard_commit" "$builder_image" "$builder_platform" "$cargo_target" "$target_architecture" <<'PY' +import hashlib +import json +import pathlib +import sys +import tomllib + +output = pathlib.Path(sys.argv[1]) +repository, commit, builder, builder_platform, cargo_target, target_architecture = sys.argv[2:] +manifest_path = output / "relay-plugin.toml" +if not manifest_path.is_file(): + raise SystemExit("bundle did not contain relay-plugin.toml") +with manifest_path.open("rb") as stream: + manifest = tomllib.load(stream) +if manifest.get("plugin", {}).get("id") != "nvidia.switchyard": + raise SystemExit("bundle manifest has the wrong plugin id") +libraries = [ + path for path in output.iterdir() + if path.is_file() and path.suffix in {".so", ".dylib", ".dll"} +] +if len(libraries) != 1: + raise SystemExit(f"expected one native library, found {len(libraries)}") +with libraries[0].open("rb") as stream: + elf_header = stream.read(20) +if elf_header[:4] != b"\x7fELF" or elf_header[5] != 1: + raise SystemExit("native library must be a little-endian ELF artifact") +machine = int.from_bytes(elf_header[18:20], "little") +expected_machine = {"x86_64": 62, "aarch64": 183}[target_architecture] +if machine != expected_machine: + raise SystemExit( + f"native library architecture mismatch: expected {target_architecture}, ELF e_machine={machine}" + ) + +def digest(path: pathlib.Path) -> str: + value = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + value.update(block) + return value.hexdigest() + +provenance = { + "schema_version": "harbor-hermes-switchyard.bundle.v1", + "repository": repository, + "commit": commit, + "builder_image": builder, + "builder_platform": builder_platform, + "cargo_target": cargo_target or f"native-{target_architecture}", + "target_architecture": target_architecture, + "plugin_id": "nvidia.switchyard", + "manifest_sha256": digest(manifest_path), + "library": libraries[0].name, + "library_sha256": digest(libraries[0]), +} +(output / "bundle-provenance.json").write_text( + json.dumps(provenance, indent=2, sort_keys=True) + "\n", + encoding="utf-8", +) +print(json.dumps(provenance, indent=2)) +PY + +mv "$staging_dir" "$output_dir" diff --git a/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py b/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py new file mode 100755 index 000000000..d85377e94 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small OpenAI Chat-compatible provider used by the offline Phase 1 smoke.""" + +from __future__ import annotations + +import argparse +import json +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +class Handler(BaseHTTPRequestHandler): + token: str + request_log: Path + + def log_message(self, _format: str, *_args: Any) -> None: + return + + def do_GET(self) -> None: # noqa: N802 + if self.path == "/healthz": + self.send_response(200) + self.send_header("content-type", "application/json") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + return + self.send_error(404) + + def do_POST(self) -> None: # noqa: N802 + if self.path != "/v1/chat/completions": + self.send_error(404) + return + if self.headers.get("authorization") != f"Bearer {self.token}": + self.send_error(401) + return + try: + length = int(self.headers.get("content-length", "0")) + request = json.loads(self.rfile.read(length)) + except (ValueError, json.JSONDecodeError): + self.send_error(400) + return + log_entry = { + "path": self.path, + "model": request.get("model"), + "message_count": len(request.get("messages", [])), + "authorization_present": True, + } + with self.request_log.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(log_entry, separators=(",", ":")) + "\n") + response = { + "id": "chatcmpl-phase1", + "object": "chat.completion", + "created": int(time.time()), + "model": request.get("model", "phase1-model"), + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "OFFLINE_SWITCHYARD_OK", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 4, + "completion_tokens": 3, + "total_tokens": 7, + }, + } + body = json.dumps(response).encode("utf-8") + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--bind", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--token", required=True) + parser.add_argument("--request-log", type=Path, required=True) + args = parser.parse_args() + args.request_log.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + Handler.token = args.token + Handler.request_log = args.request_log + ThreadingHTTPServer((args.bind, args.port), Handler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py b/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py new file mode 100755 index 000000000..24312a824 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Accept OTLP/HTTP protobuf requests for the offline compatibility smoke.""" + +from __future__ import annotations + +import argparse +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +class Handler(BaseHTTPRequestHandler): + request_log: Path + + def log_message(self, _format: str, *_args: Any) -> None: + return + + def do_GET(self) -> None: # noqa: N802 + if self.path == "/healthz": + self.send_response(200) + self.end_headers() + return + self.send_error(404) + + def do_POST(self) -> None: # noqa: N802 + if self.path != "/v1/traces": + self.send_error(404) + return + length = int(self.headers.get("content-length", "0")) + body = self.rfile.read(length) + with self.request_log.open("a", encoding="utf-8") as stream: + stream.write( + json.dumps( + { + "path": self.path, + "content_type": self.headers.get("content-type"), + "bytes": len(body), + }, + separators=(",", ":"), + ) + + "\n" + ) + response = b"{}" + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=4318) + parser.add_argument("--request-log", type=Path, required=True) + args = parser.parse_args() + args.request_log.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + Handler.request_log = args.request_log + ThreadingHTTPServer(("127.0.0.1", args.port), Handler).serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py new file mode 100755 index 000000000..a2c390c5e --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Create the direct Hermes result and lifecycle receipt inside a Harbor task.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import time +import tomllib +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = "harbor-hermes-switchyard.phase1.v1" +MAX_DIAGNOSTIC_BYTES = 1024 * 1024 +HERMES_SESSION = Path("/logs/agent/hermes-session.jsonl") +HERMES_LOG = Path("/logs/agent/hermes.txt") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def atomic_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.chmod(temporary, 0o600) + temporary.replace(path) + + +def checked_artifact_root(raw: str) -> Path: + path = Path(raw) + if not path.is_absolute(): + raise ValueError("artifact root must be absolute") + resolved = path.resolve(strict=False) + allowed = Path("/logs/agent").resolve() + if resolved == allowed or allowed not in resolved.parents: + raise ValueError("artifact root must be a child of /logs/agent") + resolved.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chmod(resolved, 0o700) + return resolved + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=("initialize", "complete")) + parser.add_argument("--artifact-root", required=True) + parser.add_argument("--hermes-repository", required=True) + parser.add_argument("--hermes-commit", required=True) + parser.add_argument("--switchyard-commit", required=True) + parser.add_argument("--relay-wheel-sha256", required=True) + parser.add_argument("--relay-config", type=Path, required=True) + parser.add_argument("--switchyard-manifest", type=Path, required=True) + parser.add_argument("--switchyard-library", type=Path, required=True) + parser.add_argument("--session-handle", default="") + parser.add_argument("--started-at", type=float) + parser.add_argument("--error-type", default="") + return parser.parse_args() + + +def initialize(args: argparse.Namespace, root: Path) -> None: + with args.switchyard_manifest.open("rb") as stream: + manifest = tomllib.load(stream) + plugin_id = manifest.get("plugin", {}).get("id") + if plugin_id != "nvidia.switchyard": + raise ValueError(f"unexpected Switchyard plugin id: {plugin_id!r}") + + config_digest = sha256(args.relay_config) + receipt = { + "schema_version": SCHEMA_VERSION, + "status": "initialized", + "activation_mode": "relay_standard_dynamic", + "session_handle": args.session_handle or None, + "dependencies": { + "nemo_relay": { + "version": importlib.metadata.version("nemo-relay"), + "wheel_sha256": args.relay_wheel_sha256, + }, + "hermes": { + "repository": args.hermes_repository, + "commit": args.hermes_commit, + }, + "switchyard": { + "commit": args.switchyard_commit, + "plugin_id": plugin_id, + "manifest_sha256": sha256(args.switchyard_manifest), + "library_sha256": sha256(args.switchyard_library), + }, + }, + "relay_config_sha256": config_digest, + "dynamic_plugin_ids": [plugin_id], + "routing_contract": { + "relay_outer_lifecycle": True, + "execution_intercept_owner": plugin_id, + "provider_http_client_owner": "switchyard-llm-client", + "separate_switchyard_service": False, + }, + "artifacts": { + "root": str(root), + "atof": str(root / "relay" / "trajectory.atof.jsonl"), + "atif_directory": str(root / "relay" / "atif"), + "bounded_diagnostics": str(root / "diagnostics" / "hermes-tail.txt"), + }, + "cleanup": { + "plugin_host_closed": False, + "exporters_flushed": False, + "completion_marker_written": False, + }, + } + if receipt["dependencies"]["nemo_relay"]["version"] != "0.7.0": + raise RuntimeError("Hermes environment did not install nemo-relay==0.7.0") + (root / "relay" / "atif").mkdir(mode=0o700, parents=True, exist_ok=True) + (root / "diagnostics").mkdir(mode=0o700, parents=True, exist_ok=True) + atomic_json(root / "direct-hermes-receipt.json", receipt) + + +def _read_session_messages(path: Path) -> tuple[list[dict[str, Any]], str | None]: + messages: list[dict[str, Any]] = [] + session_id: str | None = None + if not path.is_file(): + return messages, session_id + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(payload, dict): + continue + candidate = payload.get("session_id") or payload.get("id") + if isinstance(candidate, str) and candidate: + session_id = candidate + nested = payload.get("messages") + if isinstance(nested, list): + messages.extend(item for item in nested if isinstance(item, dict)) + elif payload.get("role"): + messages.append(payload) + return messages, session_id + + +def _text_content(content: Any) -> str: + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict) and isinstance(item.get("text"), str): + parts.append(item["text"]) + return "\n".join(parts).strip() + return "" + + +def _last_assistant_response(messages: list[dict[str, Any]]) -> str | None: + for message in reversed(messages): + if message.get("role") == "assistant": + text = _text_content(message.get("content")) + if text: + return text + return None + + +def _write_bounded_diagnostics(root: Path) -> str: + destination = root / "diagnostics" / "hermes-tail.txt" + if not HERMES_LOG.is_file(): + destination.write_text("", encoding="utf-8") + os.chmod(destination, 0o600) + return "" + size = HERMES_LOG.stat().st_size + with HERMES_LOG.open("rb") as stream: + if size > MAX_DIAGNOSTIC_BYTES: + stream.seek(-MAX_DIAGNOSTIC_BYTES, os.SEEK_END) + content = stream.read(MAX_DIAGNOSTIC_BYTES) + destination.write_bytes(content) + os.chmod(destination, 0o600) + return content.decode("utf-8", errors="replace") + + +def complete(args: argparse.Namespace, root: Path) -> None: + receipt_path = root / "direct-hermes-receipt.json" + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + messages, exported_session_id = _read_session_messages(HERMES_SESSION) + response = _last_assistant_response(messages) + diagnostic_text = _write_bounded_diagnostics(root) + lowered = diagnostic_text.lower() + cleanup_failure = any( + marker in lowered + for marker in ( + "plugin configuration cleanup failed", + "plugin subscriber flush failed", + "exporter flush failed", + "plugin teardown failed", + ) + ) + late_failure = bool(args.error_type or cleanup_failure) + if response and late_failure: + status = "preserved_completed_response" + elif response: + status = "completed" + else: + status = "failed" + + ended_at = time.time() + result = { + "schema_version": SCHEMA_VERSION, + "status": status, + "final_response": response, + "session_id": exported_session_id or args.session_handle or None, + "timing": { + "started_at_unix": args.started_at, + "ended_at_unix": ended_at, + "duration_seconds": (max(0.0, ended_at - args.started_at) if args.started_at is not None else None), + }, + "error": ({"type": args.error_type or "RelayCleanupError", "phase": "shutdown"} if late_failure else None), + } + atomic_json(root / "direct-hermes-result.json", result) + + receipt["status"] = "completed" if status != "failed" else "failed" + receipt["result_status"] = status + receipt["cleanup"] = { + "plugin_host_closed": not cleanup_failure, + "exporters_flushed": not cleanup_failure, + "completion_marker_written": True, + "late_failure": late_failure, + } + atomic_json(receipt_path, receipt) + completion = { + "schema_version": SCHEMA_VERSION, + "status": "passed" if status != "failed" else "failed", + "result_status": status, + "completed_at_unix": ended_at, + } + atomic_json(root / "completion.json", completion) + marker = root / ".complete" + marker.write_text("completed\n", encoding="utf-8") + os.chmod(marker, 0o600) + + +def main() -> int: + args = parse_args() + root = checked_artifact_root(args.artifact_root) + if args.mode == "initialize": + initialize(args, root) + else: + complete(args, root) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/native_plugin_loader_smoke.py b/examples/harbor-hermes-switchyard/scripts/native_plugin_loader_smoke.py new file mode 100644 index 000000000..9936fc202 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/native_plugin_loader_smoke.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Load and close the configured native plugin without starting Hermes.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +from pathlib import Path + +from nemo_relay import plugin + + +async def exercise(config: Path) -> dict[str, object]: + specs = plugin.load_dynamic_plugin_activation_specs(config) + if len(specs) != 1 or specs[0].plugin_id != "nvidia.switchyard": + raise AssertionError("expected one nvidia.switchyard activation spec") + host = await plugin.initialize_with_dynamic_plugins( + {"version": 1, "components": []}, + specs, + ) + try: + report = host.report + if not host.is_active: + raise AssertionError("dynamic plugin host did not become active") + return report.to_dict() if hasattr(report, "to_dict") else report + finally: + await host.close() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--plugins", type=Path, required=True) + args = parser.parse_args() + report = asyncio.run(exercise(args.plugins.resolve())) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py b/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py new file mode 100755 index 000000000..c95230858 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise Hermes #77915 and Switchyard #270 against a fake provider.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import threading +from pathlib import Path +from typing import Any + + +async def exercise(model: str, session_id: str) -> tuple[dict[str, Any], dict[str, Any]]: + from agent.relay_runtime import RelayRuntime + + import nemo_relay + + host = RelayRuntime(profile_key="phase1-offline") + session = host.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay runtime did not open a session") + downstream_called = False + + async def forbidden_downstream(_request: Any) -> dict[str, Any]: + nonlocal downstream_called + downstream_called = True + raise AssertionError("Switchyard managed request reached Relay downstream callback") + + request = nemo_relay.LLMRequest( + {}, + { + "model": model, + "messages": [{"role": "user", "content": "reply with the smoke marker"}], + "stream": False, + }, + ) + try: + response = await host.run_in_session_async( + session, + nemo_relay.llm.execute, + "openai.chat_completions", + request, + forbidden_downstream, + model_name=model, + response_codec=nemo_relay.codecs.OpenAIChatCodec(), + ) + active_report = nemo_relay.plugin.report() + if active_report is None: + raise AssertionError("Relay did not expose an active plugin report") + report = active_report.to_dict() if hasattr(active_report, "to_dict") else active_report + host.close_session({"session_id": session_id}) + finally: + host.shutdown() + if downstream_called: + raise AssertionError("Relay downstream callback was invoked") + content = response["choices"][0]["message"]["content"] + if content != "OFFLINE_SWITCHYARD_OK": + raise AssertionError(f"unexpected fake-provider response: {content!r}") + return response, report + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--plugins", type=Path, required=True) + parser.add_argument("--artifacts", type=Path, required=True) + parser.add_argument("--request-log", type=Path, required=True) + parser.add_argument("--model", default="phase1/fake-model") + args = parser.parse_args() + artifacts = args.artifacts.resolve() + artifacts.mkdir(mode=0o700, parents=True, exist_ok=True) + os.environ["HERMES_NEMO_RELAY_PLUGINS_TOML"] = str(args.plugins.resolve()) + session_id = "phase1-offline-session" + response, report = asyncio.run(exercise(args.model, session_id)) + + atof = artifacts / "relay" / "trajectory.atof.jsonl" + atif = sorted((artifacts / "relay" / "atif").glob("trajectory-*.atif.json")) + if not atof.is_file() or atof.stat().st_size == 0: + raise AssertionError("ATOF file sink did not emit") + if not atif: + raise AssertionError("ATIF file sink did not emit") + marks = [] + for line in atof.read_text(encoding="utf-8").splitlines(): + event = json.loads(line) + name = event.get("name") + if isinstance(name, str) and name.startswith("switchyard.routing."): + marks.append(name) + if not marks: + raise AssertionError("Switchyard routing marks were not emitted") + requests = [json.loads(line) for line in args.request_log.read_text(encoding="utf-8").splitlines() if line.strip()] + if len(requests) != 1 or not requests[0].get("authorization_present"): + raise AssertionError("fake provider did not receive exactly one authenticated request") + surviving = [ + thread.name for thread in threading.enumerate() if thread.name.startswith("hermes-nemo-relay-shutdown-") + ] + if surviving: + raise AssertionError(f"Hermes shutdown threads survived: {surviving}") + + result = { + "schema_version": "harbor-hermes-switchyard.offline-smoke.v1", + "status": "passed", + "response": response["choices"][0]["message"]["content"], + "provider_requests": len(requests), + "relay_downstream_callback_called": False, + "switchyard_routing_marks": marks, + "active_plugin_report_before_shutdown": report, + "atof": str(atof), + "atif": [str(path) for path in atif], + "surviving_shutdown_threads": surviving, + } + output = artifacts / "offline-smoke.json" + output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py new file mode 100755 index 000000000..4cd5a3c15 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -0,0 +1,230 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare one immutable Phase 1 run root and render its Relay config.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tomllib +from pathlib import Path +from urllib.parse import urlsplit +from zipfile import ZipFile + +HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" +HERMES_REF = "feat/relay-native-plugin-init" +HERMES_COMMIT = "a07830e086b3055e313b74cc0c8fd5326a4c2c00" +SWITCHYARD_REPOSITORY = "https://github.com/bbednarski9/Switchyard.git" +SWITCHYARD_COMMIT = "8293936a0f5758aa1a782639d485b8b8948cf03e" +RELAY_VERSION = "0.7.0" +ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]*") +SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def checked_url(value: str, name: str) -> str: + parsed = urlsplit(value) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.fragment + ): + raise ValueError(f"{name} must be a credential-free HTTP(S) URL") + return value.rstrip("/") + + +def checked_label(value: str, name: str) -> str: + if not SAFE_LABEL.fullmatch(value): + raise ValueError(f"{name} contains unsupported characters") + return value + + +def download_relay_wheel(destination: Path, architecture: str) -> Path: + destination.mkdir(mode=0o700, parents=True) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "download", + "--only-binary=:all:", + "--no-deps", + "--platform", + f"manylinux2014_{architecture}", + "--implementation", + "cp", + "--python-version", + "311", + "--abi", + "abi3", + "--dest", + str(destination), + f"nemo-relay=={RELAY_VERSION}", + ], + check=True, + ) + wheels = sorted(destination.glob("nemo_relay-0.7.0-*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"expected one Relay wheel, found {len(wheels)}") + return wheels[0] + + +def verify_relay_wheel(path: Path, architecture: str) -> None: + if not path.is_file() or not path.name.startswith("nemo_relay-0.7.0-"): + raise ValueError("Relay wheel must be a nemo_relay-0.7.0 wheel") + if "manylinux" not in path.name or architecture not in path.name: + raise ValueError(f"Relay wheel must target Linux {architecture}") + with ZipFile(path) as wheel: + metadata_names = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")] + if len(metadata_names) != 1: + raise ValueError("Relay wheel has an ambiguous METADATA payload") + metadata = wheel.read(metadata_names[0]).decode("utf-8", errors="strict") + if "Name: nemo-relay\n" not in metadata or "Version: 0.7.0\n" not in metadata: + raise ValueError("Relay wheel metadata does not identify nemo-relay==0.7.0") + + +def verify_native_library(path: Path, architecture: str) -> None: + with path.open("rb") as stream: + header = stream.read(20) + if header[:4] != b"\x7fELF" or len(header) < 20 or header[5] != 1: + raise ValueError("Switchyard native library must be a little-endian ELF artifact") + machine = int.from_bytes(header[18:20], "little") + expected = {"x86_64": 62, "aarch64": 183}[architecture] + if machine != expected: + raise ValueError(f"Switchyard library does not target {architecture}: ELF e_machine={machine}") + + +def render_config(template: Path, output: Path, replacements: dict[str, str]) -> None: + rendered = template.read_text(encoding="utf-8") + for key, value in replacements.items(): + if "\n" in value or "\r" in value: + raise ValueError(f"replacement {key} contains a newline") + rendered = rendered.replace(f"@{key}@", value) + unresolved = sorted(set(re.findall(r"@[A-Z0-9_]+@", rendered))) + if unresolved: + raise ValueError(f"unresolved Relay config placeholders: {unresolved}") + output.write_text(rendered, encoding="utf-8") + os.chmod(output, 0o600) + with output.open("rb") as stream: + tomllib.load(stream) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--switchyard-bundle", type=Path, required=True) + parser.add_argument("--relay-wheel", type=Path) + parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), default="x86_64") + parser.add_argument("--upstream-base-url", required=True) + parser.add_argument("--upstream-auth-env", default="SWITCHYARD_PROVIDER_AUTHORIZATION") + parser.add_argument("--target-model", required=True) + parser.add_argument("--openinference-endpoint", required=True) + parser.add_argument("--phoenix-project", required=True) + parser.add_argument("--eval-cohort", required=True) + args = parser.parse_args() + + example_root = Path(__file__).resolve().parents[1] + run_root = args.run_root.expanduser().resolve() + if run_root.exists(): + raise FileExistsError(f"run root already exists: {run_root}") + run_root.mkdir(mode=0o700, parents=True) + runtime = run_root / "runtime" + artifacts = run_root / "artifacts" + jobs = run_root / "jobs" + for path in (runtime, artifacts, jobs): + path.mkdir(mode=0o700) + + source_bundle = args.switchyard_bundle.expanduser().resolve() + if not (source_bundle / "relay-plugin.toml").is_file(): + raise FileNotFoundError(source_bundle / "relay-plugin.toml") + bundle = runtime / "switchyard-plugin" + shutil.copytree(source_bundle, bundle) + + if args.relay_wheel: + source_wheel = args.relay_wheel.expanduser().resolve() + verify_relay_wheel(source_wheel, args.relay_architecture) + wheel_dir = runtime / "wheels" + wheel_dir.mkdir(mode=0o700) + relay_wheel = wheel_dir / source_wheel.name + shutil.copy2(source_wheel, relay_wheel) + else: + relay_wheel = download_relay_wheel(runtime / "wheels", args.relay_architecture) + verify_relay_wheel(relay_wheel, args.relay_architecture) + + upstream_base_url = checked_url(args.upstream_base_url, "upstream_base_url") + openinference_endpoint = checked_url(args.openinference_endpoint, "openinference_endpoint") + if not ENV_NAME.fullmatch(args.upstream_auth_env): + raise ValueError("upstream_auth_env must be an uppercase environment variable name") + target_model = checked_label(args.target_model, "target_model") + phoenix_project = checked_label(args.phoenix_project, "phoenix_project") + eval_cohort = checked_label(args.eval_cohort, "eval_cohort") + + config_path = runtime / "plugins.toml" + render_config( + example_root / "config" / "relay.toml.in", + config_path, + { + "TARGET_MODEL": target_model, + "HERMES_COMMIT": HERMES_COMMIT, + "OPENINFERENCE_ENDPOINT": openinference_endpoint, + "PHOENIX_PROJECT": phoenix_project, + "EVAL_COHORT": eval_cohort, + "UPSTREAM_BASE_URL": upstream_base_url, + "UPSTREAM_AUTH_ENV": args.upstream_auth_env, + }, + ) + + manifest = bundle / "relay-plugin.toml" + libraries = sorted(path for path in bundle.iterdir() if path.is_file() and path.suffix in {".so", ".dylib", ".dll"}) + if len(libraries) != 1: + raise ValueError("Switchyard bundle must contain exactly one native library") + verify_native_library(libraries[0], args.relay_architecture) + provenance = { + "schema_version": "harbor-hermes-switchyard.phase1.v1", + "nemo_relay": { + "version": RELAY_VERSION, + "architecture": args.relay_architecture, + "wheel": relay_wheel.name, + "wheel_sha256": sha256(relay_wheel), + }, + "hermes": { + "repository": HERMES_REPOSITORY, + "ref": HERMES_REF, + "commit": HERMES_COMMIT, + }, + "switchyard": { + "repository": SWITCHYARD_REPOSITORY, + "commit": SWITCHYARD_COMMIT, + "manifest_sha256": sha256(manifest), + "library": libraries[0].name, + "library_sha256": sha256(libraries[0]), + }, + "relay_config_sha256": sha256(config_path), + "phoenix_project": phoenix_project, + "eval_cohort": eval_cohort, + } + provenance_path = runtime / "provenance.json" + provenance_path.write_text(json.dumps(provenance, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.chmod(provenance_path, 0o600) + print(json.dumps({"run_root": str(run_root), "provenance": provenance}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh new file mode 100755 index 000000000..9184a7ff7 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +run_root="${1:-}" +image="${PHASE1_COMPAT_IMAGE:-python:3.11-bookworm}" +platform="${PHASE1_COMPAT_PLATFORM:-linux/amd64}" +hermes_repository="${HERMES_REPOSITORY:-https://github.com/bbednarski9/hermes-agent.git}" +hermes_ref="${HERMES_REF:-feat/relay-native-plugin-init}" +hermes_commit="${HERMES_COMMIT:-a07830e086b3055e313b74cc0c8fd5326a4c2c00}" + +if [[ -z "$run_root" || "$run_root" != /* ]]; then + echo "usage: $0 /absolute/prepared-run-root" >&2 + exit 2 +fi +for required in \ + "$run_root/runtime/plugins.toml" \ + "$run_root/runtime/provenance.json" \ + "$run_root/runtime/switchyard-plugin/relay-plugin.toml"; do + [[ -f "$required" ]] || { echo "missing prepared runtime file: $required" >&2; exit 1; } +done +command -v docker >/dev/null || { echo "docker is required" >&2; exit 1; } +docker info >/dev/null +case "$platform" in + linux/amd64) expected_architecture=x86_64 ;; + linux/arm64) expected_architecture=aarch64 ;; + *) echo "PHASE1_COMPAT_PLATFORM must be linux/amd64 or linux/arm64" >&2; exit 2 ;; +esac +prepared_architecture="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["nemo_relay"].get("architecture", "x86_64"))' "$run_root/runtime/provenance.json")" +[[ "$prepared_architecture" == "$expected_architecture" ]] || { + echo "prepared Relay architecture $prepared_architecture does not match $platform" >&2 + exit 2 +} + +artifacts="$run_root/artifacts/offline-compatibility" +if [[ -e "$artifacts" ]]; then + [[ -d "$artifacts" ]] || { echo "artifact path is not a directory: $artifacts" >&2; exit 1; } + [[ -z "$(find "$artifacts" -mindepth 1 -maxdepth 1 -print -quit)" ]] || { + echo "offline compatibility artifacts already exist: $artifacts" >&2 + exit 1 + } +else + mkdir -m 0700 "$artifacts" +fi + +docker run --rm \ + --platform "$platform" \ + --volume "$example_root:/example:ro" \ + --volume "$run_root/runtime:/runtime:ro" \ + --volume "$run_root/runtime/switchyard-plugin:/opt/relay-plugins/nvidia.switchyard:ro" \ + --volume "$artifacts:/logs/agent/direct-hermes" \ + "$image" \ + bash -lc ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + export HERMES_HOME=/tmp/hermes + export HERMES_NEMO_RELAY_PLUGINS_TOML=/runtime/plugins.toml + export SWITCHYARD_PROVIDER_AUTHORIZATION="Bearer phase1-offline-secret-value" + apt-get update + apt-get install -y --no-install-recommends build-essential ca-certificates curl git ripgrep xz-utils + git clone --no-tags --branch "'"$hermes_ref"'" "'"$hermes_repository"'" /tmp/hermes-agent-src + git -C /tmp/hermes-agent-src fetch --depth 1 origin "'"$hermes_commit"'" + git -C /tmp/hermes-agent-src checkout --detach "'"$hermes_commit"'" + # The installer treats ffmpeg as optional, but a root-owned Debian smoke + # otherwise installs ~500 MB of unrelated TTS/video packages. Advertise a + # command only during installation; it is not on PATH for the runtime. + mkdir /tmp/hermes-install-path + ln -s /bin/true /tmp/hermes-install-path/ffmpeg + HERMES_INSTALL_DIR=/tmp/hermes-agent-src \ + PATH=/tmp/hermes-install-path:$PATH \ + bash /tmp/hermes-agent-src/scripts/install.sh \ + --skip-setup --skip-browser --no-skills \ + --dir /tmp/hermes-agent-src \ + --branch "'"$hermes_ref"'" \ + --commit "'"$hermes_commit"'" --force-commit + test "$(git -C /tmp/hermes-agent-src rev-parse HEAD)" = "'"$hermes_commit"'" + cd /tmp/hermes-agent-src + UV_PROJECT_ENVIRONMENT=/tmp/hermes-agent-src/venv \ + /tmp/hermes/bin/uv sync --frozen --extra all + cd / + /tmp/hermes-agent-src/venv/bin/python -c \ + "import importlib.metadata as m; assert m.version(\"nemo-relay\") == \"0.7.0\"" + relay_wheel="$(find /runtime/wheels -maxdepth 1 -type f -name "nemo_relay-0.7.0-*.whl" -print)" + test -n "$relay_wheel" + expected_wheel_sha="$(python3 -c "import json; print(json.load(open(\"/runtime/provenance.json\"))[\"nemo_relay\"][\"wheel_sha256\"])")" + test "$(sha256sum "$relay_wheel" | cut -d" " -f1)" = "$expected_wheel_sha" + /tmp/hermes/bin/uv pip install \ + --python /tmp/hermes-agent-src/venv/bin/python \ + --force-reinstall --no-deps "$relay_wheel" + python3 /example/scripts/fake_openai_upstream.py \ + --token phase1-offline-secret-value \ + --request-log /logs/agent/direct-hermes/provider-requests.jsonl & + provider_pid=$! + python3 /example/scripts/fake_otlp_collector.py \ + --request-log /logs/agent/direct-hermes/otlp-requests.jsonl & + otlp_pid=$! + cleanup() { + kill "$provider_pid" "$otlp_pid" >/dev/null 2>&1 || true + wait "$provider_pid" "$otlp_pid" >/dev/null 2>&1 || true + } + trap cleanup EXIT + for endpoint in http://127.0.0.1:8000/healthz http://127.0.0.1:4318/healthz; do + for _ in $(seq 1 50); do + curl --fail --silent "$endpoint" >/dev/null && break + sleep 0.1 + done + curl --fail --silent "$endpoint" >/dev/null + done + PYTHONPATH=/tmp/hermes-agent-src \ + /tmp/hermes-agent-src/venv/bin/python \ + /example/scripts/offline_compatibility_smoke.py \ + --plugins /runtime/plugins.toml \ + --artifacts /logs/agent/direct-hermes \ + --request-log /logs/agent/direct-hermes/provider-requests.jsonl + test -s /logs/agent/direct-hermes/otlp-requests.jsonl + if grep -R -F phase1-offline-secret-value /logs/agent/direct-hermes >/dev/null; then + echo "offline secret leaked into persisted evidence" >&2 + exit 1 + fi + ' + +python3 - "$artifacts" <<'PY' +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +result = json.loads((root / "offline-smoke.json").read_text()) +if result.get("status") != "passed": + raise SystemExit("offline compatibility smoke did not pass") +print(json.dumps(result, indent=2)) +PY diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase1_regressions.sh b/examples/harbor-hermes-switchyard/scripts/run_phase1_regressions.sh new file mode 100755 index 000000000..a9583f709 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_phase1_regressions.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +regression_root="${1:-}" + +if [[ -z "$regression_root" || "$regression_root" != /* ]]; then + echo "usage: $0 /absolute/new-regression-root" >&2 + exit 2 +fi +if [[ -e "$regression_root" ]]; then + echo "regression root already exists: $regression_root" >&2 + exit 2 +fi +mkdir -m 0700 "$regression_root" + +tasks=( + adaptive-rejection-sampler + circuit-fibsqrt + gpt2-codegolf + overfull-hbox +) + +for task in "${tasks[@]}"; do + echo "Running Phase 1 regression: $task" + run_root="$regression_root/$task" + inject=false + if [[ "$task" == "circuit-fibsqrt" ]]; then + inject=true + fi + TASK_NAME="$task" \ + PHOENIX_PROJECT="${PHOENIX_PROJECT:-harbor-hermes-switchyard-phase1}-$task" \ + EVAL_COHORT="${EVAL_COHORT:-harbor-hermes-switchyard-phase1}-$task" \ + INJECT_POST_RESPONSE_FAILURE="$inject" \ + "$example_root/run_terminal_bench.sh" "$run_root" +done + +python_bin="${PHASE1_PYTHON:-python3}" +"$python_bin" - "$regression_root" "${tasks[@]}" <<'PY' +import json +import pathlib +import sys + +root = pathlib.Path(sys.argv[1]) +tasks = sys.argv[2:] +summaries = [] +for task in tasks: + summary_path = root / task / "summary.json" + if not summary_path.is_file(): + raise SystemExit(f"missing summary: {summary_path}") + summary = json.loads(summary_path.read_text(encoding="utf-8")) + if summary.get("status") != "passed": + raise SystemExit(f"regression did not pass: {task}") + summaries.append(summary) + +result = { + "schema_version": "harbor-hermes-switchyard.phase1-regressions.v1", + "status": "passed", + "planned": len(tasks), + "completed": len(summaries), + "tasks": tasks, +} +(root / "summary.json").write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" +) +print(json.dumps(result, indent=2)) +PY diff --git a/examples/harbor-hermes-switchyard/scripts/upload_openinference.py b/examples/harbor-hermes-switchyard/scripts/upload_openinference.py new file mode 100755 index 000000000..b1dcddb1e --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/upload_openinference.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stream bounded OTLP JSON batches into a Phoenix HTTP receiver.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +from google.protobuf import json_format +from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 + + +def _serialize(resource_spans: list[dict[str, Any]]) -> bytes: + request = trace_service_pb2.ExportTraceServiceRequest() + json_format.ParseDict({"resourceSpans": resource_spans}, request) + return request.SerializeToString() + + +def _hex_id_to_base64(span: dict[str, Any], field: str, width: int) -> None: + value = span.get(field) + if isinstance(value, str) and re.fullmatch(rf"[0-9a-fA-F]{{{width}}}", value): + span[field] = base64.b64encode(bytes.fromhex(value)).decode("ascii") + + +def _normalize_ids(payload: dict[str, Any]) -> None: + for resource_span in payload.get("resourceSpans", []): + for scope_span in resource_span.get("scopeSpans", []): + for span in scope_span.get("spans", []): + _hex_id_to_base64(span, "traceId", 32) + _hex_id_to_base64(span, "spanId", 16) + _hex_id_to_base64(span, "parentSpanId", 16) + + +def _set_project(resource_span: dict[str, Any], project: str) -> None: + attributes = resource_span.setdefault("resource", {}).setdefault("attributes", []) + for attribute in attributes: + if attribute.get("key") == "openinference.project.name": + attribute["value"] = {"stringValue": project} + return + attributes.append( + { + "key": "openinference.project.name", + "value": {"stringValue": project}, + } + ) + + +def _post(endpoint: str, body: bytes, timeout: float, attempts: int) -> int: + for attempt in range(1, attempts + 1): + request = urllib.request.Request( + endpoint, + data=body, + headers={"content-type": "application/x-protobuf"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + if response.status != 200: + raise RuntimeError(f"Phoenix returned HTTP {response.status}") + return attempt - 1 + except (TimeoutError, urllib.error.URLError): + if attempt == attempts: + raise + time.sleep(2 ** (attempt - 1)) + raise AssertionError("unreachable") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--openinference", type=Path, required=True) + parser.add_argument("--phoenix-url", required=True) + parser.add_argument("--project", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--batch-size", type=int, default=8) + parser.add_argument("--max-batch-bytes", type=int, default=1024 * 1024) + parser.add_argument("--timeout-seconds", type=float, default=60) + parser.add_argument("--max-attempts", type=int, default=3) + args = parser.parse_args() + if args.batch_size < 1 or args.max_batch_bytes < 1 or args.max_attempts < 1: + raise ValueError("batch and retry limits must be positive") + + endpoint = args.phoenix_url.rstrip("/") + "/v1/traces" + pending: list[dict[str, Any]] = [] + pending_documents = 0 + uploaded_documents = 0 + uploaded_batches = 0 + uploaded_spans = 0 + retries = 0 + + def upload(items: list[dict[str, Any]]) -> None: + nonlocal uploaded_batches, retries + retries += _post( + endpoint, + _serialize(items), + args.timeout_seconds, + args.max_attempts, + ) + uploaded_batches += 1 + + with args.openinference.open(encoding="utf-8") as stream: + for line in stream: + if not line.strip(): + continue + payload: dict[str, Any] = json.loads(line) + _normalize_ids(payload) + resource_spans = list(payload.get("resourceSpans", [])) + for resource_span in resource_spans: + _set_project(resource_span, args.project) + uploaded_spans += sum(len(scope.get("spans", [])) for scope in resource_span.get("scopeSpans", [])) + candidate = pending + resource_spans + if pending and (pending_documents >= args.batch_size or len(_serialize(candidate)) > args.max_batch_bytes): + upload(pending) + pending = resource_spans + pending_documents = 1 + else: + pending = candidate + pending_documents += 1 + uploaded_documents += 1 + if pending: + upload(pending) + if uploaded_documents == 0 or uploaded_spans == 0: + raise RuntimeError("OpenInference artifact did not contain any spans") + + result = { + "status": "passed", + "project": args.project, + "endpoint": endpoint, + "uploaded_documents": uploaded_documents, + "uploaded_batches": uploaded_batches, + "upload_retries": retries, + "uploaded_spans": uploaded_spans, + } + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py new file mode 100755 index 000000000..34f9df30c --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate one Harbor/Hermes/Switchyard Phase 1 task evidence set.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any, Iterable + +SCHEMA_VERSION = "harbor-hermes-switchyard.validation.v1" + + +def read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"expected a JSON object: {path}") + return value + + +def contained_files(root: Path) -> list[Path]: + resolved_root = root.resolve(strict=True) + files: list[Path] = [] + for path in root.rglob("*"): + if path.is_symlink(): + raise ValueError(f"artifact symlink is forbidden: {path}") + if not path.is_file(): + continue + resolved = path.resolve(strict=True) + if resolved_root not in resolved.parents: + raise ValueError(f"artifact escaped its root: {path}") + files.append(path) + return files + + +def scan_files(root: Path) -> list[Path]: + """Return regular files below a scan root without following symlinks.""" + if not root.is_dir(): + return [] + files: list[Path] = [] + for path in root.rglob("*"): + if path.is_symlink(): + continue + if path.is_file(): + files.append(path) + return files + + +def scan_secrets(files: Iterable[Path], values: list[bytes]) -> list[str]: + findings: list[str] = [] + for path in files: + with path.open("rb") as stream: + overlap = b"" + while True: + block = stream.read(1024 * 1024) + if not block: + break + haystack = overlap + block + for index, secret in enumerate(values): + if secret and secret in haystack: + findings.append(f"{path.name}:secret[{index}]") + max_width = max((len(secret) for secret in values), default=1) + overlap = haystack[-max_width:] + return sorted(set(findings)) + + +def read_atof(path: Path) -> tuple[int, list[str], list[str]]: + count = 0 + marks: list[str] = [] + models: list[str] = [] + with path.open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + if not line.strip(): + continue + payload = json.loads(line) + if not isinstance(payload, dict): + raise ValueError(f"ATOF line {line_number} is not an object") + count += 1 + name = payload.get("name") + if isinstance(name, str) and name.startswith("switchyard.routing."): + marks.append(name) + for container in (payload.get("data"), payload.get("metadata")): + if isinstance(container, dict): + for key in ("model", "selected_model", "target_model"): + value = container.get(key) + if isinstance(value, str) and value: + models.append(value) + return count, sorted(set(marks)), sorted(set(models)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--artifacts", type=Path, required=True) + parser.add_argument("--provenance", type=Path, required=True) + parser.add_argument("--openinference", type=Path, required=True) + parser.add_argument("--harbor-job-dir", type=Path) + parser.add_argument("--scan-root", type=Path, action="append", default=[]) + parser.add_argument("--expect-late-failure", action="store_true") + parser.add_argument("--secret-env", action="append", default=[]) + parser.add_argument("--secret-file", type=Path, action="append", default=[]) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + errors: list[str] = [] + root = args.artifacts.resolve() + try: + files = contained_files(root) + except Exception as error: + files = [] + errors.append(str(error)) + + required = { + "result": root / "direct-hermes-result.json", + "receipt": root / "direct-hermes-receipt.json", + "completion": root / "completion.json", + "atof": root / "relay" / "trajectory.atof.jsonl", + } + for name, path in required.items(): + if not path.is_file(): + errors.append(f"missing {name}: {path}") + atif_files = sorted((root / "relay" / "atif").glob("trajectory-*.atif.json")) + if not atif_files: + errors.append("missing ATIF trajectory") + if not args.openinference.is_file() or args.openinference.stat().st_size == 0: + errors.append("missing OpenInference OTLP artifact") + + result: dict[str, Any] = {} + receipt: dict[str, Any] = {} + provenance: dict[str, Any] = {} + if required["result"].is_file(): + result = read_json(required["result"]) + if result.get("status") not in {"completed", "preserved_completed_response"}: + errors.append(f"invalid direct result status: {result.get('status')!r}") + if not isinstance(result.get("final_response"), str) or not result["final_response"]: + errors.append("direct result has no normalized final response") + if args.expect_late_failure: + if result.get("status") != "preserved_completed_response": + errors.append("expected a preserved completed response after late failure") + if result.get("error", {}).get("type") != "InjectedPostResponseFailure": + errors.append("deterministic post-response failure was not recorded") + if required["receipt"].is_file(): + receipt = read_json(required["receipt"]) + if args.provenance.is_file(): + provenance = read_json(args.provenance) + else: + errors.append("missing runtime provenance") + + if receipt: + dependencies = receipt.get("dependencies", {}) + relay = dependencies.get("nemo_relay", {}) + hermes = dependencies.get("hermes", {}) + switchyard = dependencies.get("switchyard", {}) + if relay.get("version") != "0.7.0": + errors.append("receipt did not record nemo-relay==0.7.0") + if relay.get("wheel_sha256") != provenance.get("nemo_relay", {}).get("wheel_sha256"): + errors.append("Relay wheel digest does not match runtime provenance") + if hermes.get("commit") != provenance.get("hermes", {}).get("commit"): + errors.append("Hermes commit does not match runtime provenance") + if switchyard.get("commit") != provenance.get("switchyard", {}).get("commit"): + errors.append("Switchyard commit does not match runtime provenance") + if receipt.get("dynamic_plugin_ids") != ["nvidia.switchyard"]: + errors.append("receipt did not record only nvidia.switchyard") + if receipt.get("activation_mode") != "relay_standard_dynamic": + errors.append("receipt did not record standard dynamic activation") + cleanup = receipt.get("cleanup", {}) + if not cleanup.get("plugin_host_closed") or not cleanup.get("exporters_flushed"): + errors.append("receipt did not prove plugin close and exporter flush") + + event_count = 0 + routing_marks: list[str] = [] + routed_models: list[str] = [] + if required["atof"].is_file(): + event_count, routing_marks, routed_models = read_atof(required["atof"]) + if event_count == 0: + errors.append("ATOF artifact is empty") + if not routing_marks: + errors.append("ATOF artifact has no Switchyard routing evidence") + + secret_values: list[bytes] = [] + for name in args.secret_env: + value = os.environ.get(name) + if value: + secret_values.append(value.encode()) + for path in args.secret_file: + for line in path.read_bytes().splitlines(): + value = line.split(b"=", 1)[-1].strip() + if value: + secret_values.append(value) + files_to_scan = list(files) + if args.openinference.is_file(): + files_to_scan.append(args.openinference) + for scan_root in args.scan_root: + files_to_scan.extend(scan_files(scan_root.resolve())) + findings = scan_secrets(sorted(set(files_to_scan)), secret_values) + if findings: + errors.append(f"secret scan found {len(findings)} persisted value(s)") + + harbor_results: list[Path] = [] + benchmark_passed: bool | None = None + if args.harbor_job_dir: + harbor_results = sorted(args.harbor_job_dir.glob("**/result.json")) + if len(harbor_results) != 1: + errors.append(f"expected one Harbor trial result, found {len(harbor_results)}") + elif harbor_results: + harbor_result = read_json(harbor_results[0]) + reward = harbor_result.get("reward") + if isinstance(reward, dict): + candidate = reward.get("task_passed") + benchmark_passed = candidate if isinstance(candidate, bool) else None + + validation = { + "schema_version": SCHEMA_VERSION, + "status": "passed" if not errors else "failed", + "errors": errors, + "direct_result_status": result.get("status"), + "harbor_trial_count": len(harbor_results) if args.harbor_job_dir else None, + "benchmark_task_passed": benchmark_passed, + "atof_event_count": event_count, + "atif_trajectory_count": len(atif_files), + "switchyard_routing_marks": routing_marks, + "routed_models": routed_models, + "secret_values_scanned": len(secret_values), + "secret_findings": findings, + } + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(validation, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(validation, indent=2)) + return 0 if not errors else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/verify_harbor_hermes_compat.py b/examples/harbor-hermes-switchyard/scripts/verify_harbor_hermes_compat.py new file mode 100755 index 000000000..92fede137 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/verify_harbor_hermes_compat.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verify that the temporary agent is a narrow Harbor Hermes compatibility bridge.""" + +from __future__ import annotations + +import argparse +import ast +import importlib.metadata +import importlib.util +import json +import tempfile +from pathlib import Path + +from harbor.agents.installed.hermes import Hermes + + +def load_bridge(path: Path): + spec = importlib.util.spec_from_file_location("harbor_hermes_agent", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not import {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def run_mixed_mode_rejection(module, valid_config: Path) -> str: + text = valid_config.read_text(encoding="utf-8") + text += """ + +[[dynamic_plugins]] +plugin_id = "invalid.worker" +kind = "worker" +manifest_ref = "worker/relay-plugin.toml" +environment_ref = "worker-env" +""" + with tempfile.TemporaryDirectory(prefix="harbor-hermes-mixed-") as directory: + path = Path(directory) / "plugins.toml" + path.write_text(text, encoding="utf-8") + try: + module._validate_relay_config(path) + except ValueError as error: + return str(error) + raise AssertionError("mixed standard and worker plugin modes were accepted") + + +def verify_run_wrapper(source: str) -> None: + tree = ast.parse(source) + classes = [node for node in tree.body if isinstance(node, ast.ClassDef)] + bridge = next(node for node in classes if node.name == "HarborHermesAgent") + run_method = next( + node for node in bridge.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "run" + ) + super_run_calls = [ + node + for node in ast.walk(run_method) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "run" + and isinstance(node.func.value, ast.Call) + and isinstance(node.func.value.func, ast.Name) + and node.func.value.func.id == "super" + ] + if len(super_run_calls) != 1: + raise AssertionError("bridge run() must delegate exactly once to super().run()") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--bridge", type=Path, required=True) + parser.add_argument("--relay-config", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + module = load_bridge(args.bridge.resolve()) + bridge = module.HarborHermesAgent + if not issubclass(bridge, Hermes): + raise AssertionError("HarborHermesAgent must subclass Harbor's built-in Hermes") + if bridge._build_config_yaml is not Hermes._build_config_yaml: + raise AssertionError("bridge must inherit Harbor's Hermes config.yaml behavior") + if bridge.populate_context_post_run is not Hermes.populate_context_post_run: + raise AssertionError("bridge must inherit Harbor's ATIF conversion behavior") + verify_run_wrapper(args.bridge.read_text(encoding="utf-8")) + module._validate_relay_config(args.relay_config.resolve()) + mixed_error = run_mixed_mode_rejection(module, args.relay_config.resolve()) + + result = { + "schema_version": "harbor-hermes-switchyard.compatibility.v1", + "status": "passed", + "harbor_version": importlib.metadata.version("harbor"), + "bridge_base": "harbor.agents.installed.hermes.Hermes", + "inherited_contracts": [ + "task lifecycle", + "provider environment", + "prompt rendering", + "Hermes session export", + "Harbor ATIF conversion", + ], + "overrides": ["installation", "configuration staging", "artifact framing"], + "mixed_mode_rejection": mixed_error, + } + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py new file mode 100644 index 000000000..b0d310f91 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import importlib.util +import json +from pathlib import Path + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def load_finalizer(): + path = EXAMPLE_ROOT / "scripts" / "finalize_artifacts.py" + spec = importlib.util.spec_from_file_location("phase1_finalizer", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def make_args(tmp_path: Path, *, error_type: str = "") -> argparse.Namespace: + config = tmp_path / "plugins.toml" + config.write_text("version = 1\n", encoding="utf-8") + manifest = tmp_path / "relay-plugin.toml" + manifest.write_text('[plugin]\nid = "nvidia.switchyard"\n', encoding="utf-8") + library = tmp_path / "libswitchyard.so" + library.write_bytes(b"native-plugin-test") + return argparse.Namespace( + relay_config=config, + switchyard_manifest=manifest, + switchyard_library=library, + relay_wheel_sha256="a" * 64, + hermes_repository="https://github.com/bbednarski9/hermes-agent.git", + hermes_commit="a07830e086b3055e313b74cc0c8fd5326a4c2c00", + switchyard_commit="8293936a0f5758aa1a782639d485b8b8948cf03e", + session_handle="phase1-session", + started_at=1.0, + error_type=error_type, + ) + + +def test_completed_response_is_preserved_after_post_response_failure(tmp_path: Path, monkeypatch) -> None: + module = load_finalizer() + root = tmp_path / "artifacts" + root.mkdir() + session = tmp_path / "hermes-session.jsonl" + session.write_text( + json.dumps( + { + "session_id": "phase1-session", + "messages": [ + {"role": "user", "content": "solve"}, + {"role": "assistant", "content": "completed answer"}, + ], + } + ) + + "\n", + encoding="utf-8", + ) + log = tmp_path / "hermes.txt" + log.write_text("normal shutdown\n", encoding="utf-8") + monkeypatch.setattr(module, "HERMES_SESSION", session) + monkeypatch.setattr(module, "HERMES_LOG", log) + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.0") + + args = make_args(tmp_path, error_type="InjectedPostResponseFailure") + module.initialize(args, root) + module.complete(args, root) + + result = json.loads((root / "direct-hermes-result.json").read_text()) + receipt = json.loads((root / "direct-hermes-receipt.json").read_text()) + assert result["status"] == "preserved_completed_response" + assert result["final_response"] == "completed answer" + assert result["error"]["type"] == "InjectedPostResponseFailure" + assert receipt["cleanup"]["plugin_host_closed"] is True + assert receipt["cleanup"]["exporters_flushed"] is True + assert (root / ".complete").read_text() == "completed\n" + + +def test_no_response_never_creates_a_passed_completion(tmp_path: Path, monkeypatch) -> None: + module = load_finalizer() + root = tmp_path / "artifacts" + root.mkdir() + session = tmp_path / "hermes-session.jsonl" + session.write_text('{"messages": []}\n', encoding="utf-8") + log = tmp_path / "hermes.txt" + log.write_text("agent stopped\n", encoding="utf-8") + monkeypatch.setattr(module, "HERMES_SESSION", session) + monkeypatch.setattr(module, "HERMES_LOG", log) + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.0") + + args = make_args(tmp_path) + module.initialize(args, root) + module.complete(args, root) + + completion = json.loads((root / "completion.json").read_text()) + assert completion["status"] == "failed" diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py new file mode 100644 index 000000000..e4615bb94 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_config_contract.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import re +import tomllib +from pathlib import Path + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def render_template(**values: str) -> dict: + text = (EXAMPLE_ROOT / "config" / "relay.toml.in").read_text(encoding="utf-8") + defaults = { + "TARGET_MODEL": "phase1-test-model", + "HERMES_COMMIT": "a07830e086b3055e313b74cc0c8fd5326a4c2c00", + "OPENINFERENCE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", + "PHOENIX_PROJECT": "phase1-test", + "EVAL_COHORT": "phase1-test", + "UPSTREAM_BASE_URL": "http://127.0.0.1:8000/v1", + "UPSTREAM_AUTH_ENV": "SWITCHYARD_PROVIDER_AUTHORIZATION", + } + defaults.update(values) + for key, value in defaults.items(): + text = text.replace(f"@{key}@", value) + assert not re.search(r"@[A-Z0-9_]+@", text) + return tomllib.loads(text) + + +def test_config_uses_static_schema_v3_and_one_standard_dynamic_plugin() -> None: + config = render_template() + assert config["version"] == 1 + components = {item["kind"]: item for item in config["components"]} + assert components["pricing"]["enabled"] is True + assert components["observability"]["config"]["version"] == 3 + assert "dynamic_plugins" not in config + assert len(config["plugins"]["dynamic"]) == 1 + plugin = config["plugins"]["dynamic"][0] + assert plugin["manifest"].endswith("/nvidia.switchyard/relay-plugin.toml") + assert plugin["config"]["targets"]["primary"]["header_env"] == { + "authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION" + } + + +def test_config_contains_no_literal_provider_headers_or_credentials() -> None: + config = render_template() + + def walk(value: object) -> None: + if isinstance(value, dict): + assert "headers" not in value + for nested in value.values(): + walk(nested) + elif isinstance(value, list): + for nested in value: + walk(nested) + + walk(config) + + +def test_pricing_does_not_duplicate_relay_generated_aliases() -> None: + config = render_template(TARGET_MODEL="namespace/model") + entry = config["components"][0]["config"]["sources"][0]["catalog"]["entries"][0] + assert entry["model_id"] == "namespace/model" + assert "aliases" not in entry diff --git a/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py new file mode 100644 index 000000000..ed4e25ab0 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import ast +from pathlib import Path + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] +BRIDGE = EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py" + + +def bridge_class() -> ast.ClassDef: + tree = ast.parse(BRIDGE.read_text(encoding="utf-8")) + return next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "HarborHermesAgent") + + +def method(name: str) -> ast.AsyncFunctionDef: + node = next(item for item in bridge_class().body if isinstance(item, ast.AsyncFunctionDef) and item.name == name) + return node + + +def super_calls(node: ast.AST, attribute: str) -> list[ast.Call]: + return [ + item + for item in ast.walk(node) + if isinstance(item, ast.Call) + and isinstance(item.func, ast.Attribute) + and item.func.attr == attribute + and isinstance(item.func.value, ast.Call) + and isinstance(item.func.value.func, ast.Name) + and item.func.value.func.id == "super" + ] + + +def test_bridge_delegates_exactly_once_to_each_inherited_lifecycle_phase() -> None: + assert len(super_calls(method("setup"), "setup")) == 1 + assert len(super_calls(method("run"), "run")) == 1 + + +def test_run_frames_artifacts_in_finally_after_inherited_run() -> None: + run = method("run") + tries = [item for item in run.body if isinstance(item, ast.Try)] + assert len(tries) == 1 + lifecycle = tries[0] + assert super_calls(lifecycle, "run") + assert any( + isinstance(item, ast.Call) and isinstance(item.func, ast.Attribute) and item.func.attr == "exec_as_agent" + for final in lifecycle.finalbody + for item in ast.walk(final) + ) + + +def test_install_verifies_detached_commit_and_relay_release() -> None: + source = ast.unparse(method("install")) + assert "checkout --detach" in source + assert "rev-parse HEAD" in source + assert "uv sync --frozen --extra all" in source + assert "m.version('nemo-relay') == '0.7.0'" in source From cd065953255d27943c9404c1cd20b758ce8d000a Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 5 Aug 2026 18:35:23 -0600 Subject: [PATCH 02/34] fix(examples): harden Harbor Hermes validation Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 9 ++++++ .../agents/harbor_hermes_agent.py | 23 ++++++++++++-- .../run_terminal_bench.sh | 17 ++++++++-- .../scripts/fake_otlp_collector.py | 2 +- .../scripts/finalize_artifacts.py | 20 ++++++++++++ .../scripts/validate_run.py | 31 ++++++++++++++++--- .../tests/test_agent_result_contract.py | 22 +++++++++++++ .../tests/test_config_contract.py | 8 +++++ .../tests/test_plugin_lifecycle.py | 1 + .../tests/test_validation_contract.py | 30 ++++++++++++++++++ 10 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 examples/harbor-hermes-switchyard/tests/test_validation_contract.py diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index e39579da8..c9a6eed59 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -24,6 +24,9 @@ execution. The Hermes installer is followed by a final `uv sync --frozen` against that commit's checked-in lock because its date-relative resolution guard can otherwise make an older checkout appear stale. The verified Relay 0.7.0 platform wheel is then force-installed by digest without dependencies. +During installation only, the bridge advertises an inert `ffmpeg` command so +the task does not install an unrelated media stack; browser setup and bundled +skills are also disabled for this terminal-only evaluation. ## Request and lifecycle ownership @@ -145,6 +148,12 @@ That control validates the same source commits and lifecycle on a different released Relay wheel architecture. It does **not** replace a passing `linux/amd64` run on native amd64 infrastructure before merge. +`run_terminal_bench.sh` also accepts `RELAY_ARCHITECTURE=aarch64` together +with matching `SWITCHYARD_BUNDLE` and `RELAY_WHEEL` inputs. This is useful for +exercising Harbor's complete bridge and artifact path on a native Apple +Silicon Docker daemon. It remains a diagnostic control; the default and merge +gate stay `x86_64`. + ## Run one Terminal-Bench task Use a new absolute run root on every invocation: diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py index e76e5ba93..676e41a1a 100644 --- a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -55,6 +55,17 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _verify_elf_architecture(path: Path, architecture: str) -> None: + with path.open("rb") as stream: + header = stream.read(20) + if header[:4] != b"\x7fELF" or len(header) < 20 or header[5] != 1: + raise ValueError("Switchyard native library must be a little-endian ELF artifact") + expected_machine = {"x86_64": 62, "aarch64": 183}[architecture] + machine = int.from_bytes(header[18:20], "little") + if machine != expected_machine: + raise ValueError(f"Switchyard native library does not target {architecture}: ELF e_machine={machine}") + + def _require_public_https_git_url(value: str) -> str: parsed = urlsplit(value) if ( @@ -130,6 +141,7 @@ def __init__( switchyard_bundle_dir: str, relay_wheel_path: str, relay_wheel_sha256: str, + relay_architecture: str = "x86_64", switchyard_commit: str = _DEFAULT_SWITCHYARD_COMMIT, artifact_root: str = "/logs/agent/direct-hermes", inject_post_response_failure: bool = False, @@ -142,6 +154,9 @@ def __init__( self.commit = _require_full_sha(commit, "commit") self.switchyard_commit = _require_full_sha(switchyard_commit, "switchyard_commit") self.relay_wheel_sha256 = _require_sha256(relay_wheel_sha256, "relay_wheel_sha256") + if relay_architecture not in {"x86_64", "aarch64"}: + raise ValueError("relay_architecture must be x86_64 or aarch64") + self.relay_architecture = relay_architecture self.relay_config_path = Path(relay_config_path).expanduser().resolve() self.switchyard_bundle_dir = Path(switchyard_bundle_dir).expanduser().resolve() @@ -156,8 +171,8 @@ def __init__( raise FileNotFoundError(self.relay_wheel_path) if _sha256(self.relay_wheel_path) != self.relay_wheel_sha256: raise ValueError("Relay wheel digest does not match relay_wheel_sha256") - if "manylinux" not in self.relay_wheel_path.name or "x86_64" not in self.relay_wheel_path.name: - raise ValueError("Relay wheel must target Linux x86_64") + if "manylinux" not in self.relay_wheel_path.name or relay_architecture not in self.relay_wheel_path.name: + raise ValueError(f"Relay wheel must target Linux {relay_architecture}") _validate_relay_config(self.relay_config_path) self.switchyard_manifest = self.switchyard_bundle_dir / "relay-plugin.toml" @@ -171,6 +186,7 @@ def __init__( if len(libraries) != 1: raise ValueError("Switchyard bundle must contain exactly one native library") self.switchyard_library = libraries[0] + _verify_elf_architecture(self.switchyard_library, relay_architecture) self._example_root = Path(__file__).resolve().parents[1] self._finalizer_path = self._example_root / "scripts" / "finalize_artifacts.py" @@ -205,7 +221,10 @@ async def install(self, environment: BaseEnvironment) -> None: f"git -C {install_dir} fetch --depth 1 origin {commit}; " f"git -C {install_dir} checkout --detach {commit}; " f'test "$(git -C {install_dir} rev-parse HEAD)" = {commit}; ' + "mkdir -p /tmp/hermes-install-path; " + "ln -sf /bin/true /tmp/hermes-install-path/ffmpeg; " f"HERMES_HOME=/tmp/hermes HERMES_INSTALL_DIR={install_dir} " + "PATH=/tmp/hermes-install-path:$PATH " f"bash {install_dir}/scripts/install.sh --skip-setup --skip-browser " f"--no-skills --dir {install_dir} --branch {repository_ref} " f"--commit {commit} --force-commit; " diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index c40a92be9..6153b0efe 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -17,6 +17,7 @@ harbor_bin="${HARBOR_BIN:-harbor}" python_bin="${PHASE1_PYTHON:-python3}" switchyard_bundle="${SWITCHYARD_BUNDLE:-}" relay_wheel="${RELAY_WHEEL:-}" +relay_architecture="${RELAY_ARCHITECTURE:-x86_64}" agent_timeout_multiplier="${AGENT_TIMEOUT_MULTIPLIER:-3}" agent_setup_timeout_multiplier="${AGENT_SETUP_TIMEOUT_MULTIPLIER:-6}" environment_build_timeout_multiplier="${ENVIRONMENT_BUILD_TIMEOUT_MULTIPLIER:-6}" @@ -47,6 +48,10 @@ if [[ -z "${!upstream_auth_env:-}" ]]; then echo "required provider authorization environment variable is unset: $upstream_auth_env" >&2 exit 2 fi +if [[ "$relay_architecture" != "x86_64" && "$relay_architecture" != "aarch64" ]]; then + echo "RELAY_ARCHITECTURE must be x86_64 or aarch64" >&2 + exit 2 +fi docker info >/dev/null curl --fail --silent --show-error --max-time 10 "$phoenix_base" >/dev/null @@ -69,7 +74,8 @@ trap cleanup EXIT if [[ -z "$switchyard_bundle" ]]; then temporary_build="$(mktemp -d "$(dirname "$run_root")/.phase1-switchyard-build.XXXXXX")" switchyard_bundle="$temporary_build/bundle" - "$example_root/scripts/build_switchyard_plugin.sh" "$switchyard_bundle" + SWITCHYARD_TARGET_ARCHITECTURE="$relay_architecture" \ + "$example_root/scripts/build_switchyard_plugin.sh" "$switchyard_bundle" fi free_port="$($python_bin - <<'PY' @@ -86,6 +92,7 @@ prepare_args=( "$example_root/scripts/prepare_runtime.py" --run-root "$run_root" --switchyard-bundle "$switchyard_bundle" + --relay-architecture "$relay_architecture" --upstream-base-url "$upstream_base_url" --upstream-auth-env "$upstream_auth_env" --target-model "$target_model" @@ -148,12 +155,12 @@ fi --ak "switchyard_bundle_dir=$run_root/runtime/switchyard-plugin" \ --ak "relay_wheel_path=$relay_wheel_path" \ --ak "relay_wheel_sha256=$relay_wheel_sha256" \ + --ak "relay_architecture=$relay_architecture" \ "${agent_kwargs[@]}" \ --ae "$upstream_auth_env=${!upstream_auth_env}" \ --ae OPENAI_API_KEY=relay-managed-placeholder \ "${agent_hosts[@]}" \ --artifact /logs/agent/direct-hermes \ - --agent-include-logs 'direct-hermes/**' \ --agent-include-logs hermes-session.jsonl \ --agent-include-logs hermes.txt \ --job-name "$job_name" \ @@ -174,7 +181,11 @@ direct_result="$($python_bin - "$run_root/jobs/$job_name" <<'PY' import pathlib import sys -matches = sorted(pathlib.Path(sys.argv[1]).glob("**/direct-hermes-result.json")) +matches = sorted( + pathlib.Path(sys.argv[1]).glob( + "*/artifacts/logs/agent/direct-hermes/direct-hermes-result.json" + ) +) if len(matches) != 1: raise SystemExit(f"expected one direct Hermes result, found {len(matches)}") print(matches[0]) diff --git a/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py b/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py index 24312a824..ac38e2fb9 100755 --- a/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py +++ b/examples/harbor-hermes-switchyard/scripts/fake_otlp_collector.py @@ -19,7 +19,7 @@ def log_message(self, _format: str, *_args: Any) -> None: return def do_GET(self) -> None: # noqa: N802 - if self.path == "/healthz": + if self.path in {"/", "/healthz"}: self.send_response(200) self.end_headers() return diff --git a/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py index a2c390c5e..5f7743a1b 100755 --- a/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py +++ b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py @@ -10,6 +10,7 @@ import importlib.metadata import json import os +import re import time import tomllib from pathlib import Path @@ -169,6 +170,22 @@ def _last_assistant_response(messages: list[dict[str, Any]]) -> str | None: return None +def _response_from_cli_log(text: str) -> tuple[str | None, str | None]: + """Recover quiet-mode output when Hermes produced an empty session export.""" + lines = text.splitlines() + marker_index: int | None = None + session_id: str | None = None + for index, line in enumerate(lines): + match = re.fullmatch(r"\s*session_id:\s*(\S+)\s*", line) + if match: + marker_index = index + session_id = match.group(1) + if marker_index is None: + return None, None + response = "\n".join(lines[marker_index + 1 :]).strip() + return response or None, session_id + + def _write_bounded_diagnostics(root: Path) -> str: destination = root / "diagnostics" / "hermes-tail.txt" if not HERMES_LOG.is_file(): @@ -191,6 +208,9 @@ def complete(args: argparse.Namespace, root: Path) -> None: messages, exported_session_id = _read_session_messages(HERMES_SESSION) response = _last_assistant_response(messages) diagnostic_text = _write_bounded_diagnostics(root) + log_response, log_session_id = _response_from_cli_log(diagnostic_text) + response = response or log_response + exported_session_id = exported_session_id or log_session_id lowered = diagnostic_text.lower() cleanup_failure = any( marker in lowered diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py index 34f9df30c..3691e1653 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_run.py +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -21,6 +21,28 @@ def read_json(path: Path) -> dict[str, Any]: return value +def is_trial_result(value: dict[str, Any]) -> bool: + return "task_name" in value and "verifier_result" in value + + +def read_benchmark_passed(value: dict[str, Any]) -> bool | None: + reward = value.get("reward") + if isinstance(reward, dict): + candidate = reward.get("task_passed") + if isinstance(candidate, bool): + return candidate + verifier = value.get("verifier_result") + rewards = verifier.get("rewards") if isinstance(verifier, dict) else None + if isinstance(rewards, dict): + candidate = rewards.get("task_passed") + if isinstance(candidate, bool): + return candidate + candidate = rewards.get("reward") + if isinstance(candidate, (int, float)) and not isinstance(candidate, bool): + return candidate > 0 + return None + + def contained_files(root: Path) -> list[Path]: resolved_root = root.resolve(strict=True) files: list[Path] = [] @@ -201,15 +223,14 @@ def main() -> int: harbor_results: list[Path] = [] benchmark_passed: bool | None = None if args.harbor_job_dir: - harbor_results = sorted(args.harbor_job_dir.glob("**/result.json")) + harbor_results = [ + path for path in sorted(args.harbor_job_dir.glob("**/result.json")) if is_trial_result(read_json(path)) + ] if len(harbor_results) != 1: errors.append(f"expected one Harbor trial result, found {len(harbor_results)}") elif harbor_results: harbor_result = read_json(harbor_results[0]) - reward = harbor_result.get("reward") - if isinstance(reward, dict): - candidate = reward.get("task_passed") - benchmark_passed = candidate if isinstance(candidate, bool) else None + benchmark_passed = read_benchmark_passed(harbor_result) validation = { "schema_version": SCHEMA_VERSION, diff --git a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py index b0d310f91..865dc05c7 100644 --- a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py @@ -97,3 +97,25 @@ def test_no_response_never_creates_a_passed_completion(tmp_path: Path, monkeypat completion = json.loads((root / "completion.json").read_text()) assert completion["status"] == "failed" + + +def test_empty_session_uses_bounded_quiet_cli_output(tmp_path: Path, monkeypatch) -> None: + module = load_finalizer() + root = tmp_path / "artifacts" + root.mkdir() + session = tmp_path / "hermes-session.jsonl" + session.write_text("", encoding="utf-8") + log = tmp_path / "hermes.txt" + log.write_text("startup warning\n\nsession_id: cli-session\ncompleted\nanswer\n", encoding="utf-8") + monkeypatch.setattr(module, "HERMES_SESSION", session) + monkeypatch.setattr(module, "HERMES_LOG", log) + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.0") + + args = make_args(tmp_path) + module.initialize(args, root) + module.complete(args, root) + + result = json.loads((root / "direct-hermes-result.json").read_text()) + assert result["status"] == "completed" + assert result["session_id"] == "cli-session" + assert result["final_response"] == "completed\nanswer" diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py index e4615bb94..133d49e72 100644 --- a/examples/harbor-hermes-switchyard/tests/test_config_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_config_contract.py @@ -63,3 +63,11 @@ def test_pricing_does_not_duplicate_relay_generated_aliases() -> None: entry = config["components"][0]["config"]["sources"][0]["catalog"]["entries"][0] assert entry["model_id"] == "namespace/model" assert "aliases" not in entry + + +def test_task_runner_defaults_to_production_x86_64_architecture() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + assert 'relay_architecture="${RELAY_ARCHITECTURE:-x86_64}"' in runner + assert '--relay-architecture "$relay_architecture"' in runner + assert 'SWITCHYARD_TARGET_ARCHITECTURE="$relay_architecture"' in runner + assert '--ak "relay_architecture=$relay_architecture"' in runner diff --git a/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py index ed4e25ab0..6321243f5 100644 --- a/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py +++ b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py @@ -55,5 +55,6 @@ def test_install_verifies_detached_commit_and_relay_release() -> None: source = ast.unparse(method("install")) assert "checkout --detach" in source assert "rev-parse HEAD" in source + assert "/tmp/hermes-install-path/ffmpeg" in source assert "uv sync --frozen --extra all" in source assert "m.version('nemo-relay') == '0.7.0'" in source diff --git a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py new file mode 100644 index 000000000..7a38fd491 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def load_validator(): + path = EXAMPLE_ROOT / "scripts" / "validate_run.py" + spec = importlib.util.spec_from_file_location("phase1_validator", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_harbor_job_summary_is_not_a_trial_result() -> None: + module = load_validator() + assert module.is_trial_result({"n_total_trials": 1, "stats": {}}) is False + assert module.is_trial_result({"task_name": "task", "verifier_result": {}}) is True + + +def test_harbor_018_numeric_reward_is_normalized() -> None: + module = load_validator() + assert module.read_benchmark_passed({"verifier_result": {"rewards": {"reward": 0.0}}}) is False + assert module.read_benchmark_passed({"verifier_result": {"rewards": {"reward": 1.0}}}) is True From 2130f3c576d4314069cd6942aca3c93382691f0b Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 5 Aug 2026 19:01:28 -0600 Subject: [PATCH 03/34] chore(examples): refresh Hermes integration pin Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 27 ++++++++++++++++++- .../agents/harbor_hermes_agent.py | 2 +- .../run_terminal_bench.sh | 2 +- .../scripts/prepare_runtime.py | 2 +- .../run_offline_compatibility_smoke.sh | 2 +- .../tests/test_agent_result_contract.py | 2 +- .../tests/test_config_contract.py | 2 +- 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index c9a6eed59..d95954a8d 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -14,7 +14,7 @@ The four-task regression command below is the Phase 1 readiness gate. | Dependency | Input used by this example | |---|---| | NeMo Relay | Released Linux/amd64 `nemo-relay==0.7.0` wheel; that exact wheel is installed and its digest is recorded per run. | -| Hermes | `bbednarski9/hermes-agent`, branch `feat/relay-native-plugin-init`, detached commit `a07830e086b3055e313b74cc0c8fd5326a4c2c00` (PR #77915). | +| Hermes | `bbednarski9/hermes-agent`, branch `feat/relay-native-plugin-init`, detached commit `efb63e714abc436af88af9b0d6734751c199aa6d` (PR #77915). | | Switchyard | `bbednarski9/Switchyard`, detached commit `8293936a0f5758aa1a782639d485b8b8948cf03e` (PR #270). | | Harbor | `harbor==0.18.0`, dataset `terminal-bench@2.0`. | @@ -97,8 +97,33 @@ configuration. They pass the selected environment variable into the task and scan direct artifacts, Harbor logs, ATOF, ATIF, and OpenInference evidence for the exact secret value. +### Provider configuration ownership + +The rendered `/runtime/plugins.toml` is Relay and Switchyard's +authoritative provider configuration. It contains the target model, protocol, +upstream base URL and endpoint, and the **name** of the environment variable +holding the authorization header. `TARGET_MODEL`, `UPSTREAM_BASE_URL`, and +`UPSTREAM_AUTH_ENV` are preparation inputs used to materialize that immutable +per-run file; they are not an independent provider configuration consumed by +Relay. + +Harbor 0.18.0 still requires a `provider/model` value when constructing its +built-in Hermes lifecycle, and Hermes writes that call-side model into its CLI +configuration before Relay intercepts the operation. The runner therefore +passes `openai/` to Harbor while Switchyard uses the matching +target from `plugins.toml`. `openai` describes the caller protocol here; it +does not bypass Switchyard. Likewise, the placeholder `OPENAI_API_KEY` only +satisfies Harbor/Hermes provider validation. The real authorization value is +resolved by Switchyard from `header_env` and must remain in the environment, +not in TOML. + ## Offline compatibility gate +This is a preflight prerequisite for the first Harbor task run and whenever a +Hermes, Relay, Switchyard, plugin-config, or shutdown-lifecycle input changes. +It is not repeated before every task when those inputs are unchanged, and it +does not replace the single-task or regression gates. + Build the pinned Linux plugin bundle, prepare a fresh run root, and run the forked Hermes/Relay runtime against local fake provider and OTLP endpoints: diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py index 676e41a1a..cf4936d54 100644 --- a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -29,7 +29,7 @@ _SHA256 = re.compile(r"[0-9a-f]{64}") _DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" _DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" -_DEFAULT_HERMES_COMMIT = "a07830e086b3055e313b74cc0c8fd5326a4c2c00" +_DEFAULT_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" _DEFAULT_SWITCHYARD_COMMIT = "8293936a0f5758aa1a782639d485b8b8948cf03e" diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index 6153b0efe..7f107bf2d 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -150,7 +150,7 @@ fi --model "openai/$target_model" \ --ak "repository_url=https://github.com/bbednarski9/hermes-agent.git" \ --ak "repository_ref=feat/relay-native-plugin-init" \ - --ak "commit=a07830e086b3055e313b74cc0c8fd5326a4c2c00" \ + --ak "commit=efb63e714abc436af88af9b0d6734751c199aa6d" \ --ak "relay_config_path=$run_root/runtime/plugins.toml" \ --ak "switchyard_bundle_dir=$run_root/runtime/switchyard-plugin" \ --ak "relay_wheel_path=$relay_wheel_path" \ diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index 4cd5a3c15..7279c159c 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -20,7 +20,7 @@ HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" HERMES_REF = "feat/relay-native-plugin-init" -HERMES_COMMIT = "a07830e086b3055e313b74cc0c8fd5326a4c2c00" +HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" SWITCHYARD_REPOSITORY = "https://github.com/bbednarski9/Switchyard.git" SWITCHYARD_COMMIT = "8293936a0f5758aa1a782639d485b8b8948cf03e" RELAY_VERSION = "0.7.0" diff --git a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh index 9184a7ff7..eb86e12e8 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh +++ b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh @@ -10,7 +10,7 @@ image="${PHASE1_COMPAT_IMAGE:-python:3.11-bookworm}" platform="${PHASE1_COMPAT_PLATFORM:-linux/amd64}" hermes_repository="${HERMES_REPOSITORY:-https://github.com/bbednarski9/hermes-agent.git}" hermes_ref="${HERMES_REF:-feat/relay-native-plugin-init}" -hermes_commit="${HERMES_COMMIT:-a07830e086b3055e313b74cc0c8fd5326a4c2c00}" +hermes_commit="${HERMES_COMMIT:-efb63e714abc436af88af9b0d6734751c199aa6d}" if [[ -z "$run_root" || "$run_root" != /* ]]; then echo "usage: $0 /absolute/prepared-run-root" >&2 diff --git a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py index 865dc05c7..4c244d3db 100644 --- a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py @@ -33,7 +33,7 @@ def make_args(tmp_path: Path, *, error_type: str = "") -> argparse.Namespace: switchyard_library=library, relay_wheel_sha256="a" * 64, hermes_repository="https://github.com/bbednarski9/hermes-agent.git", - hermes_commit="a07830e086b3055e313b74cc0c8fd5326a4c2c00", + hermes_commit="efb63e714abc436af88af9b0d6734751c199aa6d", switchyard_commit="8293936a0f5758aa1a782639d485b8b8948cf03e", session_handle="phase1-session", started_at=1.0, diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py index 133d49e72..d051d883d 100644 --- a/examples/harbor-hermes-switchyard/tests/test_config_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_config_contract.py @@ -14,7 +14,7 @@ def render_template(**values: str) -> dict: text = (EXAMPLE_ROOT / "config" / "relay.toml.in").read_text(encoding="utf-8") defaults = { "TARGET_MODEL": "phase1-test-model", - "HERMES_COMMIT": "a07830e086b3055e313b74cc0c8fd5326a4c2c00", + "HERMES_COMMIT": "efb63e714abc436af88af9b0d6734751c199aa6d", "OPENINFERENCE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", "PHOENIX_PROJECT": "phase1-test", "EVAL_COHORT": "phase1-test", From 91599524d3ae78280e0310ea7d3e4263d843a7d4 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 5 Aug 2026 19:54:06 -0600 Subject: [PATCH 04/34] feat(examples): route Hermes through inference hub tiers Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 67 +++++++++----- .../config/relay.toml.in | 49 +++++++++-- .../run_terminal_bench.sh | 62 +++++++++++-- .../scripts/fake_openai_upstream.py | 25 +++++- .../scripts/offline_compatibility_smoke.py | 88 ++++++++++++------- .../scripts/prepare_runtime.py | 24 ++++- .../scripts/validate_run.py | 23 ++++- .../tests/test_config_contract.py | 54 ++++++++++-- .../tests/test_validation_contract.py | 32 +++++++ 9 files changed, 344 insertions(+), 80 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index d95954a8d..4556f14bc 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -74,7 +74,7 @@ upstream-repository-only and branch-only. - Docker with enough space to build one Linux/amd64 Rust plugin and task image; - Python 3.11 or newer; -- a provider endpoint compatible with OpenAI Chat Completions; +- access to `inference.nvidia.com` through its OpenAI-compatible endpoint; - a Phoenix endpoint accepting OTLP/HTTP traces; and - the provider authorization value in an environment variable. @@ -100,22 +100,42 @@ the exact secret value. ### Provider configuration ownership The rendered `/runtime/plugins.toml` is Relay and Switchyard's -authoritative provider configuration. It contains the target model, protocol, -upstream base URL and endpoint, and the **name** of the environment variable -holding the authorization header. `TARGET_MODEL`, `UPSTREAM_BASE_URL`, and -`UPSTREAM_AUTH_ENV` are preparation inputs used to materialize that immutable -per-run file; they are not an independent provider configuration consumed by -Relay. +authoritative provider configuration. It contains the strong and weak models, +classifier policy, protocol, upstream base URL and endpoint, and the **name** +of the environment variable holding the authorization header. The defaults +are the inference.nvidia.com catalog entries +`aws/anthropic/bedrock-claude-opus-4-6` (strong) and +`aws/anthropic/bedrock-claude-sonnet-4-6` (weak). + +The `llm_classifier` policy uses Sonnet as both the classifier and weak target. +On the first request it asks Sonnet for a structured capability verdict, then +routes the original request to Sonnet when `p_solve >= 0.5` or Opus otherwise. +Invalid, unavailable, or low-confidence classifier output fails safe to Opus. +The first decision is retained for the session, so later turns do not incur a +second classifier call. This two-model policy avoids adding a third provider, +but the packaged classifier prompt is not model-neutral; the threshold must be +revalidated if either model changes. + +`STRONG_MODEL`, `WEAK_MODEL`, `UPSTREAM_BASE_URL`, and `UPSTREAM_AUTH_ENV` are +preparation inputs used to materialize that immutable per-run file; they are +not independent provider configuration consumed by Relay. When +`INFERENCE_SECRETS_FILE` is set, the runner reads `NV_INFERENCEHUB_ENDPOINT` +and `NV_INFERENCEHUB_KEY` from it in short-lived subshells. It derives the +Bearer authorization value only in memory, unsets the raw variables, and +passes only `SWITCHYARD_PROVIDER_AUTHORIZATION` into the task. The secrets file +is never copied into the run root or a container. Harbor 0.18.0 still requires a `provider/model` value when constructing its built-in Hermes lifecycle, and Hermes writes that call-side model into its CLI -configuration before Relay intercepts the operation. The runner therefore -passes `openai/` to Harbor while Switchyard uses the matching -target from `plugins.toml`. `openai` describes the caller protocol here; it -does not bypass Switchyard. Likewise, the placeholder `OPENAI_API_KEY` only -satisfies Harbor/Hermes provider validation. The real authorization value is -resolved by Switchyard from `header_env` and must remain in the environment, -not in TOML. +configuration before Relay intercepts the operation. The runner passes +`openai/ollama-route-stub`: an intentionally unserved, Ollama-shaped caller +identity. `openai` describes only the caller protocol required by Harbor; the +stub is not a Switchyard target. `OPENAI_BASE_URL` is projected as the dead +local endpoint `http://127.0.0.1:9/v1`, so a request that bypasses Switchyard +fails closed rather than reaching a provider. The placeholder +`OPENAI_API_KEY` only satisfies Harbor/Hermes validation. Successful provider +traffic must use the real targets and authorization resolved by Switchyard +from `plugins.toml` and `header_env`. ## Offline compatibility gate @@ -136,7 +156,9 @@ export SPIKE_ROOT="/absolute/new/spike-root" --run-root "$SPIKE_ROOT" \ --switchyard-bundle /absolute/new/switchyard-bundle \ --upstream-base-url http://127.0.0.1:8000/v1 \ - --target-model phase1/fake-model \ + --strong-model phase1/fake-strong \ + --weak-model phase1/fake-weak \ + --hermes-caller-model ollama-route-stub \ --openinference-endpoint http://127.0.0.1:4318/v1/traces \ --phoenix-project phase1-offline \ --eval-cohort phase1-offline @@ -160,7 +182,9 @@ SWITCHYARD_TARGET_ARCHITECTURE=aarch64 \ --switchyard-bundle /absolute/new/arm64-bundle \ --relay-architecture aarch64 \ --upstream-base-url http://127.0.0.1:8000/v1 \ - --target-model phase1/fake-model \ + --strong-model phase1/fake-strong \ + --weak-model phase1/fake-weak \ + --hermes-caller-model ollama-route-stub \ --openinference-endpoint http://127.0.0.1:4318/v1/traces \ --phoenix-project phase1-offline-arm64 \ --eval-cohort phase1-offline-arm64 @@ -184,10 +208,7 @@ gate stay `x86_64`. Use a new absolute run root on every invocation: ```bash -export TARGET_MODEL="your-provider-model" -export UPSTREAM_BASE_URL="https://your-openai-compatible-endpoint/v1" -export UPSTREAM_AUTH_ENV="SWITCHYARD_PROVIDER_AUTHORIZATION" -export SWITCHYARD_PROVIDER_AUTHORIZATION="Bearer ..." +export INFERENCE_SECRETS_FILE="/absolute/path/to/.inference_secrets" export PHOENIX_BASE_URL="https://your-phoenix-endpoint" export PHOENIX_PROJECT="harbor-hermes-switchyard-phase1" export EVAL_COHORT="harbor-hermes-switchyard-phase1" @@ -195,6 +216,12 @@ export EVAL_COHORT="harbor-hermes-switchyard-phase1" ./run_terminal_bench.sh /absolute/new/run-root ``` +The secrets file must define `NV_INFERENCEHUB_ENDPOINT` and +`NV_INFERENCEHUB_KEY`. Its path is only a launch input and is never rendered +into generated configuration or provenance. Advanced runs may override +`STRONG_MODEL`, `WEAK_MODEL`, or `UPSTREAM_BASE_URL`; changing either model +requires rerunning the offline gate and classifier routing smokes. + The default task is `adaptive-rejection-sampler`. Override it with `TASK_NAME`. To avoid rebuilding Switchyard for each task, set `SWITCHYARD_BUNDLE` to a previously built, immutable bundle. Set `RELAY_WHEEL` diff --git a/examples/harbor-hermes-switchyard/config/relay.toml.in b/examples/harbor-hermes-switchyard/config/relay.toml.in index e40ea8f41..983e8d94e 100644 --- a/examples/harbor-hermes-switchyard/config/relay.toml.in +++ b/examples/harbor-hermes-switchyard/config/relay.toml.in @@ -15,7 +15,23 @@ version = 1 [[components.config.sources.catalog.entries]] provider = "openai" -model_id = "@TARGET_MODEL@" +model_id = "@STRONG_MODEL@" +currency = "USD" +unit = "per_token" +pricing_as_of = "2026-08-05" +pricing_source = "harbor-hermes-switchyard-example" + +[components.config.sources.catalog.entries.rates] +input_per_million = 0.0 +output_per_million = 0.0 +cache_read_per_million = 0.0 + +[components.config.sources.catalog.entries.prompt_cache] +read_accounting = "included_in_prompt_tokens" + +[[components.config.sources.catalog.entries]] +provider = "openai" +model_id = "@WEAK_MODEL@" currency = "USD" unit = "per_token" pricing_as_of = "2026-08-05" @@ -49,7 +65,7 @@ filename = "trajectory.atof.jsonl" enabled = true agent_name = "Hermes" agent_version = "@HERMES_COMMIT@" -model_name = "@TARGET_MODEL@" +model_name = "@HERMES_CALLER_MODEL@" output_directory = "/logs/agent/direct-hermes/relay/atif" filename_template = "trajectory-{session_id}.atif.json" @@ -78,18 +94,35 @@ priority = 0 max_retries = 1 [plugins.dynamic.config.algorithm] -kind = "random" -seed = 42 +kind = "llm_classifier" +classifier_target = "weak" +weak_target = "weak" +strong_target = "strong" +base_threshold = 0.5 +min_confidence = 0.0 +recent_turn_window = 0 +session_affinity = true +message_hash_fallback = true [plugins.dynamic.config.default_targets] -openai_chat = "primary" +openai_chat = "strong" + +[plugins.dynamic.config.targets.strong] +model = "@STRONG_MODEL@" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "@UPSTREAM_BASE_URL@" +weight = 1 + +[plugins.dynamic.config.targets.strong.header_env] +authorization = "@UPSTREAM_AUTH_ENV@" -[plugins.dynamic.config.targets.primary] -model = "@TARGET_MODEL@" +[plugins.dynamic.config.targets.weak] +model = "@WEAK_MODEL@" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "@UPSTREAM_BASE_URL@" weight = 1 -[plugins.dynamic.config.targets.primary.header_env] +[plugins.dynamic.config.targets.weak.header_env] authorization = "@UPSTREAM_AUTH_ENV@" diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index 7f107bf2d..5a87a00ba 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -7,9 +7,12 @@ set -euo pipefail example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_root="${1:-}" task_name="${TASK_NAME:-adaptive-rejection-sampler}" -target_model="${TARGET_MODEL:-}" -upstream_base_url="${UPSTREAM_BASE_URL:-}" +strong_model="${STRONG_MODEL:-aws/anthropic/bedrock-claude-opus-4-6}" +weak_model="${WEAK_MODEL:-aws/anthropic/bedrock-claude-sonnet-4-6}" +hermes_caller_model="${HERMES_CALLER_MODEL:-ollama-route-stub}" +inference_secrets_file="${INFERENCE_SECRETS_FILE:-}" upstream_auth_env="${UPSTREAM_AUTH_ENV:-SWITCHYARD_PROVIDER_AUTHORIZATION}" +fail_closed_openai_base_url="http://127.0.0.1:9/v1" phoenix_base="${PHOENIX_BASE_URL:-}" phoenix_project="${PHOENIX_PROJECT:-harbor-hermes-switchyard-phase1}" eval_cohort="${EVAL_COHORT:-harbor-hermes-switchyard-phase1}" @@ -32,12 +35,56 @@ if [[ -e "$run_root" ]]; then echo "run root already exists: $run_root" >&2 exit 2 fi -for required in "$target_model" "$upstream_base_url" "$phoenix_base"; do +if [[ ! "$upstream_auth_env" =~ ^[A-Z_][A-Z0-9_]*$ ]]; then + echo "UPSTREAM_AUTH_ENV must be an uppercase environment variable name" >&2 + exit 2 +fi + +nv_inferencehub_endpoint="${NV_INFERENCEHUB_ENDPOINT:-}" +nv_inferencehub_key="${NV_INFERENCEHUB_KEY:-}" +if [[ -n "$inference_secrets_file" ]]; then + [[ -r "$inference_secrets_file" ]] || { + echo "INFERENCE_SECRETS_FILE is not readable: $inference_secrets_file" >&2 + exit 2 + } + load_secret_value() { + local variable_name="$1" + ( + set +x + # The file is sourced only in this short-lived subshell. Its other + # variables never enter Harbor's environment. + source "$inference_secrets_file" + printf '%s' "${!variable_name:-}" + ) + } + [[ -n "$nv_inferencehub_endpoint" ]] || \ + nv_inferencehub_endpoint="$(load_secret_value NV_INFERENCEHUB_ENDPOINT)" + [[ -n "$nv_inferencehub_key" ]] || \ + nv_inferencehub_key="$(load_secret_value NV_INFERENCEHUB_KEY)" +fi + +upstream_base_url="${UPSTREAM_BASE_URL:-$nv_inferencehub_endpoint}" +upstream_base_url="${upstream_base_url%/chat/completions}" +if [[ -z "${!upstream_auth_env:-}" && -n "$nv_inferencehub_key" ]]; then + if [[ "$nv_inferencehub_key" == "Bearer "* ]]; then + printf -v "$upstream_auth_env" '%s' "$nv_inferencehub_key" + else + printf -v "$upstream_auth_env" 'Bearer %s' "$nv_inferencehub_key" + fi + export "$upstream_auth_env" +fi +# Do not propagate the raw inference variables to Harbor or task containers. +unset NV_INFERENCEHUB_ENDPOINT NV_INFERENCEHUB_KEY nv_inferencehub_endpoint nv_inferencehub_key +for required in "$strong_model" "$weak_model" "$hermes_caller_model" "$upstream_base_url" "$phoenix_base"; do [[ -n "$required" ]] || { - echo "TARGET_MODEL, UPSTREAM_BASE_URL, and PHOENIX_BASE_URL are required" >&2 + echo "model names, inference endpoint, and PHOENIX_BASE_URL are required" >&2 exit 2 } done +if [[ "$strong_model" == "$weak_model" ]]; then + echo "STRONG_MODEL and WEAK_MODEL must be distinct" >&2 + exit 2 +fi for dependency in curl docker "$harbor_bin" "$python_bin"; do command -v "$dependency" >/dev/null || { echo "missing required command: $dependency" >&2 @@ -95,7 +142,9 @@ prepare_args=( --relay-architecture "$relay_architecture" --upstream-base-url "$upstream_base_url" --upstream-auth-env "$upstream_auth_env" - --target-model "$target_model" + --strong-model "$strong_model" + --weak-model "$weak_model" + --hermes-caller-model "$hermes_caller_model" --openinference-endpoint "$openinference_endpoint" --phoenix-project "$phoenix_project" --eval-cohort "$eval_cohort" @@ -147,7 +196,7 @@ fi --include-task-name "$task_name" \ --n-tasks 1 \ --agent harbor_hermes_agent:HarborHermesAgent \ - --model "openai/$target_model" \ + --model "openai/$hermes_caller_model" \ --ak "repository_url=https://github.com/bbednarski9/hermes-agent.git" \ --ak "repository_ref=feat/relay-native-plugin-init" \ --ak "commit=efb63e714abc436af88af9b0d6734751c199aa6d" \ @@ -159,6 +208,7 @@ fi "${agent_kwargs[@]}" \ --ae "$upstream_auth_env=${!upstream_auth_env}" \ --ae OPENAI_API_KEY=relay-managed-placeholder \ + --ae "OPENAI_BASE_URL=$fail_closed_openai_base_url" \ "${agent_hosts[@]}" \ --artifact /logs/agent/direct-hermes \ --agent-include-logs hermes-session.jsonl \ diff --git a/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py b/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py index d85377e94..e095510bf 100755 --- a/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py +++ b/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py @@ -42,14 +42,35 @@ def do_POST(self) -> None: # noqa: N802 except (ValueError, json.JSONDecodeError): self.send_error(400) return + messages = request.get("messages", []) + serialized_messages = json.dumps(messages, separators=(",", ":")) + is_classifier = request.get("response_format") is not None or ( + "p_solve" in serialized_messages and "capability_boundary" in serialized_messages + ) log_entry = { "path": self.path, "model": request.get("model"), - "message_count": len(request.get("messages", [])), + "message_count": len(messages), + "request_kind": "classifier" if is_classifier else "completion", "authorization_present": True, } with self.request_log.open("a", encoding="utf-8") as stream: stream.write(json.dumps(log_entry, separators=(",", ":")) + "\n") + content = "OFFLINE_SWITCHYARD_OK" + if is_classifier: + force_strong = "force strong route" in serialized_messages + content = json.dumps( + { + "recommended_route": "strong" if force_strong else "weak", + "p_solve": 0.01 if force_strong else 0.99, + "confidence": 0.99, + "abstain": False, + "capability_boundary": "supported", + "primary_rule": "SUP-1", + "crux": "deterministic offline smoke task", + }, + separators=(",", ":"), + ) response = { "id": "chatcmpl-phase1", "object": "chat.completion", @@ -60,7 +81,7 @@ def do_POST(self) -> None: # noqa: N802 "index": 0, "message": { "role": "assistant", - "content": "OFFLINE_SWITCHYARD_OK", + "content": content, }, "finish_reason": "stop", } diff --git a/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py b/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py index c95230858..05f1fc0ac 100755 --- a/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py +++ b/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py @@ -14,15 +14,12 @@ from typing import Any -async def exercise(model: str, session_id: str) -> tuple[dict[str, Any], dict[str, Any]]: +async def exercise(model: str) -> tuple[list[dict[str, Any]], dict[str, Any]]: from agent.relay_runtime import RelayRuntime import nemo_relay host = RelayRuntime(profile_key="phase1-offline") - session = host.ensure_session({"session_id": session_id}) - if session is None: - raise RuntimeError("Hermes Relay runtime did not open a session") downstream_called = False async def forbidden_downstream(_request: Any) -> dict[str, Any]: @@ -30,37 +27,47 @@ async def forbidden_downstream(_request: Any) -> dict[str, Any]: downstream_called = True raise AssertionError("Switchyard managed request reached Relay downstream callback") - request = nemo_relay.LLMRequest( - {}, - { - "model": model, - "messages": [{"role": "user", "content": "reply with the smoke marker"}], - "stream": False, - }, - ) + responses: list[dict[str, Any]] = [] try: - response = await host.run_in_session_async( - session, - nemo_relay.llm.execute, - "openai.chat_completions", - request, - forbidden_downstream, - model_name=model, - response_codec=nemo_relay.codecs.OpenAIChatCodec(), + cases = ( + ("phase1-offline-weak-session", "reply with the smoke marker"), + ("phase1-offline-strong-session", "force strong route and reply with the smoke marker"), ) + for session_id, prompt in cases: + session = host.ensure_session({"session_id": session_id}) + if session is None: + raise RuntimeError("Hermes Relay runtime did not open a session") + request = nemo_relay.LLMRequest( + {}, + { + "model": model, + "messages": [{"role": "user", "content": prompt}], + "stream": False, + }, + ) + response = await host.run_in_session_async( + session, + nemo_relay.llm.execute, + "openai.chat_completions", + request, + forbidden_downstream, + model_name=model, + response_codec=nemo_relay.codecs.OpenAIChatCodec(), + ) + responses.append(response) + host.close_session({"session_id": session_id}) active_report = nemo_relay.plugin.report() if active_report is None: raise AssertionError("Relay did not expose an active plugin report") report = active_report.to_dict() if hasattr(active_report, "to_dict") else active_report - host.close_session({"session_id": session_id}) finally: host.shutdown() if downstream_called: raise AssertionError("Relay downstream callback was invoked") - content = response["choices"][0]["message"]["content"] - if content != "OFFLINE_SWITCHYARD_OK": - raise AssertionError(f"unexpected fake-provider response: {content!r}") - return response, report + contents = [response["choices"][0]["message"]["content"] for response in responses] + if contents != ["OFFLINE_SWITCHYARD_OK", "OFFLINE_SWITCHYARD_OK"]: + raise AssertionError(f"unexpected fake-provider responses: {contents!r}") + return responses, report def main() -> int: @@ -68,13 +75,15 @@ def main() -> int: parser.add_argument("--plugins", type=Path, required=True) parser.add_argument("--artifacts", type=Path, required=True) parser.add_argument("--request-log", type=Path, required=True) - parser.add_argument("--model", default="phase1/fake-model") + parser.add_argument("--model", default="ollama-route-stub") + parser.add_argument("--classifier-model", default="phase1/fake-weak") + parser.add_argument("--expected-routed-model", default="phase1/fake-weak") + parser.add_argument("--expected-strong-model", default="phase1/fake-strong") args = parser.parse_args() artifacts = args.artifacts.resolve() artifacts.mkdir(mode=0o700, parents=True, exist_ok=True) os.environ["HERMES_NEMO_RELAY_PLUGINS_TOML"] = str(args.plugins.resolve()) - session_id = "phase1-offline-session" - response, report = asyncio.run(exercise(args.model, session_id)) + responses, report = asyncio.run(exercise(args.model)) atof = artifacts / "relay" / "trajectory.atof.jsonl" atif = sorted((artifacts / "relay" / "atif").glob("trajectory-*.atif.json")) @@ -91,8 +100,22 @@ def main() -> int: if not marks: raise AssertionError("Switchyard routing marks were not emitted") requests = [json.loads(line) for line in args.request_log.read_text(encoding="utf-8").splitlines() if line.strip()] - if len(requests) != 1 or not requests[0].get("authorization_present"): - raise AssertionError("fake provider did not receive exactly one authenticated request") + if len(requests) != 4 or not all(item.get("authorization_present") for item in requests): + raise AssertionError("fake provider did not receive exactly four authenticated requests") + request_kinds = [item.get("request_kind") for item in requests] + if request_kinds != ["classifier", "completion", "classifier", "completion"]: + raise AssertionError(f"unexpected provider request sequence: {request_kinds}") + request_models = [item.get("model") for item in requests] + expected_models = [ + args.classifier_model, + args.expected_routed_model, + args.classifier_model, + args.expected_strong_model, + ] + if request_models != expected_models: + raise AssertionError(f"unexpected provider model sequence: {request_models}") + if args.model in request_models: + raise AssertionError("Hermes caller stub reached the provider") surviving = [ thread.name for thread in threading.enumerate() if thread.name.startswith("hermes-nemo-relay-shutdown-") ] @@ -102,8 +125,11 @@ def main() -> int: result = { "schema_version": "harbor-hermes-switchyard.offline-smoke.v1", "status": "passed", - "response": response["choices"][0]["message"]["content"], + "responses": [response["choices"][0]["message"]["content"] for response in responses], "provider_requests": len(requests), + "provider_request_kinds": request_kinds, + "provider_models": request_models, + "hermes_caller_model": args.model, "relay_downstream_callback_called": False, "switchyard_routing_marks": marks, "active_plugin_report_before_shutdown": report, diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index 7279c159c..9e5f3d6e0 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -24,6 +24,9 @@ SWITCHYARD_REPOSITORY = "https://github.com/bbednarski9/Switchyard.git" SWITCHYARD_COMMIT = "8293936a0f5758aa1a782639d485b8b8948cf03e" RELAY_VERSION = "0.7.0" +DEFAULT_STRONG_MODEL = "aws/anthropic/bedrock-claude-opus-4-6" +DEFAULT_WEAK_MODEL = "aws/anthropic/bedrock-claude-sonnet-4-6" +DEFAULT_HERMES_CALLER_MODEL = "ollama-route-stub" ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]*") SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}") @@ -133,7 +136,9 @@ def main() -> int: parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), default="x86_64") parser.add_argument("--upstream-base-url", required=True) parser.add_argument("--upstream-auth-env", default="SWITCHYARD_PROVIDER_AUTHORIZATION") - parser.add_argument("--target-model", required=True) + parser.add_argument("--strong-model", default=DEFAULT_STRONG_MODEL) + parser.add_argument("--weak-model", default=DEFAULT_WEAK_MODEL) + parser.add_argument("--hermes-caller-model", default=DEFAULT_HERMES_CALLER_MODEL) parser.add_argument("--openinference-endpoint", required=True) parser.add_argument("--phoenix-project", required=True) parser.add_argument("--eval-cohort", required=True) @@ -171,7 +176,11 @@ def main() -> int: openinference_endpoint = checked_url(args.openinference_endpoint, "openinference_endpoint") if not ENV_NAME.fullmatch(args.upstream_auth_env): raise ValueError("upstream_auth_env must be an uppercase environment variable name") - target_model = checked_label(args.target_model, "target_model") + strong_model = checked_label(args.strong_model, "strong_model") + weak_model = checked_label(args.weak_model, "weak_model") + hermes_caller_model = checked_label(args.hermes_caller_model, "hermes_caller_model") + if strong_model == weak_model: + raise ValueError("strong_model and weak_model must be distinct") phoenix_project = checked_label(args.phoenix_project, "phoenix_project") eval_cohort = checked_label(args.eval_cohort, "eval_cohort") @@ -180,7 +189,9 @@ def main() -> int: example_root / "config" / "relay.toml.in", config_path, { - "TARGET_MODEL": target_model, + "STRONG_MODEL": strong_model, + "WEAK_MODEL": weak_model, + "HERMES_CALLER_MODEL": hermes_caller_model, "HERMES_COMMIT": HERMES_COMMIT, "OPENINFERENCE_ENDPOINT": openinference_endpoint, "PHOENIX_PROJECT": phoenix_project, @@ -216,6 +227,13 @@ def main() -> int: "library_sha256": sha256(libraries[0]), }, "relay_config_sha256": sha256(config_path), + "routing": { + "algorithm": "llm_classifier", + "classifier_target": "weak", + "strong_model": strong_model, + "weak_model": weak_model, + "hermes_caller_model": hermes_caller_model, + }, "phoenix_project": phoenix_project, "eval_cohort": eval_cohort, } diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py index 3691e1653..fc55d12db 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_run.py +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -89,10 +89,11 @@ def scan_secrets(files: Iterable[Path], values: list[bytes]) -> list[str]: return sorted(set(findings)) -def read_atof(path: Path) -> tuple[int, list[str], list[str]]: +def read_atof(path: Path) -> tuple[int, list[str], list[str], list[str]]: count = 0 marks: list[str] = [] models: list[str] = [] + targets: list[str] = [] with path.open(encoding="utf-8") as stream: for line_number, line in enumerate(stream, 1): if not line.strip(): @@ -110,7 +111,10 @@ def read_atof(path: Path) -> tuple[int, list[str], list[str]]: value = container.get(key) if isinstance(value, str) and value: models.append(value) - return count, sorted(set(marks)), sorted(set(models)) + value = container.get("selected_target") + if isinstance(value, str) and value: + targets.append(value) + return count, sorted(set(marks)), sorted(set(models)), sorted(set(targets)) def main() -> int: @@ -194,18 +198,30 @@ def main() -> int: event_count = 0 routing_marks: list[str] = [] routed_models: list[str] = [] + routed_targets: list[str] = [] if required["atof"].is_file(): - event_count, routing_marks, routed_models = read_atof(required["atof"]) + event_count, routing_marks, routed_models, routed_targets = read_atof(required["atof"]) if event_count == 0: errors.append("ATOF artifact is empty") if not routing_marks: errors.append("ATOF artifact has no Switchyard routing evidence") + if not routed_targets: + errors.append("ATOF artifact has no selected Switchyard target") + unexpected_targets = sorted(set(routed_targets) - {"strong", "weak"}) + if unexpected_targets: + errors.append(f"ATOF artifact selected unexpected targets: {unexpected_targets}") + + caller_model = provenance.get("routing", {}).get("hermes_caller_model") + if caller_model and caller_model in routed_models: + errors.append("Hermes caller stub appeared as a routed provider model") secret_values: list[bytes] = [] for name in args.secret_env: value = os.environ.get(name) if value: secret_values.append(value.encode()) + if value.startswith("Bearer ") and value[7:]: + secret_values.append(value[7:].encode()) for path in args.secret_file: for line in path.read_bytes().splitlines(): value = line.split(b"=", 1)[-1].strip() @@ -243,6 +259,7 @@ def main() -> int: "atif_trajectory_count": len(atif_files), "switchyard_routing_marks": routing_marks, "routed_models": routed_models, + "routed_targets": routed_targets, "secret_values_scanned": len(secret_values), "secret_findings": findings, } diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py index d051d883d..a6ccf9235 100644 --- a/examples/harbor-hermes-switchyard/tests/test_config_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_config_contract.py @@ -13,7 +13,9 @@ def render_template(**values: str) -> dict: text = (EXAMPLE_ROOT / "config" / "relay.toml.in").read_text(encoding="utf-8") defaults = { - "TARGET_MODEL": "phase1-test-model", + "STRONG_MODEL": "phase1-test-strong", + "WEAK_MODEL": "phase1-test-weak", + "HERMES_CALLER_MODEL": "ollama-route-stub", "HERMES_COMMIT": "efb63e714abc436af88af9b0d6734751c199aa6d", "OPENINFERENCE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", "PHOENIX_PROJECT": "phase1-test", @@ -38,9 +40,24 @@ def test_config_uses_static_schema_v3_and_one_standard_dynamic_plugin() -> None: assert len(config["plugins"]["dynamic"]) == 1 plugin = config["plugins"]["dynamic"][0] assert plugin["manifest"].endswith("/nvidia.switchyard/relay-plugin.toml") - assert plugin["config"]["targets"]["primary"]["header_env"] == { - "authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION" + algorithm = plugin["config"]["algorithm"] + assert algorithm == { + "kind": "llm_classifier", + "classifier_target": "weak", + "weak_target": "weak", + "strong_target": "strong", + "base_threshold": 0.5, + "min_confidence": 0.0, + "recent_turn_window": 0, + "session_affinity": True, + "message_hash_fallback": True, } + assert plugin["config"]["default_targets"] == {"openai_chat": "strong"} + assert set(plugin["config"]["targets"]) == {"strong", "weak"} + for target in plugin["config"]["targets"].values(): + assert target["header_env"] == { + "authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION" + } def test_config_contains_no_literal_provider_headers_or_credentials() -> None: @@ -59,10 +76,23 @@ def walk(value: object) -> None: def test_pricing_does_not_duplicate_relay_generated_aliases() -> None: - config = render_template(TARGET_MODEL="namespace/model") - entry = config["components"][0]["config"]["sources"][0]["catalog"]["entries"][0] - assert entry["model_id"] == "namespace/model" - assert "aliases" not in entry + config = render_template( + STRONG_MODEL="namespace/strong", + WEAK_MODEL="namespace/weak", + ) + entries = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] + assert [entry["model_id"] for entry in entries] == ["namespace/strong", "namespace/weak"] + assert all("aliases" not in entry for entry in entries) + + +def test_switchyard_models_are_distinct_from_fail_closed_hermes_caller() -> None: + config = render_template() + plugin = config["plugins"]["dynamic"][0] + provider_models = {target["model"] for target in plugin["config"]["targets"].values()} + assert provider_models == {"phase1-test-strong", "phase1-test-weak"} + observability = next(item for item in config["components"] if item["kind"] == "observability") + assert observability["config"]["atif"]["model_name"] == "ollama-route-stub" + assert "ollama-route-stub" not in provider_models def test_task_runner_defaults_to_production_x86_64_architecture() -> None: @@ -71,3 +101,13 @@ def test_task_runner_defaults_to_production_x86_64_architecture() -> None: assert '--relay-architecture "$relay_architecture"' in runner assert 'SWITCHYARD_TARGET_ARCHITECTURE="$relay_architecture"' in runner assert '--ak "relay_architecture=$relay_architecture"' in runner + + +def test_task_runner_defaults_to_inference_hub_tiers_and_fail_closed_caller() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + assert "aws/anthropic/bedrock-claude-opus-4-6" in runner + assert "aws/anthropic/bedrock-claude-sonnet-4-6" in runner + assert 'hermes_caller_model="${HERMES_CALLER_MODEL:-ollama-route-stub}"' in runner + assert '--model "openai/$hermes_caller_model"' in runner + assert 'fail_closed_openai_base_url="http://127.0.0.1:9/v1"' in runner + assert '--ae "OPENAI_BASE_URL=$fail_closed_openai_base_url"' in runner diff --git a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py index 7a38fd491..58c166c6e 100644 --- a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py @@ -4,6 +4,7 @@ from __future__ import annotations import importlib.util +import json from pathlib import Path EXAMPLE_ROOT = Path(__file__).resolve().parents[1] @@ -28,3 +29,34 @@ def test_harbor_018_numeric_reward_is_normalized() -> None: module = load_validator() assert module.read_benchmark_passed({"verifier_result": {"rewards": {"reward": 0.0}}}) is False assert module.read_benchmark_passed({"verifier_result": {"rewards": {"reward": 1.0}}}) is True + + +def test_atof_reader_extracts_switchyard_selected_targets(tmp_path: Path) -> None: + module = load_validator() + path = tmp_path / "trajectory.atof.jsonl" + events = [ + {"name": "switchyard.routing.requested", "data": {"algorithm": "llm_task_classifier"}}, + { + "name": "switchyard.routing.decision", + "data": {"selected_target": "weak", "routing_tier": "weak"}, + }, + { + "name": "switchyard.routing.decision", + "data": {"selected_target": "strong", "routing_tier": "strong"}, + }, + ] + path.write_text("\n".join(json.dumps(event) for event in events) + "\n") + count, marks, models, targets = module.read_atof(path) + assert count == 3 + assert marks == ["switchyard.routing.decision", "switchyard.routing.requested"] + assert models == [] + assert targets == ["strong", "weak"] + + +def test_secret_scan_finds_raw_key_within_artifact(tmp_path: Path) -> None: + module = load_validator() + artifact = tmp_path / "artifact.log" + artifact.write_text("raw-provider-key") + assert module.scan_secrets([artifact], [b"Bearer raw-provider-key", b"raw-provider-key"]) == [ + "artifact.log:secret[1]" + ] From 65536f35881570990750843a36d3e7652f7e5f98 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 6 Aug 2026 08:26:07 -0600 Subject: [PATCH 05/34] feat(examples): make Harbor eval runs repeatable --- examples/harbor-hermes-switchyard/.gitignore | 2 + examples/harbor-hermes-switchyard/README.md | 463 ++++----- .../agents/harbor_hermes_agent.py | 142 ++- .../config/{relay.toml.in => plugins.toml.in} | 43 +- .../phase2-run.env.example | 34 + .../harbor-hermes-switchyard/requirements.txt | 1 + .../run_phase2_cohort.sh | 90 ++ .../run_terminal_bench.sh | 154 +-- .../scripts/build_switchyard_plugin.sh | 2 +- .../scripts/exec_process_group.py | 31 + .../scripts/fake_openai_upstream.py | 3 - .../scripts/finalize_artifacts.py | 24 +- .../scripts/launch_phase2_tmux.sh | 30 + .../scripts/offline_compatibility_smoke.py | 25 +- .../scripts/prepare_runtime.py | 128 ++- .../run_offline_compatibility_smoke.sh | 36 +- .../scripts/run_phase2_cohort.py | 901 ++++++++++++++++++ .../scripts/run_phase2_from_env.sh | 25 + .../scripts/smoke_phase2_dataset.py | 377 ++++++++ .../scripts/validate_parallel_isolation.py | 93 ++ .../scripts/validate_phase2_environment.sh | 90 ++ .../scripts/validate_run.py | 245 ++++- .../supervise_phase2_cohort.sh | 71 ++ .../tests/test_agent_result_contract.py | 29 +- .../tests/test_config_contract.py | 124 ++- .../tests/test_phase2_cohort.py | 427 +++++++++ .../tests/test_validation_contract.py | 128 +++ 27 files changed, 3295 insertions(+), 423 deletions(-) create mode 100644 examples/harbor-hermes-switchyard/.gitignore rename examples/harbor-hermes-switchyard/config/{relay.toml.in => plugins.toml.in} (74%) create mode 100644 examples/harbor-hermes-switchyard/phase2-run.env.example create mode 100755 examples/harbor-hermes-switchyard/run_phase2_cohort.sh create mode 100755 examples/harbor-hermes-switchyard/scripts/exec_process_group.py create mode 100755 examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh create mode 100755 examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py create mode 100755 examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh create mode 100755 examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py create mode 100755 examples/harbor-hermes-switchyard/scripts/validate_parallel_isolation.py create mode 100755 examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh create mode 100755 examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh create mode 100644 examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py diff --git a/examples/harbor-hermes-switchyard/.gitignore b/examples/harbor-hermes-switchyard/.gitignore new file mode 100644 index 000000000..bedbb2773 --- /dev/null +++ b/examples/harbor-hermes-switchyard/.gitignore @@ -0,0 +1,2 @@ +phase2-run.env +phase2-run.*.env diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 4556f14bc..808de23c0 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -1,276 +1,283 @@ # Harbor + Hermes + Switchyard evaluation -This example runs a Terminal-Bench 2.0 task through Harbor and Hermes while -Hermes owns an in-process NeMo Relay 0.7.0 runtime. Relay initializes static -pricing and observability components and activates Switchyard as a standard -dynamic native plugin. - -The example is deliberately a one-task integration reference. It does not -replace Harbor's task lifecycle and it is not the full 89-task coordinator. -The four-task regression command below is the Phase 1 readiness gate. +This example runs one complete Terminal-Bench 2.0 cohort through Harbor and +Hermes. Hermes owns an in-process NeMo Relay 0.7.0 runtime; Relay loads the +Switchyard native plugin, and Switchyard selects and calls the configured +provider route. Phase 2 is one resumable 89-task cohort. Multi-cohort execution +and result aggregation belong to Phase 3 and are intentionally out of scope. ## Pinned inputs | Dependency | Input used by this example | |---|---| -| NeMo Relay | Released Linux/amd64 `nemo-relay==0.7.0` wheel; that exact wheel is installed and its digest is recorded per run. | -| Hermes | `bbednarski9/hermes-agent`, branch `feat/relay-native-plugin-init`, detached commit `efb63e714abc436af88af9b0d6734751c199aa6d` (PR #77915). | -| Switchyard | `bbednarski9/Switchyard`, detached commit `8293936a0f5758aa1a782639d485b8b8948cf03e` (PR #270). | -| Harbor | `harbor==0.18.0`, dataset `terminal-bench@2.0`. | - -The branch names make the development inputs discoverable; only the full -commits are authoritative. Every checkout is detached and verified before -execution. The Hermes installer is followed by a final `uv sync --frozen` -against that commit's checked-in lock because its date-relative resolution -guard can otherwise make an older checkout appear stale. The verified Relay -0.7.0 platform wheel is then force-installed by digest without dependencies. -During installation only, the bridge advertises an inert `ffmpeg` command so -the task does not install an unrelated media stack; browser setup and bundled -skills are also disabled for this terminal-only evaluation. +| NeMo Relay | Released `nemo-relay==0.7.0` platform wheel, installed by digest rather than from this source checkout. | +| Hermes | `bbednarski9/hermes-agent`, detached commit `efb63e714abc436af88af9b0d6734751c199aa6d` from PR #77915. | +| Switchyard | `bbednarski9/Switchyard`, detached commit `5d9d3292d6154e44d50295d0d4a3fd4f144f2528` from PR #270. | +| Harbor | `harbor==0.18.0`, local export of dataset `terminal-bench@2.0`. | + +Every source checkout is detached and verified. The Hermes installer is +followed by `uv sync --frozen`, then the selected Relay 0.7.0 wheel is +force-installed without dependencies and verified by digest. ## Request and lifecycle ownership There is no Switchyard service in this topology: -1. Harbor creates the Terminal-Bench task environment and invokes its built-in - Hermes lifecycle through the temporary subclass in - `agents/harbor_hermes_agent.py`. -2. Hermes initializes Relay and asks Relay's public dynamic-plugin loader to - activate `nvidia.switchyard` from `[[plugins.dynamic]]`. -3. Relay owns the outer managed LLM operation and invokes the native execution - intercept. -4. The Switchyard plugin selects the route and its `switchyard-llm-client` - performs the provider HTTP request. -5. Hermes waits for Relay operations, plugin cleanup, subscribers, and - exporters before returning to Harbor. - -That split is important: Relay dispatches into the plugin intercept, while the -pinned Switchyard plugin owns the provider HTTP client. The direct receipt -records both facts without claiming a second routing service exists. - -Static components and dynamic plugins are separate concepts. The pricing and -schema-v3 observability components in `config/relay.toml.in` are static Relay -components. Switchyard is a standard dynamic native Relay plugin. Hermes -`[[dynamic_plugins]]` Python workers are not used, and the bridge rejects a -configuration that mixes the two activation models before provider traffic. - -## Why the temporary Harbor agent exists - -Harbor 0.18.0's built-in Hermes agent accepts a branch-like `version` and -clones the upstream NousResearch repository. It cannot select a fork plus an -immutable arbitrary commit, nor can it project this example's Relay config and -native bundle. `HarborHermesAgent` changes only installation, configuration -projection, and additional artifact framing; the inherited Harbor setup/run, -timeout, task, session export, and ATIF conversion remain in control. - -Remove this bridge and use `--agent hermes` once +1. Harbor owns the task container, Hermes lifecycle, timeout, verifier, and + task artifact collection. +2. The temporary adapter in `agents/harbor_hermes_agent.py` installs the exact + Hermes commit and projects Relay's config and native bundle. +3. Hermes initializes Relay; Relay's public loader activates + `nvidia.switchyard` from `[[plugins.dynamic]]`. +4. Relay dispatches the managed operation into the native intercept. +5. Switchyard selects a route and its client owns the provider HTTP request. +6. Hermes waits for operations, plugins, subscribers, and exporters before + returning to Harbor. + +The adapter can be removed after [hermes-agent#77915](https://github.com/NousResearch/hermes-agent/pull/77915) -is upstream **and** Harbor's built-in agent can install a released, pinned -compatible Hermes revision while projecting the Relay config and plugin -bundle. Merging the Hermes PR alone is not sufficient while Harbor remains -upstream-repository-only and branch-only. +is upstream and Harbor can install an immutable compatible Hermes revision +while projecting the Relay configuration and plugin bundle. + +## Configuration ownership + +The two configuration files have deliberately different responsibilities: + +- `phase2-run.env.example` is copied to an untracked, mode-`0600` + `phase2-run.env`. It contains per-machine paths, the run identity, Phoenix + destination, manually selected capacity, and the real + `SWITCHYARD_PROVIDER_AUTHORIZATION` header. +- `config/plugins.toml.in` is checked in and non-secret. It is the only source + of provider URLs, protocols, strong and weak models, routing/classifier + policy, native plugin manifest, authorization variable **name**, Relay + components, and OpenInference export behavior. + +The template configures Opus 4.6 as the strong route and Sonnet 4.6 as the +classifier and weak route. The coordinator derives its required route-diversity +gates from this TOML; environment variables cannot override these settings. +Runtime rendering is limited to the Hermes revision, collector endpoint, +Phoenix project/cohort attributes, and task-owned artifact locations. + +For each task, the runner writes only the provider Authorization header to a +mode-`0600` file in the host's canonical private temporary directory, +explicitly outside the run root, and bind-mounts it read-only at +`/run/secrets/switchyard-provider-authorization`. +The Hermes bridge reads and exports it inside the task container immediately +before the agent command. The credential value is therefore absent from Harbor +configuration, Docker Compose arguments, plans, logs, and retained evidence; +the temporary file is removed when the task runner exits. + +Harbor still requires a caller model. The example uses the intentionally +unserved `openai/ollama-route-stub` identity and projects a dead local OpenAI +endpoint. If Switchyard is bypassed, the request fails closed instead of +reaching a provider. + +## Host prerequisites + +- Linux or macOS, Bash, Python 3.11+, Docker, and `tmux`; +- a local, immutable Terminal-Bench 2.0 dataset export containing 89 tasks; +- a Switchyard plugin bundle and Relay 0.7.0 wheel matching Docker's + architecture (`x86_64` or `aarch64`); +- a Phoenix endpoint accepting OTLP/HTTP OpenInference traces; and +- provider and registry access for the full cohort. The all-89 admission uses + neither; the Docker admission makes no provider calls but may pull its image, + pinned sources, and packages when they are not cached. + +On macOS, keep the dataset, bundle, wheel, admission, and run roots under a +directory shared with Docker (normally `/Users/...`). + +Install the exact Harbor-side requirements: + +```bash +cd examples/harbor-hermes-switchyard +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt +``` + +Copy and protect the environment file outside the checkout. Replace every +placeholder, including the complete provider Authorization header. Do not +source this file into the interactive shell used to start `tmux`. -## Prerequisites +```bash +cp phase2-run.env.example /absolute/private/phase2-run.env +chmod 0600 /absolute/private/phase2-run.env +./scripts/validate_phase2_environment.sh /absolute/private/phase2-run.env +``` + +The validator reports names and paths only. It rejects legacy secret-file +variables and never renders or prints the authorization value. -- Docker with enough space to build one Linux/amd64 Rust plugin and task image; -- Python 3.11 or newer; -- access to `inference.nvidia.com` through its OpenAI-compatible endpoint; -- a Phoenix endpoint accepting OTLP/HTTP traces; and -- the provider authorization value in an environment variable. +## Phase 2 admission and runbook -On macOS, place bundle and run roots below a directory shared with Docker -(normally `/Users/...`). Do not assume `$TMPDIR` or `/private/tmp` is shared by -Colima merely because the same path exists inside its VM. +Run every stage with the same immutable inputs. If the dataset, concurrency, +architecture, Relay wheel, Switchyard library, plugin template, or Hermes +commit changes, regenerate the affected admission evidence before creating a +plan. -Create a host-side environment for Harbor and the validation tools: +For the commands below, enter a short-lived shell with tracing disabled: ```bash -cd examples/harbor-hermes-switchyard -python3 -m venv .venv -.venv/bin/python -m pip install -r requirements.txt -export HARBOR_BIN="$PWD/.venv/bin/harbor" -export PHASE1_PYTHON="$PWD/.venv/bin/python" +set +x +set -a +source /absolute/private/phase2-run.env +set +a +set +x ``` -The scripts never put the authorization value into TOML or command-line -configuration. They pass the selected environment variable into the task and -scan direct artifacts, Harbor logs, ATOF, ATIF, and OpenInference evidence for -the exact secret value. - -### Provider configuration ownership - -The rendered `/runtime/plugins.toml` is Relay and Switchyard's -authoritative provider configuration. It contains the strong and weak models, -classifier policy, protocol, upstream base URL and endpoint, and the **name** -of the environment variable holding the authorization header. The defaults -are the inference.nvidia.com catalog entries -`aws/anthropic/bedrock-claude-opus-4-6` (strong) and -`aws/anthropic/bedrock-claude-sonnet-4-6` (weak). - -The `llm_classifier` policy uses Sonnet as both the classifier and weak target. -On the first request it asks Sonnet for a structured capability verdict, then -routes the original request to Sonnet when `p_solve >= 0.5` or Opus otherwise. -Invalid, unavailable, or low-confidence classifier output fails safe to Opus. -The first decision is retained for the session, so later turns do not incur a -second classifier call. This two-model policy avoids adding a third provider, -but the packaged classifier prompt is not model-neutral; the threshold must be -revalidated if either model changes. - -`STRONG_MODEL`, `WEAK_MODEL`, `UPSTREAM_BASE_URL`, and `UPSTREAM_AUTH_ENV` are -preparation inputs used to materialize that immutable per-run file; they are -not independent provider configuration consumed by Relay. When -`INFERENCE_SECRETS_FILE` is set, the runner reads `NV_INFERENCEHUB_ENDPOINT` -and `NV_INFERENCEHUB_KEY` from it in short-lived subshells. It derives the -Bearer authorization value only in memory, unsets the raw variables, and -passes only `SWITCHYARD_PROVIDER_AUTHORIZATION` into the task. The secrets file -is never copied into the run root or a container. - -Harbor 0.18.0 still requires a `provider/model` value when constructing its -built-in Hermes lifecycle, and Hermes writes that call-side model into its CLI -configuration before Relay intercepts the operation. The runner passes -`openai/ollama-route-stub`: an intentionally unserved, Ollama-shaped caller -identity. `openai` describes only the caller protocol required by Harbor; the -stub is not a Switchyard target. `OPENAI_BASE_URL` is projected as the dead -local endpoint `http://127.0.0.1:9/v1`, so a request that bypasses Switchyard -fails closed rather than reaching a provider. The placeholder -`OPENAI_API_KEY` only satisfies Harbor/Hermes validation. Successful provider -traffic must use the real targets and authorization resolved by Switchyard -from `plugins.toml` and `header_env`. - -## Offline compatibility gate - -This is a preflight prerequisite for the first Harbor task run and whenever a -Hermes, Relay, Switchyard, plugin-config, or shutdown-lifecycle input changes. -It is not repeated before every task when those inputs are unchanged, and it -does not replace the single-task or regression gates. - -Build the pinned Linux plugin bundle, prepare a fresh run root, and run the -forked Hermes/Relay runtime against local fake provider and OTLP endpoints: +### 1. All-89 no-token admission + +This loads and uniquely selects all tasks, hashes their instructions and +verifiers, expands the complete Harbor job graph, denies registry/provider +access, and renders the runtime. It starts neither Docker nor an agent. ```bash -export EXAMPLE_ROOT="$PWD" -export SPIKE_ROOT="/absolute/new/spike-root" - -"$EXAMPLE_ROOT/scripts/build_switchyard_plugin.sh" /absolute/new/switchyard-bundle -"$PHASE1_PYTHON" "$EXAMPLE_ROOT/scripts/prepare_runtime.py" \ - --run-root "$SPIKE_ROOT" \ - --switchyard-bundle /absolute/new/switchyard-bundle \ - --upstream-base-url http://127.0.0.1:8000/v1 \ - --strong-model phase1/fake-strong \ - --weak-model phase1/fake-weak \ - --hermes-caller-model ollama-route-stub \ - --openinference-endpoint http://127.0.0.1:4318/v1/traces \ - --phoenix-project phase1-offline \ - --eval-cohort phase1-offline -"$EXAMPLE_ROOT/scripts/run_offline_compatibility_smoke.sh" "$SPIKE_ROOT" +mkdir -p "$PHASE2_ADMISSION_ROOT" +chmod 0700 "$PHASE2_ADMISSION_ROOT" +"$EVAL_PYTHON" "$EXAMPLE_ROOT/scripts/smoke_phase2_dataset.py" \ + --dataset-root "$TBENCH_DATASET_PATH" \ + --expected-count 89 \ + --concurrency "$TBENCH_CONCURRENCY" \ + --harbor-bin "$HARBOR_BIN" \ + --switchyard-bundle "$SWITCHYARD_BUNDLE" \ + --relay-wheel "$RELAY_WHEEL" \ + --relay-architecture "$RELAY_ARCHITECTURE" \ + --plugin-config-template "$PLUGIN_CONFIG_TEMPLATE" \ + --output "$PHASE2_SMOKE_EVIDENCE" ``` -This gate proves the exact detached Hermes checkout, released Relay wheel, -public loader path, one native Switchyard activation, a real fake-provider HTTP -request, routing marks, file sinks, mixed-mode rejection, and clean shutdown. +The passed evidence binds task names, task/instruction/verifier hashes, +concurrency, architecture, Relay wheel, Switchyard library, and plugin config. + +### 2. Docker offline runtime admission -The review gate targets `linux/amd64`, matching the Terminal-Bench task -environment. On an Apple Silicon Docker host, QEMU may crash while unloading a -native Rust plugin; an ARM control can distinguish that emulator failure from -an integration failure: +Prepare a fresh admission root with test-only structured overrides. Production +model, URL, and routing values remain owned by `plugins.toml.in`; these flags +exist only to point this closed offline test at its fake endpoints. ```bash -SWITCHYARD_TARGET_ARCHITECTURE=aarch64 \ - "$EXAMPLE_ROOT/scripts/build_switchyard_plugin.sh" /absolute/new/arm64-bundle -"$PHASE1_PYTHON" "$EXAMPLE_ROOT/scripts/prepare_runtime.py" \ - --run-root /absolute/new/arm64-spike-root \ - --switchyard-bundle /absolute/new/arm64-bundle \ - --relay-architecture aarch64 \ - --upstream-base-url http://127.0.0.1:8000/v1 \ - --strong-model phase1/fake-strong \ - --weak-model phase1/fake-weak \ - --hermes-caller-model ollama-route-stub \ +OFFLINE_ROOT="$PHASE2_ADMISSION_ROOT/offline-runtime" +"$EVAL_PYTHON" "$EXAMPLE_ROOT/scripts/prepare_runtime.py" \ + --run-root "$OFFLINE_ROOT" \ + --switchyard-bundle "$SWITCHYARD_BUNDLE" \ + --relay-wheel "$RELAY_WHEEL" \ + --relay-architecture "$RELAY_ARCHITECTURE" \ + --plugin-config-template "$PLUGIN_CONFIG_TEMPLATE" \ + --test-provider-base-url http://127.0.0.1:8000/v1 \ + --test-strong-model phase2/fake-strong \ + --test-weak-model phase2/fake-weak \ --openinference-endpoint http://127.0.0.1:4318/v1/traces \ - --phoenix-project phase1-offline-arm64 \ - --eval-cohort phase1-offline-arm64 -PHASE1_COMPAT_PLATFORM=linux/arm64 \ - "$EXAMPLE_ROOT/scripts/run_offline_compatibility_smoke.sh" \ - /absolute/new/arm64-spike-root + --phoenix-project phase2-offline \ + --eval-cohort phase2-offline + +case "$RELAY_ARCHITECTURE" in + x86_64) export OFFLINE_COMPAT_PLATFORM=linux/amd64 ;; + aarch64) export OFFLINE_COMPAT_PLATFORM=linux/arm64 ;; + *) echo "unsupported architecture" >&2; return 2 ;; +esac +"$EXAMPLE_ROOT/scripts/run_offline_compatibility_smoke.sh" \ + "$OFFLINE_ROOT" "$PHASE2_OFFLINE_EVIDENCE" ``` -That control validates the same source commits and lifecycle on a different -released Relay wheel architecture. It does **not** replace a passing -`linux/amd64` run on native amd64 infrastructure before merge. +This performs real Hermes→Relay→Switchyard calls against local fake provider +and OTLP endpoints and proves route selection, authorization injection, +observability, pinned-library loading, and clean shutdown. Its evidence binds +the same Hermes commit, Relay wheel, Switchyard library, architecture, and +plugin template consumed by the cohort. -`run_terminal_bench.sh` also accepts `RELAY_ARCHITECTURE=aarch64` together -with matching `SWITCHYARD_BUNDLE` and `RELAY_WHEEL` inputs. This is useful for -exercising Harbor's complete bridge and artifact path on a native Apple -Silicon Docker daemon. It remains a diagnostic control; the default and merge -gate stay `x86_64`. +### 3. Immutable plan -## Run one Terminal-Bench task - -Use a new absolute run root on every invocation: +The first command writes `plan.json`; any later invocation with different +immutable inputs is refused. Choose concurrency before this point. ```bash -export INFERENCE_SECRETS_FILE="/absolute/path/to/.inference_secrets" -export PHOENIX_BASE_URL="https://your-phoenix-endpoint" -export PHOENIX_PROJECT="harbor-hermes-switchyard-phase1" -export EVAL_COHORT="harbor-hermes-switchyard-phase1" +"$EXAMPLE_ROOT/run_phase2_cohort.sh" "$PHASE2_RUN_ROOT" --plan-only +``` + +`adaptive-rejection-sampler` is always first and serial. A passed validation +and upload result opens the parallel lane even when its benchmark reward is a +non-pass. -./run_terminal_bench.sh /absolute/new/run-root +### 4. Capacity and network preflight + +```bash +"$EXAMPLE_ROOT/run_phase2_cohort.sh" "$PHASE2_RUN_ROOT" --preflight-only ``` -The secrets file must define `NV_INFERENCEHUB_ENDPOINT` and -`NV_INFERENCEHUB_KEY`. Its path is only a launch input and is never rendered -into generated configuration or provenance. Advanced runs may override -`STRONG_MODEL`, `WEAK_MODEL`, or `UPSTREAM_BASE_URL`; changing either model -requires rerunning the offline gate and classifier routing smokes. +Preflight authenticates to each configured provider's model catalog and +requires both TOML-owned route models to be present without persisting the +authorization value. It writes `preflight.json` with the verified model IDs, +Docker CPU/memory/architecture, free disk, configured endpoints, selected +concurrency, reserve, and the calculated requirement. It rejects: -The default task is `adaptive-rejection-sampler`. Override it with -`TASK_NAME`. To avoid rebuilding Switchyard for each task, set -`SWITCHYARD_BUNDLE` to a previously built, immutable bundle. Set `RELAY_WHEEL` -to a downloaded 0.7.0 wheel to avoid a repeated package download. +```text +max(concurrency × parallel_task_memory_gb, largest_task_memory_gb) + + docker_reserve_gb > Docker memory +``` -A task is complete only if both of these files contain `"status": "passed"`: +It also rejects less than the configured free-disk minimum (100G by default), +concurrency above Docker's CPU count, and an architecture mismatch. The +defaults are a 2G parallel lane and 4G Docker reserve. -- `/validation.json` -- `/phoenix-upload.json` +### 5. Durable launch under tmux -`reward.task_passed=false` is a valid completed benchmark observation and is -not retried when both evidence gates pass. +Exit the secret-bearing admission shell first. From a shell where the protected +file has **not** been sourced, start one detached supervisor. Only the file path +is placed in the tmux server environment; the child sources it with xtrace +disabled and persists output below the run root. -## Phase 1 regression gate +```bash +exit # only when returning from the short-lived admission shell above +./scripts/launch_phase2_tmux.sh \ + /absolute/private/phase2-run.env \ + harbor-hermes-switchyard-phase2-run-1 +``` -Run all historical risk cases independently: +Operational commands: ```bash -./scripts/run_phase1_regressions.sh /absolute/new/regression-root +# Detect a live duplicate (success means the session exists). +tmux has-session -t harbor-hermes-switchyard-phase2-run-1 + +# Attach; detach without stopping the run with Ctrl-b d. +tmux attach-session -t harbor-hermes-switchyard-phase2-run-1 + +# Inspect durable output and sanitized cohort progress. +tail -F /absolute/path/to/phase2-run-root/supervisor.log +jq '{status,completed_tasks,planned_tasks,benchmark_pass_count,benchmark_nonpass_count}' \ + /absolute/path/to/phase2-run-root/summary.json + +# Graceful interruption. +tmux send-keys -t harbor-hermes-switchyard-phase2-run-1 C-c + +# After the old session exits, resume the same immutable root. +./scripts/launch_phase2_tmux.sh \ + /absolute/private/phase2-run.env \ + harbor-hermes-switchyard-phase2-run-1 ``` -| Task | Assertion | -|---|---| -| `adaptive-rejection-sampler` | Provider/config projection, routing marks, receipt, cleanup, and secret scan. | -| `circuit-fibsqrt` | A deterministic post-response test fault preserves the completed response and records the late failure separately. | -| `gpt2-codegolf` | Harbor's bounded agent timeout applies and no Hermes/plugin process survives the task container. | -| `overfull-hbox` | Streaming validation and bounded Phoenix batching preserve the completed result under a larger export load. | - -The deterministic `circuit-fibsqrt` fault is injected only after inherited -Hermes execution returns. It tests the result-framing regression without -corrupting the Relay plugin lifecycle or disabling Phoenix upload. - -## Evidence and safety properties - -Each run root is immutable and private. Preparation refuses an existing root. -The runtime snapshot contains config and dependency digests; it never contains -credential values. Direct task artifacts include: - -- `direct-hermes-result.json`; -- `direct-hermes-receipt.json`; -- `relay/trajectory.atof.jsonl`; -- `relay/atif/trajectory-.atif.json`; -- bounded Hermes diagnostics; -- `validation.json`; and -- `phoenix-upload.json`. - -Artifact validation rejects symlinks and canonical paths escaping the declared -root. Phoenix import is streaming, bounded in batches, retry-limited, and runs -only after the task has returned and OpenInference evidence exists. - -Phase 2 (a parallel 89-task cohort) and Phase 3 (multiple independent cohorts -and aggregated reporting) intentionally remain outside this first example PR. +The supervisor returns `0` after complete acceptance, `20` for a preserved +integration/harness blocker, and retries other exits with bounded exponential +backoff. The OS advisory lock rejects a second live coordinator. Validated +attempts are immutable and preserved on restart. `tmux` survives terminal +logout, not host reboot; after reboot, launch it again against the same root. +Agent setup also retries transient `apt-get` failures three times locally; +exhausted package-manager failures are classified as infrastructure and remain +subject to the cohort's bounded retry limit. + +## Completion gates + +A task is complete only when both its `validation.json` and +`phoenix-upload.json` have `status=passed`. A benchmark +`reward.task_passed=false` is a valid completed result and is never retried. + +The cohort passes only when: + +- all 89 tasks are independently validated and uploaded; +- direct artifacts and logs pass secret scans; +- cache-read evidence is nonzero; +- both models derived from `plugins.toml.in` appear in committed routes; and +- `summary.json.status` is `passed`. + +`report.md` is regenerated after each completed attempt and is safe for +progress review. Phase 3 multiple-run orchestration and aggregated reports are +not part of this runbook. diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py index cf4936d54..6f11aef58 100644 --- a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -30,7 +30,9 @@ _DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" _DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" _DEFAULT_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" -_DEFAULT_SWITCHYARD_COMMIT = "8293936a0f5758aa1a782639d485b8b8948cf03e" +_DEFAULT_SWITCHYARD_COMMIT = "5d9d3292d6154e44d50295d0d4a3fd4f144f2528" +_ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]*") +_PROVIDER_AUTHORIZATION_FILE = "/run/secrets/switchyard-provider-authorization" def _require_full_sha(value: str, name: str) -> str: @@ -105,15 +107,102 @@ def _validate_relay_config(path: Path) -> None: dynamic = plugins.get("dynamic") if isinstance(plugins, dict) else None if not isinstance(dynamic, list) or len(dynamic) != 1: raise ValueError("Relay config must contain exactly one [[plugins.dynamic]] record") - manifest = dynamic[0].get("manifest") if isinstance(dynamic[0], dict) else None - if not isinstance(manifest, str) or not manifest.endswith("/relay-plugin.toml"): - raise ValueError("the dynamic plugin must reference a Relay plugin manifest") + plugin = dynamic[0] + manifest = plugin.get("manifest") if isinstance(plugin, dict) else None + if manifest != "/opt/relay-plugins/nvidia.switchyard/relay-plugin.toml": + raise ValueError("the dynamic plugin must reference the staged Switchyard manifest") + + plugin_config = plugin.get("config") + if not isinstance(plugin_config, dict) or plugin_config.get("version") != 2: + raise ValueError("the Switchyard plugin must use config version = 2") + algorithm = plugin_config.get("algorithm") + expected_algorithm = { + "kind": "llm_classifier", + "classifier_target": "weak", + "weak_target": "weak", + "strong_target": "strong", + "base_threshold": 0.5, + "recent_turn_window": 0, + "session_affinity": True, + "message_hash_fallback": True, + } + if algorithm != expected_algorithm: + raise ValueError("the Switchyard classifier contract does not match the Phase 1 design") + if plugin_config.get("default_targets") != {"openai_chat": "strong"}: + raise ValueError("the Switchyard OpenAI default target must be strong") + + targets = plugin_config.get("targets") + if not isinstance(targets, dict) or set(targets) != {"strong", "weak"}: + raise ValueError("Switchyard must define exactly the strong and weak targets") + provider_models: set[str] = set() + for name, target in targets.items(): + if not isinstance(target, dict): + raise ValueError(f"Switchyard target {name!r} must be a table") + if target.get("protocol") != "openai_chat" or target.get("endpoint") != "/v1/chat/completions": + raise ValueError(f"Switchyard target {name!r} must use the OpenAI chat protocol") + if target.get("drop_caller_extra_body") is not True: + raise ValueError(f"Switchyard target {name!r} must drop Hermes' caller-specific extra_body wrapper") + base_url = target.get("base_url") + parsed_base_url = urlsplit(base_url) if isinstance(base_url, str) else None + if ( + parsed_base_url is None + or parsed_base_url.scheme not in {"http", "https"} + or not parsed_base_url.hostname + or parsed_base_url.username is not None + or parsed_base_url.password is not None + ): + raise ValueError(f"Switchyard target {name!r} must use a credential-free HTTP(S) base URL") + model = target.get("model") + if not isinstance(model, str) or not model: + raise ValueError(f"Switchyard target {name!r} must define a model") + provider_models.add(model) + header_env = target.get("header_env") + authorization_env = header_env.get("authorization") if isinstance(header_env, dict) else None + if not isinstance(authorization_env, str) or not _ENV_NAME.fullmatch(authorization_env): + raise ValueError(f"Switchyard target {name!r} must source authorization from an environment variable") + if len(provider_models) != 2: + raise ValueError("Switchyard strong and weak targets must use distinct models") + + pricing = _find_named_component(config, "pricing") + pricing_config = pricing.get("config") + sources = pricing_config.get("sources") if isinstance(pricing_config, dict) else None + if not isinstance(sources, list) or len(sources) != 1 or sources[0].get("type") != "inline": + raise ValueError("pricing must use exactly one inline catalog") + catalog = sources[0].get("catalog") + entries = catalog.get("entries") if isinstance(catalog, dict) and catalog.get("version") == 1 else None + if not isinstance(entries, list) or {entry.get("model_id") for entry in entries} != provider_models: + raise ValueError("pricing entries must match the Switchyard provider models") + for entry in entries: + rates = entry.get("rates") if isinstance(entry, dict) else None + if not isinstance(rates, dict) or any( + not isinstance(rates.get(key), (int, float)) or rates[key] <= 0 + for key in ("input_per_million", "output_per_million", "cache_read_per_million") + ): + raise ValueError("pricing entries must contain positive input, output, and cache-read rates") - _find_named_component(config, "pricing") observability = _find_named_component(config, "observability") observability_config = observability.get("config") if not isinstance(observability_config, dict) or observability_config.get("version") != 3: raise ValueError("the observability component must use schema version = 3") + atif = observability_config.get("atif") + caller_model = atif.get("model_name") if isinstance(atif, dict) else None + if not isinstance(caller_model, str) or not caller_model or caller_model in provider_models: + raise ValueError("the fail-closed Hermes caller model must not be a Switchyard provider model") + opentelemetry = observability_config.get("opentelemetry") + endpoints = opentelemetry.get("endpoints") if isinstance(opentelemetry, dict) else None + if not isinstance(opentelemetry, dict) or opentelemetry.get("enabled") is not True: + raise ValueError("OpenTelemetry export must be enabled") + if not isinstance(endpoints, list) or len(endpoints) != 1: + raise ValueError("observability must define exactly one OpenInference endpoint") + endpoint = endpoints[0] + if endpoint.get("type") != "openinference" or endpoint.get("transport") != "http_binary": + raise ValueError("the only telemetry endpoint must be OpenInference over OTLP/HTTP protobuf") + resource_attributes = endpoint.get("resource_attributes") + if not isinstance(resource_attributes, dict) or not all( + isinstance(resource_attributes.get(key), str) and resource_attributes[key] + for key in ("openinference.project.name", "evaluation.cohort") + ): + raise ValueError("OpenInference must carry project and evaluation cohort resource attributes") def reject_literal_headers(value: Any, location: str = "config") -> None: if isinstance(value, dict): @@ -163,6 +252,7 @@ def __init__( self.relay_wheel_path = Path(relay_wheel_path).expanduser().resolve() self.artifact_root = artifact_root.rstrip("/") self.inject_post_response_failure = inject_post_response_failure + self._load_provider_authorization = False if not self.artifact_root.startswith("/logs/agent/"): raise ValueError("artifact_root must be an absolute child of /logs/agent") if not self.relay_config_path.is_file(): @@ -197,14 +287,44 @@ def __init__( extra_env["HERMES_NEMO_RELAY_PLUGINS_TOML"] = "/tmp/hermes/relay/plugins.toml" super().__init__(*args, version=self.commit, extra_env=extra_env, **kwargs) + @override + async def exec_as_agent( + self, + environment: BaseEnvironment, + command: str, + env: dict[str, str] | None = None, + cwd: str | None = None, + timeout_sec: int | None = None, + ) -> Any: + if self._load_provider_authorization: + secret_file = shlex.quote(_PROVIDER_AUTHORIZATION_FILE) + command = ( + f"test -r {secret_file}; " + f'export SWITCHYARD_PROVIDER_AUTHORIZATION="$(cat -- {secret_file})"; ' + 'test -n "$SWITCHYARD_PROVIDER_AUTHORIZATION"; ' + f"{command}" + ) + return await super().exec_as_agent( + environment, + command, + env=env, + cwd=cwd, + timeout_sec=timeout_sec, + ) + @override async def install(self, environment: BaseEnvironment) -> None: await self.exec_as_root( environment, command=( - "apt-get update && " - "apt-get install -y --no-install-recommends " - "ca-certificates build-essential curl git ripgrep xz-utils" + "set -euo pipefail; last_status=1; " + "for attempt in 1 2 3; do " + "if apt-get update && apt-get install -y --no-install-recommends " + "ca-certificates build-essential curl git ripgrep xz-utils; then exit 0; " + "else last_status=$?; fi; " + 'if [ "$attempt" -eq 3 ]; then break; fi; ' + "rm -rf /var/lib/apt/lists/partial; sleep $((attempt * 5)); " + 'done; exit "$last_status"' ), env={"DEBIAN_FRONTEND": "noninteractive"}, ) @@ -320,7 +440,11 @@ async def run( started_at = time.time() error: BaseException | None = None try: - await super().run(instruction, environment, context) + self._load_provider_authorization = True + try: + await super().run(instruction, environment, context) + finally: + self._load_provider_authorization = False except BaseException as exc: error = exc raise diff --git a/examples/harbor-hermes-switchyard/config/relay.toml.in b/examples/harbor-hermes-switchyard/config/plugins.toml.in similarity index 74% rename from examples/harbor-hermes-switchyard/config/relay.toml.in rename to examples/harbor-hermes-switchyard/config/plugins.toml.in index 983e8d94e..06a34f157 100644 --- a/examples/harbor-hermes-switchyard/config/relay.toml.in +++ b/examples/harbor-hermes-switchyard/config/plugins.toml.in @@ -15,32 +15,34 @@ version = 1 [[components.config.sources.catalog.entries]] provider = "openai" -model_id = "@STRONG_MODEL@" +model_id = "aws/anthropic/bedrock-claude-opus-4-6" currency = "USD" unit = "per_token" -pricing_as_of = "2026-08-05" -pricing_source = "harbor-hermes-switchyard-example" +pricing_as_of = "2026-05-27" +pricing_source = "Anthropic public list pricing" [components.config.sources.catalog.entries.rates] -input_per_million = 0.0 -output_per_million = 0.0 -cache_read_per_million = 0.0 +input_per_million = 5.0 +output_per_million = 25.0 +cache_read_per_million = 0.5 +cache_write_per_million = 6.25 [components.config.sources.catalog.entries.prompt_cache] read_accounting = "included_in_prompt_tokens" [[components.config.sources.catalog.entries]] provider = "openai" -model_id = "@WEAK_MODEL@" +model_id = "aws/anthropic/bedrock-claude-sonnet-4-6" currency = "USD" unit = "per_token" -pricing_as_of = "2026-08-05" -pricing_source = "harbor-hermes-switchyard-example" +pricing_as_of = "2026-05-27" +pricing_source = "Anthropic public list pricing" [components.config.sources.catalog.entries.rates] -input_per_million = 0.0 -output_per_million = 0.0 -cache_read_per_million = 0.0 +input_per_million = 3.0 +output_per_million = 15.0 +cache_read_per_million = 0.3 +cache_write_per_million = 3.75 [components.config.sources.catalog.entries.prompt_cache] read_accounting = "included_in_prompt_tokens" @@ -65,7 +67,7 @@ filename = "trajectory.atof.jsonl" enabled = true agent_name = "Hermes" agent_version = "@HERMES_COMMIT@" -model_name = "@HERMES_CALLER_MODEL@" +model_name = "ollama-route-stub" output_directory = "/logs/agent/direct-hermes/relay/atif" filename_template = "trajectory-{session_id}.atif.json" @@ -99,7 +101,6 @@ classifier_target = "weak" weak_target = "weak" strong_target = "strong" base_threshold = 0.5 -min_confidence = 0.0 recent_turn_window = 0 session_affinity = true message_hash_fallback = true @@ -108,21 +109,23 @@ message_hash_fallback = true openai_chat = "strong" [plugins.dynamic.config.targets.strong] -model = "@STRONG_MODEL@" +model = "aws/anthropic/bedrock-claude-opus-4-6" protocol = "openai_chat" endpoint = "/v1/chat/completions" -base_url = "@UPSTREAM_BASE_URL@" +base_url = "https://inference-api.nvidia.com/v1" weight = 1 +drop_caller_extra_body = true [plugins.dynamic.config.targets.strong.header_env] -authorization = "@UPSTREAM_AUTH_ENV@" +authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" [plugins.dynamic.config.targets.weak] -model = "@WEAK_MODEL@" +model = "aws/anthropic/bedrock-claude-sonnet-4-6" protocol = "openai_chat" endpoint = "/v1/chat/completions" -base_url = "@UPSTREAM_BASE_URL@" +base_url = "https://inference-api.nvidia.com/v1" weight = 1 +drop_caller_extra_body = true [plugins.dynamic.config.targets.weak.header_env] -authorization = "@UPSTREAM_AUTH_ENV@" +authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" diff --git a/examples/harbor-hermes-switchyard/phase2-run.env.example b/examples/harbor-hermes-switchyard/phase2-run.env.example new file mode 100644 index 000000000..8b0596d68 --- /dev/null +++ b/examples/harbor-hermes-switchyard/phase2-run.env.example @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Copy this file outside the checkout, chmod 0600, and replace every /absolute +# placeholder. Never commit the populated file. +EXAMPLE_ROOT=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard +PHASE2_RUN_ID=harbor-hermes-switchyard-phase2-run-1 +PHASE2_RUN_ROOT=/absolute/path/to/phase2-runs/harbor-hermes-switchyard-phase2-run-1 +PHASE2_ADMISSION_ROOT=/absolute/path/to/phase2-admission + +HARBOR_BIN=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/harbor +EVAL_PYTHON=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/python +TBENCH_DATASET_PATH=/absolute/path/to/exported/terminal-bench +SWITCHYARD_BUNDLE=/absolute/path/to/pinned-switchyard-bundle +RELAY_WHEEL=/absolute/path/to/nemo_relay-0.7.0-platform-wheel.whl +RELAY_ARCHITECTURE=x86_64 +PLUGIN_CONFIG_TEMPLATE=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/config/plugins.toml.in + +PHASE2_SMOKE_EVIDENCE=/absolute/path/to/phase2-admission/all-89-smoke.json +PHASE2_OFFLINE_EVIDENCE=/absolute/path/to/phase2-admission/offline-admission.json +PHOENIX_BASE_URL=https://your-phoenix-endpoint +PHOENIX_PROJECT=harbor-hermes-switchyard-phase2-run-1 +EVAL_COHORT=harbor-hermes-switchyard-phase2-run-1 + +TBENCH_SAMPLE_COUNT=89 +TBENCH_CANARY_TASK=adaptive-rejection-sampler +TBENCH_CONCURRENCY=4 +TBENCH_PARALLEL_MAX_MEMORY_GB=2 +TBENCH_DOCKER_MEMORY_RESERVE_GB=4 +TBENCH_MINIMUM_FREE_GB=100 + +# This is the real Authorization header consumed by the Switchyard plugin. +# Quote the value so shell metacharacters are not interpreted when sourced. +SWITCHYARD_PROVIDER_AUTHORIZATION='Bearer replace-with-provider-token' diff --git a/examples/harbor-hermes-switchyard/requirements.txt b/examples/harbor-hermes-switchyard/requirements.txt index 5d78e7ac0..00cc8a995 100644 --- a/examples/harbor-hermes-switchyard/requirements.txt +++ b/examples/harbor-hermes-switchyard/requirements.txt @@ -5,3 +5,4 @@ harbor==0.18.0 opentelemetry-proto==1.38.0 protobuf==6.33.5 typing-extensions==4.15.0 +tomli-w==1.2.0 diff --git a/examples/harbor-hermes-switchyard/run_phase2_cohort.sh b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh new file mode 100755 index 000000000..ebf4441dd --- /dev/null +++ b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +run_root="${1:-${PHASE2_RUN_ROOT:-}}" +if [[ -z "$run_root" || "$run_root" != /* ]]; then + echo "usage: $0 /absolute/phase2-run-root [coordinator options]" >&2 + exit 2 +fi +shift + +dataset="${TBENCH_DATASET:-terminal-bench@2.0}" +dataset_name="${dataset%@*}" +dataset_export_root="${TBENCH_DATASET_EXPORT_ROOT:-$(dirname "$run_root")/harbor-datasets}" +dataset_root="${TBENCH_DATASET_PATH:-$dataset_export_root/$dataset_name}" +harbor_bin="${HARBOR_BIN:-$example_root/.venv/bin/harbor}" +python_bin="${EVAL_PYTHON:-$example_root/.venv/bin/python}" +smoke_evidence="${PHASE2_SMOKE_EVIDENCE:-}" +offline_evidence="${PHASE2_OFFLINE_EVIDENCE:-}" +phoenix_url="${PHOENIX_BASE_URL:-}" +phoenix_project="${PHOENIX_PROJECT:-}" +eval_cohort="${EVAL_COHORT:-}" +switchyard_bundle="${SWITCHYARD_BUNDLE:-}" +relay_wheel="${RELAY_WHEEL:-}" +relay_architecture="${RELAY_ARCHITECTURE:-x86_64}" +plugin_config_template="${PLUGIN_CONFIG_TEMPLATE:-$example_root/config/plugins.toml.in}" +sample_count="${TBENCH_SAMPLE_COUNT:-89}" +canary_task="${TBENCH_CANARY_TASK:-adaptive-rejection-sampler}" +concurrency="${TBENCH_CONCURRENCY:-4}" +parallel_memory_gb="${TBENCH_PARALLEL_MAX_MEMORY_GB:-2}" +docker_memory_reserve_gb="${TBENCH_DOCKER_MEMORY_RESERVE_GB:-4}" +minimum_free_gb="${TBENCH_MINIMUM_FREE_GB:-100}" + +for required in \ + "$harbor_bin" \ + "$python_bin" \ + "$smoke_evidence" \ + "$offline_evidence" \ + "$dataset_root" \ + "$plugin_config_template" \ + "$switchyard_bundle/relay-plugin.toml" \ + "$relay_wheel"; do + [[ -e "$required" ]] || { + echo "required Phase 2 input is missing: $required" >&2 + exit 2 + } +done +if [[ -z "${SWITCHYARD_PROVIDER_AUTHORIZATION:-}" ]]; then + echo "SWITCHYARD_PROVIDER_AUTHORIZATION must be set by the protected Phase 2 environment file" >&2 + exit 2 +fi +for label in phoenix_url phoenix_project eval_cohort; do + [[ -n "${!label}" ]] || { + echo "${label^^} must be set" >&2 + exit 2 + } +done + +if [[ "$dataset_root" != /* || ! -d "$dataset_root" ]]; then + echo "TBENCH_DATASET_PATH must select an existing absolute local dataset directory" >&2 + echo "Phase 2 never downloads or resolves a dataset through the Harbor registry" >&2 + exit 2 +fi + +exec "$python_bin" "$example_root/scripts/run_phase2_cohort.py" \ + --run-root "$run_root" \ + --dataset "$dataset" \ + --dataset-root "$dataset_root" \ + --sample-count "$sample_count" \ + --canary-task "$canary_task" \ + --concurrency "$concurrency" \ + --parallel-max-memory-gb "$parallel_memory_gb" \ + --docker-memory-reserve-gb "$docker_memory_reserve_gb" \ + --minimum-free-gb "$minimum_free_gb" \ + --smoke-evidence "$smoke_evidence" \ + --offline-evidence "$offline_evidence" \ + --plugin-config-template "$plugin_config_template" \ + --task-runner "$example_root/run_terminal_bench.sh" \ + --harbor-bin "$harbor_bin" \ + --python-bin "$python_bin" \ + --phoenix-url "$phoenix_url" \ + --phoenix-project "$phoenix_project" \ + --eval-cohort "$eval_cohort" \ + --switchyard-bundle "$switchyard_bundle" \ + --relay-wheel "$relay_wheel" \ + --relay-architecture "$relay_architecture" \ + "$@" diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index 5a87a00ba..471b262e8 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -7,20 +7,22 @@ set -euo pipefail example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_root="${1:-}" task_name="${TASK_NAME:-adaptive-rejection-sampler}" -strong_model="${STRONG_MODEL:-aws/anthropic/bedrock-claude-opus-4-6}" -weak_model="${WEAK_MODEL:-aws/anthropic/bedrock-claude-sonnet-4-6}" -hermes_caller_model="${HERMES_CALLER_MODEL:-ollama-route-stub}" -inference_secrets_file="${INFERENCE_SECRETS_FILE:-}" -upstream_auth_env="${UPSTREAM_AUTH_ENV:-SWITCHYARD_PROVIDER_AUTHORIZATION}" +upstream_auth_env="SWITCHYARD_PROVIDER_AUTHORIZATION" fail_closed_openai_base_url="http://127.0.0.1:9/v1" phoenix_base="${PHOENIX_BASE_URL:-}" phoenix_project="${PHOENIX_PROJECT:-harbor-hermes-switchyard-phase1}" eval_cohort="${EVAL_COHORT:-harbor-hermes-switchyard-phase1}" -harbor_bin="${HARBOR_BIN:-harbor}" -python_bin="${PHASE1_PYTHON:-python3}" +default_harbor_bin="$example_root/.venv/bin/harbor" +default_python_bin="$example_root/.venv/bin/python" +harbor_bin="${HARBOR_BIN:-$default_harbor_bin}" +python_bin="${EVAL_PYTHON:-${PHASE1_PYTHON:-$default_python_bin}}" +expected_harbor_version="0.18.0" +eval_phase="${EVAL_PHASE:-phase1}" +tbench_dataset_path="${TBENCH_DATASET_PATH:-}" switchyard_bundle="${SWITCHYARD_BUNDLE:-}" relay_wheel="${RELAY_WHEEL:-}" relay_architecture="${RELAY_ARCHITECTURE:-x86_64}" +plugin_config_template="${PLUGIN_CONFIG_TEMPLATE:-$example_root/config/plugins.toml.in}" agent_timeout_multiplier="${AGENT_TIMEOUT_MULTIPLIER:-3}" agent_setup_timeout_multiplier="${AGENT_SETUP_TIMEOUT_MULTIPLIER:-6}" environment_build_timeout_multiplier="${ENVIRONMENT_BUILD_TIMEOUT_MULTIPLIER:-6}" @@ -35,54 +37,14 @@ if [[ -e "$run_root" ]]; then echo "run root already exists: $run_root" >&2 exit 2 fi -if [[ ! "$upstream_auth_env" =~ ^[A-Z_][A-Z0-9_]*$ ]]; then - echo "UPSTREAM_AUTH_ENV must be an uppercase environment variable name" >&2 - exit 2 -fi - -nv_inferencehub_endpoint="${NV_INFERENCEHUB_ENDPOINT:-}" -nv_inferencehub_key="${NV_INFERENCEHUB_KEY:-}" -if [[ -n "$inference_secrets_file" ]]; then - [[ -r "$inference_secrets_file" ]] || { - echo "INFERENCE_SECRETS_FILE is not readable: $inference_secrets_file" >&2 - exit 2 - } - load_secret_value() { - local variable_name="$1" - ( - set +x - # The file is sourced only in this short-lived subshell. Its other - # variables never enter Harbor's environment. - source "$inference_secrets_file" - printf '%s' "${!variable_name:-}" - ) - } - [[ -n "$nv_inferencehub_endpoint" ]] || \ - nv_inferencehub_endpoint="$(load_secret_value NV_INFERENCEHUB_ENDPOINT)" - [[ -n "$nv_inferencehub_key" ]] || \ - nv_inferencehub_key="$(load_secret_value NV_INFERENCEHUB_KEY)" -fi - -upstream_base_url="${UPSTREAM_BASE_URL:-$nv_inferencehub_endpoint}" -upstream_base_url="${upstream_base_url%/chat/completions}" -if [[ -z "${!upstream_auth_env:-}" && -n "$nv_inferencehub_key" ]]; then - if [[ "$nv_inferencehub_key" == "Bearer "* ]]; then - printf -v "$upstream_auth_env" '%s' "$nv_inferencehub_key" - else - printf -v "$upstream_auth_env" 'Bearer %s' "$nv_inferencehub_key" - fi - export "$upstream_auth_env" -fi -# Do not propagate the raw inference variables to Harbor or task containers. -unset NV_INFERENCEHUB_ENDPOINT NV_INFERENCEHUB_KEY nv_inferencehub_endpoint nv_inferencehub_key -for required in "$strong_model" "$weak_model" "$hermes_caller_model" "$upstream_base_url" "$phoenix_base"; do +for required in "$plugin_config_template" "$phoenix_base"; do [[ -n "$required" ]] || { - echo "model names, inference endpoint, and PHOENIX_BASE_URL are required" >&2 + echo "PLUGIN_CONFIG_TEMPLATE and PHOENIX_BASE_URL are required" >&2 exit 2 } done -if [[ "$strong_model" == "$weak_model" ]]; then - echo "STRONG_MODEL and WEAK_MODEL must be distinct" >&2 +if [[ ! -f "$plugin_config_template" ]]; then + echo "plugin configuration template is missing: $plugin_config_template" >&2 exit 2 fi for dependency in curl docker "$harbor_bin" "$python_bin"; do @@ -91,6 +53,31 @@ for dependency in curl docker "$harbor_bin" "$python_bin"; do exit 1 } done +observed_harbor_version="$($python_bin -c 'import importlib.metadata; print(importlib.metadata.version("harbor"))')" +if [[ "$observed_harbor_version" != "$expected_harbor_version" ]]; then + echo "Harbor $expected_harbor_version is required; $python_bin provides $observed_harbor_version" >&2 + exit 2 +fi +observed_harbor_cli_version="$($harbor_bin --version)" +if [[ "$observed_harbor_cli_version" != "$expected_harbor_version" ]]; then + echo "Harbor CLI $expected_harbor_version is required; $harbor_bin reports $observed_harbor_cli_version" >&2 + exit 2 +fi +if [[ ! "$eval_phase" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "EVAL_PHASE must contain only lowercase letters, digits, and hyphens" >&2 + exit 2 +fi +dataset_args=(--dataset terminal-bench@2.0) +if [[ -n "$tbench_dataset_path" ]]; then + if [[ "$tbench_dataset_path" != /* || ! -d "$tbench_dataset_path" ]]; then + echo "TBENCH_DATASET_PATH must be an absolute local dataset directory" >&2 + exit 2 + fi + dataset_args=(--path "$tbench_dataset_path") +elif [[ "$eval_phase" == "phase2" ]]; then + echo "Phase 2 requires TBENCH_DATASET_PATH and never resolves the remote registry" >&2 + exit 2 +fi if [[ -z "${!upstream_auth_env:-}" ]]; then echo "required provider authorization environment variable is unset: $upstream_auth_env" >&2 exit 2 @@ -101,9 +88,13 @@ if [[ "$relay_architecture" != "x86_64" && "$relay_architecture" != "aarch64" ]] fi docker info >/dev/null -curl --fail --silent --show-error --max-time 10 "$phoenix_base" >/dev/null +curl --fail --silent --show-error \ + --connect-timeout 5 --max-time 10 \ + --retry 2 --retry-all-errors --retry-delay 2 \ + "$phoenix_base" >/dev/null temporary_build="" +temporary_secret_dir="" collector_name="" collector_running=0 cleanup() { @@ -114,12 +105,39 @@ cleanup() { if [[ -n "$temporary_build" && -d "$temporary_build" ]]; then rm -rf "$temporary_build" fi + if [[ -n "$temporary_secret_dir" && -d "$temporary_secret_dir" ]]; then + rm -rf "$temporary_secret_dir" + fi return "$status" } trap cleanup EXIT +host_temporary_root="$(cd "${TMPDIR:-/tmp}" && pwd -P)" +case "$host_temporary_root/" in + "$run_root/"*) + echo "Host temporary directory must be outside the task run root" >&2 + exit 2 + ;; +esac +temporary_secret_dir="$(mktemp -d "$host_temporary_root/harbor-phase2-secret.XXXXXX")" +chmod 0700 "$temporary_secret_dir" +provider_authorization_file="$temporary_secret_dir/switchyard-provider-authorization" +(umask 077; printf '%s' "${!upstream_auth_env}" >"$provider_authorization_file") +chmod 0600 "$provider_authorization_file" +provider_authorization_target="/run/secrets/switchyard-provider-authorization" +mounts_json="$($python_bin -c ' +import json, sys +print(json.dumps([{ + "type": "bind", + "source": sys.argv[1], + "target": sys.argv[2], + "read_only": True, + "bind": {"create_host_path": False}, +}], separators=(",", ":"))) +' "$provider_authorization_file" "$provider_authorization_target")" + if [[ -z "$switchyard_bundle" ]]; then - temporary_build="$(mktemp -d "$(dirname "$run_root")/.phase1-switchyard-build.XXXXXX")" + temporary_build="$(mktemp -d "$(dirname "$run_root")/.switchyard-build.XXXXXX")" switchyard_bundle="$temporary_build/bundle" SWITCHYARD_TARGET_ARCHITECTURE="$relay_architecture" \ "$example_root/scripts/build_switchyard_plugin.sh" "$switchyard_bundle" @@ -133,18 +151,13 @@ with socket.socket() as sock: PY )" openinference_endpoint="http://host.docker.internal:$free_port/v1/traces" -upstream_host="$($python_bin -c 'import sys; from urllib.parse import urlsplit; print(urlsplit(sys.argv[1]).hostname or "")' "$upstream_base_url")" prepare_args=( "$example_root/scripts/prepare_runtime.py" --run-root "$run_root" --switchyard-bundle "$switchyard_bundle" --relay-architecture "$relay_architecture" - --upstream-base-url "$upstream_base_url" - --upstream-auth-env "$upstream_auth_env" - --strong-model "$strong_model" - --weak-model "$weak_model" - --hermes-caller-model "$hermes_caller_model" + --plugin-config-template "$plugin_config_template" --openinference-endpoint "$openinference_endpoint" --phoenix-project "$phoenix_project" --eval-cohort "$eval_cohort" @@ -154,6 +167,8 @@ if [[ -n "$relay_wheel" ]]; then fi "$python_bin" "${prepare_args[@]}" >"$run_root.prepare.log" +hermes_caller_model="$($python_bin -c 'import json,sys; print(json.load(open(sys.argv[1]))["routing"]["hermes_caller_model"])' "$run_root/runtime/provenance.json")" + relay_wheel_sha256="$($python_bin -c 'import json,sys; print(json.load(open(sys.argv[1]))["nemo_relay"]["wheel_sha256"])' "$run_root/runtime/provenance.json")" relay_wheel_path="$($python_bin -c 'import json,pathlib,sys; p=json.load(open(sys.argv[1])); print(pathlib.Path(sys.argv[1]).parent / "wheels" / p["nemo_relay"]["wheel"])' "$run_root/runtime/provenance.json")" @@ -176,11 +191,18 @@ collector_running=1 export PYTHONPATH="$example_root/agents${PYTHONPATH:+:$PYTHONPATH}" export OPENAI_API_KEY="relay-managed-placeholder" -job_name="phase1-${task_name}-$(date -u +%Y%m%dT%H%M%SZ)" +job_name="${eval_phase}-${task_name}-$(date -u +%Y%m%dT%H%M%SZ)" agent_hosts=(--allow-agent-host host.docker.internal) -if [[ -n "$upstream_host" && "$upstream_host" != "host.docker.internal" ]]; then - agent_hosts+=(--allow-agent-host "$upstream_host") -fi +while IFS= read -r upstream_host; do + if [[ -n "$upstream_host" && "$upstream_host" != "host.docker.internal" ]]; then + agent_hosts+=(--allow-agent-host "$upstream_host") + fi +done < <("$python_bin" -c ' +import json, sys +from urllib.parse import urlsplit +values = json.load(open(sys.argv[1]))["routing"] +print("\n".join(sorted({urlsplit(value).hostname for key, value in values.items() if key.endswith("_base_url")}))) +' "$run_root/runtime/provenance.json") agent_kwargs=() validation_expectations=() if [[ "$inject_post_response_failure" == "true" ]]; then @@ -192,7 +214,7 @@ elif [[ "$inject_post_response_failure" != "false" ]]; then fi ( "$harbor_bin" run \ - --dataset terminal-bench@2.0 \ + "${dataset_args[@]}" \ --include-task-name "$task_name" \ --n-tasks 1 \ --agent harbor_hermes_agent:HarborHermesAgent \ @@ -206,9 +228,9 @@ fi --ak "relay_wheel_sha256=$relay_wheel_sha256" \ --ak "relay_architecture=$relay_architecture" \ "${agent_kwargs[@]}" \ - --ae "$upstream_auth_env=${!upstream_auth_env}" \ - --ae OPENAI_API_KEY=relay-managed-placeholder \ + --ae 'OPENAI_API_KEY=${OPENAI_API_KEY}' \ --ae "OPENAI_BASE_URL=$fail_closed_openai_base_url" \ + --mounts "$mounts_json" \ "${agent_hosts[@]}" \ --artifact /logs/agent/direct-hermes \ --agent-include-logs hermes-session.jsonl \ diff --git a/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh b/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh index b02f98a94..d37bdd521 100755 --- a/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh +++ b/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh @@ -5,7 +5,7 @@ set -euo pipefail switchyard_repository="${SWITCHYARD_REPOSITORY:-https://github.com/bbednarski9/Switchyard.git}" -switchyard_commit="${SWITCHYARD_COMMIT:-8293936a0f5758aa1a782639d485b8b8948cf03e}" +switchyard_commit="${SWITCHYARD_COMMIT:-5d9d3292d6154e44d50295d0d4a3fd4f144f2528}" target_architecture="${SWITCHYARD_TARGET_ARCHITECTURE:-x86_64}" output_dir="${1:-}" diff --git a/examples/harbor-hermes-switchyard/scripts/exec_process_group.py b/examples/harbor-hermes-switchyard/scripts/exec_process_group.py new file mode 100755 index 000000000..cc1ddcc38 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/exec_process_group.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exec a command as the leader of a new POSIX process group.""" + +from __future__ import annotations + +import os +import sys +from errno import EPERM + + +def main() -> int: + if len(sys.argv) < 2: + raise SystemExit("usage: exec_process_group.py COMMAND [ARG ...]") + try: + os.setsid() + except PermissionError as error: + if error.errno != EPERM: + raise + # Some launchers already make the child a process-group leader. In + # that case setsid(2) is forbidden, but the existing isolated group is + # exactly what the supervisor needs to signal as a unit. + if os.getpgrp() != os.getpid(): + raise RuntimeError("could not isolate the coordinator process group") from error + os.execvp(sys.argv[1], sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py b/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py index e095510bf..a42ef3afd 100755 --- a/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py +++ b/examples/harbor-hermes-switchyard/scripts/fake_openai_upstream.py @@ -61,10 +61,7 @@ def do_POST(self) -> None: # noqa: N802 force_strong = "force strong route" in serialized_messages content = json.dumps( { - "recommended_route": "strong" if force_strong else "weak", "p_solve": 0.01 if force_strong else 0.99, - "confidence": 0.99, - "abstain": False, "capability_boundary": "supported", "primary_rule": "SUP-1", "crux": "deterministic offline smoke task", diff --git a/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py index 5f7743a1b..449ca71c8 100755 --- a/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py +++ b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py @@ -206,10 +206,17 @@ def complete(args: argparse.Namespace, root: Path) -> None: receipt_path = root / "direct-hermes-receipt.json" receipt = json.loads(receipt_path.read_text(encoding="utf-8")) messages, exported_session_id = _read_session_messages(HERMES_SESSION) - response = _last_assistant_response(messages) + session_response = _last_assistant_response(messages) diagnostic_text = _write_bounded_diagnostics(root) log_response, log_session_id = _response_from_cli_log(diagnostic_text) - response = response or log_response + # Quiet mode sometimes leaves the exported session empty, so successful + # runs recover their response from stdout. A non-zero agent command may + # instead print an error after ``session_id:``; never promote that text to + # a completed response. The deterministic Phase 1 late-failure injection + # happens after a successful inherited run and intentionally exercises the + # bounded stdout recovery path. + allow_log_response = not args.error_type or args.error_type == "InjectedPostResponseFailure" + response = session_response or (log_response if allow_log_response else None) exported_session_id = exported_session_id or log_session_id lowered = diagnostic_text.lower() cleanup_failure = any( @@ -240,7 +247,18 @@ def complete(args: argparse.Namespace, root: Path) -> None: "ended_at_unix": ended_at, "duration_seconds": (max(0.0, ended_at - args.started_at) if args.started_at is not None else None), }, - "error": ({"type": args.error_type or "RelayCleanupError", "phase": "shutdown"} if late_failure else None), + "error": ( + { + "type": args.error_type or "RelayCleanupError", + "phase": ( + "shutdown" + if cleanup_failure or args.error_type == "InjectedPostResponseFailure" + else "agent" + ), + } + if late_failure + else None + ), } atomic_json(root / "direct-hermes-result.json", result) diff --git a/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh new file mode 100755 index 000000000..072907a6c --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +env_file="${1:-}" +session="${2:-}" +if [[ -z "$env_file" || "$env_file" != /* || -z "$session" ]]; then + echo "usage: $0 /absolute/phase2-run.env tmux-session-name" >&2 + exit 2 +fi +if [[ ! "$session" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "tmux session name may contain only letters, digits, dot, underscore, and dash" >&2 + exit 2 +fi +command -v tmux >/dev/null || { echo "tmux is required" >&2; exit 2; } +"$example_root/scripts/validate_phase2_environment.sh" "$env_file" +if tmux has-session -t "$session" 2>/dev/null; then + echo "tmux session already exists: $session" >&2 + exit 21 +fi + +# The caller must not source the protected file. Only its path is projected +# into the detached session; run_phase2_from_env.sh sources it with xtrace off. +tmux new-session -d -s "$session" \ + -e "PHASE2_ENV_FILE=$env_file" \ + "$example_root/scripts/run_phase2_from_env.sh" +echo "started tmux session: $session" diff --git a/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py b/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py index 05f1fc0ac..04cc4fa8e 100755 --- a/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py +++ b/examples/harbor-hermes-switchyard/scripts/offline_compatibility_smoke.py @@ -10,6 +10,7 @@ import json import os import threading +import tomllib from pathlib import Path from typing import Any @@ -19,7 +20,7 @@ async def exercise(model: str) -> tuple[list[dict[str, Any]], dict[str, Any]]: import nemo_relay - host = RelayRuntime(profile_key="phase1-offline") + host = RelayRuntime(profile_key="phase2-offline") downstream_called = False async def forbidden_downstream(_request: Any) -> dict[str, Any]: @@ -30,8 +31,8 @@ async def forbidden_downstream(_request: Any) -> dict[str, Any]: responses: list[dict[str, Any]] = [] try: cases = ( - ("phase1-offline-weak-session", "reply with the smoke marker"), - ("phase1-offline-strong-session", "force strong route and reply with the smoke marker"), + ("phase2-offline-weak-session", "reply with the smoke marker"), + ("phase2-offline-strong-session", "force strong route and reply with the smoke marker"), ) for session_id, prompt in cases: session = host.ensure_session({"session_id": session_id}) @@ -76,10 +77,14 @@ def main() -> int: parser.add_argument("--artifacts", type=Path, required=True) parser.add_argument("--request-log", type=Path, required=True) parser.add_argument("--model", default="ollama-route-stub") - parser.add_argument("--classifier-model", default="phase1/fake-weak") - parser.add_argument("--expected-routed-model", default="phase1/fake-weak") - parser.add_argument("--expected-strong-model", default="phase1/fake-strong") args = parser.parse_args() + config = tomllib.loads(args.plugins.read_text(encoding="utf-8")) + plugin = config["plugins"]["dynamic"][0]["config"] + algorithm = plugin["algorithm"] + targets = plugin["targets"] + classifier_model = targets[algorithm["classifier_target"]]["model"] + weak_model = targets[algorithm["weak_target"]]["model"] + strong_model = targets[algorithm["strong_target"]]["model"] artifacts = args.artifacts.resolve() artifacts.mkdir(mode=0o700, parents=True, exist_ok=True) os.environ["HERMES_NEMO_RELAY_PLUGINS_TOML"] = str(args.plugins.resolve()) @@ -107,10 +112,10 @@ def main() -> int: raise AssertionError(f"unexpected provider request sequence: {request_kinds}") request_models = [item.get("model") for item in requests] expected_models = [ - args.classifier_model, - args.expected_routed_model, - args.classifier_model, - args.expected_strong_model, + classifier_model, + weak_model, + classifier_model, + strong_model, ] if request_models != expected_models: raise AssertionError(f"unexpected provider model sequence: {request_models}") diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index 9e5f3d6e0..522d4745a 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Prepare one immutable Phase 1 run root and render its Relay config.""" +"""Prepare one immutable evaluation run root and render its Relay config.""" from __future__ import annotations @@ -18,16 +18,14 @@ from urllib.parse import urlsplit from zipfile import ZipFile +import tomli_w + HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" HERMES_REF = "feat/relay-native-plugin-init" HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" SWITCHYARD_REPOSITORY = "https://github.com/bbednarski9/Switchyard.git" -SWITCHYARD_COMMIT = "8293936a0f5758aa1a782639d485b8b8948cf03e" +SWITCHYARD_COMMIT = "5d9d3292d6154e44d50295d0d4a3fd4f144f2528" RELAY_VERSION = "0.7.0" -DEFAULT_STRONG_MODEL = "aws/anthropic/bedrock-claude-opus-4-6" -DEFAULT_WEAK_MODEL = "aws/anthropic/bedrock-claude-sonnet-4-6" -DEFAULT_HERMES_CALLER_MODEL = "ollama-route-stub" -ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]*") SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}") @@ -113,7 +111,63 @@ def verify_native_library(path: Path, architecture: str) -> None: raise ValueError(f"Switchyard library does not target {architecture}: ELF e_machine={machine}") -def render_config(template: Path, output: Path, replacements: dict[str, str]) -> None: +def plugin_settings(config: dict[str, object]) -> dict[str, str]: + plugins = config.get("plugins") + if not isinstance(plugins, dict) or not isinstance(plugins.get("dynamic"), list): + raise ValueError("plugins.toml.in must define one dynamic plugin") + dynamic = plugins["dynamic"] + if len(dynamic) != 1 or not isinstance(dynamic[0], dict): + raise ValueError("plugins.toml.in must define exactly one dynamic plugin") + plugin_config = dynamic[0].get("config") + if not isinstance(plugin_config, dict) or not isinstance(plugin_config.get("targets"), dict): + raise ValueError("Switchyard dynamic plugin targets are missing") + targets = plugin_config["targets"] + if set(targets) != {"strong", "weak"}: + raise ValueError("Switchyard must define strong and weak targets") + settings: dict[str, str] = {} + for name in ("strong", "weak"): + target = targets[name] + if not isinstance(target, dict): + raise ValueError(f"Switchyard target is invalid: {name}") + model = checked_label(str(target.get("model", "")), f"{name}_model") + base_url = checked_url(str(target.get("base_url", "")), f"{name}_base_url") + header_env = target.get("header_env") + if header_env != {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"}: + raise ValueError("plugins.toml.in must reference SWITCHYARD_PROVIDER_AUTHORIZATION") + settings[f"{name}_model"] = model + settings[f"{name}_base_url"] = base_url + if settings["strong_model"] == settings["weak_model"]: + raise ValueError("strong and weak models must be distinct") + components = config.get("components") + if not isinstance(components, list): + raise ValueError("Relay components are missing") + observability = next( + ( + component + for component in components + if isinstance(component, dict) and component.get("kind") == "observability" + ), + None, + ) + if not isinstance(observability, dict): + raise ValueError("Relay observability component is missing") + observation_config = observability.get("config") + if not isinstance(observation_config, dict) or not isinstance(observation_config.get("atif"), dict): + raise ValueError("Relay ATIF configuration is missing") + settings["hermes_caller_model"] = checked_label( + str(observation_config["atif"].get("model_name", "")), "hermes_caller_model" + ) + if settings["hermes_caller_model"] in {settings["strong_model"], settings["weak_model"]}: + raise ValueError("Hermes caller model must be distinct from Switchyard targets") + return settings + + +def render_config( + template: Path, + output: Path, + replacements: dict[str, str], + test_overrides: dict[str, str] | None = None, +) -> dict[str, str]: rendered = template.read_text(encoding="utf-8") for key, value in replacements.items(): if "\n" in value or "\r" in value: @@ -122,10 +176,21 @@ def render_config(template: Path, output: Path, replacements: dict[str, str]) -> unresolved = sorted(set(re.findall(r"@[A-Z0-9_]+@", rendered))) if unresolved: raise ValueError(f"unresolved Relay config placeholders: {unresolved}") - output.write_text(rendered, encoding="utf-8") + config = tomllib.loads(rendered) + if test_overrides: + plugin = config["plugins"]["dynamic"][0]["config"] + old_models = {name: plugin["targets"][name]["model"] for name in ("strong", "weak")} + for name in ("strong", "weak"): + plugin["targets"][name]["model"] = test_overrides[f"{name}_model"] + plugin["targets"][name]["base_url"] = test_overrides["provider_base_url"] + pricing = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] + replacement_models = {old_models[name]: test_overrides[f"{name}_model"] for name in old_models} + for entry in pricing: + entry["model_id"] = replacement_models.get(entry["model_id"], entry["model_id"]) + settings = plugin_settings(config) + output.write_text(tomli_w.dumps(config), encoding="utf-8") os.chmod(output, 0o600) - with output.open("rb") as stream: - tomllib.load(stream) + return settings def main() -> int: @@ -134,11 +199,10 @@ def main() -> int: parser.add_argument("--switchyard-bundle", type=Path, required=True) parser.add_argument("--relay-wheel", type=Path) parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), default="x86_64") - parser.add_argument("--upstream-base-url", required=True) - parser.add_argument("--upstream-auth-env", default="SWITCHYARD_PROVIDER_AUTHORIZATION") - parser.add_argument("--strong-model", default=DEFAULT_STRONG_MODEL) - parser.add_argument("--weak-model", default=DEFAULT_WEAK_MODEL) - parser.add_argument("--hermes-caller-model", default=DEFAULT_HERMES_CALLER_MODEL) + parser.add_argument("--plugin-config-template", type=Path) + parser.add_argument("--test-provider-base-url") + parser.add_argument("--test-strong-model") + parser.add_argument("--test-weak-model") parser.add_argument("--openinference-endpoint", required=True) parser.add_argument("--phoenix-project", required=True) parser.add_argument("--eval-cohort", required=True) @@ -172,33 +236,32 @@ def main() -> int: relay_wheel = download_relay_wheel(runtime / "wheels", args.relay_architecture) verify_relay_wheel(relay_wheel, args.relay_architecture) - upstream_base_url = checked_url(args.upstream_base_url, "upstream_base_url") openinference_endpoint = checked_url(args.openinference_endpoint, "openinference_endpoint") - if not ENV_NAME.fullmatch(args.upstream_auth_env): - raise ValueError("upstream_auth_env must be an uppercase environment variable name") - strong_model = checked_label(args.strong_model, "strong_model") - weak_model = checked_label(args.weak_model, "weak_model") - hermes_caller_model = checked_label(args.hermes_caller_model, "hermes_caller_model") - if strong_model == weak_model: - raise ValueError("strong_model and weak_model must be distinct") phoenix_project = checked_label(args.phoenix_project, "phoenix_project") eval_cohort = checked_label(args.eval_cohort, "eval_cohort") + plugin_template = (args.plugin_config_template or example_root / "config" / "plugins.toml.in").resolve(strict=True) + test_values = (args.test_provider_base_url, args.test_strong_model, args.test_weak_model) + if any(test_values) and not all(test_values): + raise ValueError("all test provider overrides must be supplied together") + test_overrides = None + if all(test_values): + test_overrides = { + "provider_base_url": checked_url(args.test_provider_base_url, "test_provider_base_url"), + "strong_model": checked_label(args.test_strong_model, "test_strong_model"), + "weak_model": checked_label(args.test_weak_model, "test_weak_model"), + } config_path = runtime / "plugins.toml" - render_config( - example_root / "config" / "relay.toml.in", + routing = render_config( + plugin_template, config_path, { - "STRONG_MODEL": strong_model, - "WEAK_MODEL": weak_model, - "HERMES_CALLER_MODEL": hermes_caller_model, "HERMES_COMMIT": HERMES_COMMIT, "OPENINFERENCE_ENDPOINT": openinference_endpoint, "PHOENIX_PROJECT": phoenix_project, "EVAL_COHORT": eval_cohort, - "UPSTREAM_BASE_URL": upstream_base_url, - "UPSTREAM_AUTH_ENV": args.upstream_auth_env, }, + test_overrides, ) manifest = bundle / "relay-plugin.toml" @@ -227,12 +290,11 @@ def main() -> int: "library_sha256": sha256(libraries[0]), }, "relay_config_sha256": sha256(config_path), + "plugin_config_template_sha256": sha256(plugin_template), "routing": { "algorithm": "llm_classifier", "classifier_target": "weak", - "strong_model": strong_model, - "weak_model": weak_model, - "hermes_caller_model": hermes_caller_model, + **routing, }, "phoenix_project": phoenix_project, "eval_cohort": eval_cohort, diff --git a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh index eb86e12e8..4c5a27073 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh +++ b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh @@ -6,8 +6,9 @@ set -euo pipefail example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" run_root="${1:-}" -image="${PHASE1_COMPAT_IMAGE:-python:3.11-bookworm}" -platform="${PHASE1_COMPAT_PLATFORM:-linux/amd64}" +admission_output="${2:-$run_root/artifacts/offline-admission.json}" +image="${OFFLINE_COMPAT_IMAGE:-python:3.11-bookworm}" +platform="${OFFLINE_COMPAT_PLATFORM:-linux/amd64}" hermes_repository="${HERMES_REPOSITORY:-https://github.com/bbednarski9/hermes-agent.git}" hermes_ref="${HERMES_REF:-feat/relay-native-plugin-init}" hermes_commit="${HERMES_COMMIT:-efb63e714abc436af88af9b0d6734751c199aa6d}" @@ -27,7 +28,7 @@ docker info >/dev/null case "$platform" in linux/amd64) expected_architecture=x86_64 ;; linux/arm64) expected_architecture=aarch64 ;; - *) echo "PHASE1_COMPAT_PLATFORM must be linux/amd64 or linux/arm64" >&2; exit 2 ;; + *) echo "OFFLINE_COMPAT_PLATFORM must be linux/amd64 or linux/arm64" >&2; exit 2 ;; esac prepared_architecture="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["nemo_relay"].get("architecture", "x86_64"))' "$run_root/runtime/provenance.json")" [[ "$prepared_architecture" == "$expected_architecture" ]] || { @@ -58,7 +59,7 @@ docker run --rm \ export DEBIAN_FRONTEND=noninteractive export HERMES_HOME=/tmp/hermes export HERMES_NEMO_RELAY_PLUGINS_TOML=/runtime/plugins.toml - export SWITCHYARD_PROVIDER_AUTHORIZATION="Bearer phase1-offline-secret-value" + export SWITCHYARD_PROVIDER_AUTHORIZATION="Bearer phase2-offline-secret-value" apt-get update apt-get install -y --no-install-recommends build-essential ca-certificates curl git ripgrep xz-utils git clone --no-tags --branch "'"$hermes_ref"'" "'"$hermes_repository"'" /tmp/hermes-agent-src @@ -91,7 +92,7 @@ docker run --rm \ --python /tmp/hermes-agent-src/venv/bin/python \ --force-reinstall --no-deps "$relay_wheel" python3 /example/scripts/fake_openai_upstream.py \ - --token phase1-offline-secret-value \ + --token phase2-offline-secret-value \ --request-log /logs/agent/direct-hermes/provider-requests.jsonl & provider_pid=$! python3 /example/scripts/fake_otlp_collector.py \ @@ -116,20 +117,39 @@ docker run --rm \ --artifacts /logs/agent/direct-hermes \ --request-log /logs/agent/direct-hermes/provider-requests.jsonl test -s /logs/agent/direct-hermes/otlp-requests.jsonl - if grep -R -F phase1-offline-secret-value /logs/agent/direct-hermes >/dev/null; then + if grep -R -F phase2-offline-secret-value /logs/agent/direct-hermes >/dev/null; then echo "offline secret leaked into persisted evidence" >&2 exit 1 fi ' -python3 - "$artifacts" <<'PY' +python3 - "$artifacts" "$run_root/runtime/provenance.json" "$admission_output" <<'PY' +import hashlib import json import pathlib import sys root = pathlib.Path(sys.argv[1]) +provenance_path = pathlib.Path(sys.argv[2]) +output = pathlib.Path(sys.argv[3]) result = json.loads((root / "offline-smoke.json").read_text()) if result.get("status") != "passed": raise SystemExit("offline compatibility smoke did not pass") -print(json.dumps(result, indent=2)) +provenance = json.loads(provenance_path.read_text()) +admission = { + "schema_version": "harbor-hermes-switchyard.phase2-offline-admission.v1", + "status": "passed", + "hermes_commit": provenance["hermes"]["commit"], + "relay_architecture": provenance["nemo_relay"]["architecture"], + "relay_wheel_sha256": provenance["nemo_relay"]["wheel_sha256"], + "switchyard_library_sha256": provenance["switchyard"]["library_sha256"], + "plugin_config_template_sha256": provenance["plugin_config_template_sha256"], + "offline_relay_config_sha256": provenance["relay_config_sha256"], + "offline_smoke_sha256": hashlib.sha256((root / "offline-smoke.json").read_bytes()).hexdigest(), + "provider_requests": result["provider_requests"], + "surviving_shutdown_threads": result["surviving_shutdown_threads"], +} +output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) +output.write_text(json.dumps(admission, indent=2, sort_keys=True) + "\n") +print(json.dumps(admission, indent=2)) PY diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py new file mode 100755 index 000000000..e50e56cde --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -0,0 +1,901 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run one resumable, isolated Terminal-Bench cohort for Phase 2.""" + +from __future__ import annotations + +import argparse +import asyncio +import fcntl +import hashlib +import json +import os +import re +import shutil +import subprocess +import time +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-cohort.v1" +PLAN_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-plan.v1" +TASK_STATE_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-task-state.v1" +EXPECTED_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" +INFRASTRUCTURE_PATTERNS = ( + "apt-get update && apt-get install", + "cannot connect to the docker daemon", + "connection refused", + "connection reset", + "connection timed out", + "connecterror", + "context deadline exceeded", + "docker build failed", + "docker is not running", + "failed to resolve source metadata", + "error getting dataset", + "i/o timeout", + "network is unreachable", + "no space left on device", + "phoenix upload", + "registry-1.docker.io", + "temporary failure in name resolution", + "tls handshake timeout", + "too many requests", +) + + +@dataclass(frozen=True) +class Task: + index: int + name: str + memory_gb: int + + @property + def directory_name(self) -> str: + return f"{self.index:03d}-{self.name}" + + def as_json(self) -> dict[str, Any]: + return {"index": self.index, "name": self.name, "memory_gb": self.memory_gb} + + +def read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"expected a JSON object: {path}") + return value + + +def write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def sha256_file_set(root: Path, paths: list[Path]) -> str: + digest = hashlib.sha256() + for path in sorted(paths): + relative = path.relative_to(root).as_posix() + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(sha256_file(path).encode()) + digest.update(b"\n") + return digest.hexdigest() + + +def parse_memory_gb(value: object, task_name: str) -> int: + text = str(value or "2G").strip() + match = re.fullmatch(r"([1-9][0-9]*)G", text, re.IGNORECASE) + if not match: + raise ValueError(f"unsupported memory value for {task_name}: {text}") + return int(match.group(1)) + + +def discover_tasks(dataset_root: Path, sample_count: int, excluded: set[str], canary_task: str) -> list[Task]: + if not dataset_root.is_dir(): + raise ValueError(f"dataset root is not a directory: {dataset_root}") + discovered: list[tuple[str, int]] = [] + for task_root in sorted(path for path in dataset_root.iterdir() if path.is_dir()): + if task_root.name in excluded: + continue + task_toml = task_root / "task.toml" + if not task_toml.is_file(): + continue + with task_toml.open("rb") as stream: + config = tomllib.load(stream) + memory = parse_memory_gb((config.get("environment") or {}).get("memory"), task_root.name) + discovered.append((task_root.name, memory)) + selected = discovered[:sample_count] + if len(selected) != sample_count: + raise ValueError(f"dataset contains {len(selected)} selectable tasks; requested {sample_count}") + canaries = [item for item in selected if item[0] == canary_task] + if len(canaries) != 1: + raise ValueError(f"canary task is not uniquely selectable: {canary_task}") + ordered = canaries + [item for item in selected if item[0] != canary_task] + return [Task(index, name, memory) for index, (name, memory) in enumerate(ordered, 1)] + + +def task_summary_passed(path: Path) -> bool: + if not path.is_file(): + return False + try: + summary = read_json(path) + except (OSError, ValueError, json.JSONDecodeError): + return False + return ( + summary.get("status") == "passed" + and isinstance(summary.get("validation"), dict) + and summary["validation"].get("status") == "passed" + and isinstance(summary.get("phoenix_upload"), dict) + and summary["phoenix_upload"].get("status") == "passed" + ) + + +def successful_attempt(task_root: Path) -> Path | None: + attempts = task_root / "attempts" + if not attempts.is_dir(): + return None + for attempt in sorted((path for path in attempts.iterdir() if path.is_dir()), reverse=True): + if task_summary_passed(attempt / "summary.json"): + return attempt + return None + + +def classify_failure(log_text: str) -> str: + lowered = log_text.lower() + if any(pattern in lowered for pattern in INFRASTRUCTURE_PATTERNS): + return "infrastructure" + if re.search(r"http (429|5[0-9][0-9])\b", lowered): + return "infrastructure" + return "harness_or_integration" + + +def classify_attempt_failure(log_text: str, attempt: Path) -> str: + """Classify a failed attempt using the wrapper and Harbor's nested logs.""" + + if classify_failure(log_text) == "infrastructure": + return "infrastructure" + for path in sorted(attempt.rglob("*.log")): + try: + with path.open(encoding="utf-8", errors="replace") as stream: + while chunk := stream.read(1024 * 1024): + if classify_failure(chunk) == "infrastructure": + return "infrastructure" + except OSError: + continue + return "harness_or_integration" + + +def plugin_contract(path: Path) -> dict[str, Any]: + with path.open("rb") as stream: + config = tomllib.load(stream) + plugins = config.get("plugins", {}).get("dynamic", []) + if len(plugins) != 1: + raise ValueError("plugin configuration must define exactly one dynamic plugin") + targets = plugins[0].get("config", {}).get("targets", {}) + if set(targets) != {"strong", "weak"}: + raise ValueError("plugin configuration must define strong and weak targets") + for target in targets.values(): + if target.get("header_env") != {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"}: + raise ValueError("plugin authorization must reference SWITCHYARD_PROVIDER_AUTHORIZATION") + models = [targets[name].get("model") for name in ("weak", "strong")] + base_urls = sorted({targets[name].get("base_url") for name in ("weak", "strong")}) + if any(not isinstance(value, str) or not value for value in models + base_urls): + raise ValueError("plugin target models and base URLs must be non-empty") + for base_url in base_urls: + parsed = urllib.parse.urlsplit(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password: + raise ValueError("plugin target base URLs must be credential-free HTTP(S) URLs") + components = {component.get("kind"): component for component in config.get("components", [])} + caller = components.get("observability", {}).get("config", {}).get("atif", {}).get("model_name") + if not isinstance(caller, str) or caller in models: + raise ValueError("Hermes caller model must be distinct from plugin target models") + return { + "required_models": sorted(models), + "strong_model": targets["strong"]["model"], + "weak_model": targets["weak"]["model"], + "hermes_caller_model": caller, + "provider_base_urls": base_urls, + "sha256": sha256_file(path), + } + + +def switchyard_library(bundle: Path) -> Path: + libraries = sorted(bundle.glob("libswitchyard_nemo_relay_plugin.*")) + if len(libraries) != 1: + raise ValueError("Switchyard bundle must contain exactly one native library") + return libraries[0] + + +def validate_smoke_evidence( + path: Path, + expected_count: int, + dataset_root: Path, + concurrency: int, + relay_architecture: str, + relay_wheel: Path, + switchyard_bundle: Path, + plugin_config_template: Path, +) -> None: + evidence = read_json(path) + dataset_root = dataset_root.resolve() + task_tomls = sorted(dataset_root.glob("*/task.toml")) + expected_names = [task_toml.parent.name for task_toml in task_tomls] + observed_tasks = evidence.get("tasks") + observed_names = ( + [task.get("name") for task in observed_tasks] + if isinstance(observed_tasks, list) and all(isinstance(task, dict) for task in observed_tasks) + else [] + ) + records_valid = len(observed_names) == expected_count + if records_valid: + for record in observed_tasks: + name = record.get("name") + try: + instruction = (dataset_root / record["instruction_path"]).resolve(strict=True) + verifier = (dataset_root / record["verifier_path"]).resolve(strict=True) + instruction.relative_to(dataset_root) + verifier.relative_to(dataset_root) + except (KeyError, OSError, RuntimeError, TypeError, ValueError): + records_valid = False + break + task_toml = dataset_root / str(name) / "task.toml" + if ( + instruction.parent != task_toml.parent + or verifier.parent.parent != task_toml.parent + or record.get("task_toml_sha256") != sha256_file(task_toml) + or record.get("instruction_sha256") != sha256_file(instruction) + or record.get("test_sha256") != sha256_file(verifier) + ): + records_valid = False + break + if ( + evidence.get("schema_version") != "harbor-hermes-switchyard.phase2-smoke.v1" + or evidence.get("status") != "passed" + or evidence.get("task_count") != expected_count + or len(task_tomls) != expected_count + or observed_names != expected_names + or not records_valid + or evidence.get("dataset_task_definitions_sha256") != sha256_file_set(dataset_root, task_tomls) + or evidence.get("registry_network_attempts") != 0 + or evidence.get("concurrency") != concurrency + or evidence.get("relay_architecture") != relay_architecture + or not isinstance(evidence.get("relay_runtime"), dict) + or evidence["relay_runtime"].get("status") != "passed" + or evidence["relay_runtime"].get("relay_wheel_sha256") != sha256_file(relay_wheel) + or evidence["relay_runtime"].get("switchyard_library_sha256") + != sha256_file(switchyard_library(switchyard_bundle)) + or evidence["relay_runtime"].get("plugin_config_template_sha256") != sha256_file(plugin_config_template) + ): + raise ValueError(f"Phase 2 all-task smoke evidence is not passed: {path}") + + +def validate_offline_evidence( + path: Path, + relay_architecture: str, + relay_wheel: Path, + switchyard_bundle: Path, + plugin_config_template: Path, +) -> None: + evidence = read_json(path) + if ( + evidence.get("schema_version") != "harbor-hermes-switchyard.phase2-offline-admission.v1" + or evidence.get("status") != "passed" + or evidence.get("hermes_commit") != EXPECTED_HERMES_COMMIT + or evidence.get("relay_architecture") != relay_architecture + or evidence.get("relay_wheel_sha256") != sha256_file(relay_wheel) + or evidence.get("switchyard_library_sha256") != sha256_file(switchyard_library(switchyard_bundle)) + or evidence.get("plugin_config_template_sha256") != sha256_file(plugin_config_template) + or evidence.get("provider_requests", 0) <= 0 + or evidence.get("surviving_shutdown_threads") != [] + ): + raise ValueError(f"Phase 2 offline admission evidence is not passed: {path}") + + +def probe_url(url: str, label: str, attempts: int = 3) -> None: + for attempt in range(1, attempts + 1): + request = urllib.request.Request(url, method="GET") + try: + with urllib.request.urlopen(request, timeout=10) as response: + if response.status >= 500: + raise RuntimeError(f"{label} returned HTTP {response.status}") + return + except urllib.error.HTTPError as error: + if error.code < 500: + return + failure: Exception = error + except (OSError, urllib.error.URLError) as error: + failure = error + if attempt < attempts: + time.sleep(2 ** (attempt - 1)) + raise RuntimeError(f"{label} is unreachable after {attempts} attempts: {failure}") from failure + + +def verify_provider_catalog( + base_url: str, authorization: str, required_models: list[str], attempts: int = 3 +) -> list[str]: + catalog_url = f"{base_url.rstrip('/')}/models" + for attempt in range(1, attempts + 1): + request = urllib.request.Request(catalog_url, headers={"Authorization": authorization}, method="GET") + try: + with urllib.request.urlopen(request, timeout=30) as response: + if response.status != 200: + raise RuntimeError(f"provider model catalog returned HTTP {response.status}") + payload = json.load(response) + break + except urllib.error.HTTPError as error: + if error.code < 500 and error.code != 429: + raise RuntimeError(f"provider model catalog returned HTTP {error.code}") from error + failure: Exception = error + except (OSError, ValueError, urllib.error.URLError) as error: + failure = error + if attempt < attempts: + time.sleep(2 ** (attempt - 1)) + else: + raise RuntimeError( + f"provider model catalog is unavailable or invalid after {attempts} attempts: {failure}" + ) from failure + records = payload.get("data") if isinstance(payload, dict) else None + available = { + record.get("id") for record in records or [] if isinstance(record, dict) and isinstance(record.get("id"), str) + } + missing = sorted(set(required_models) - available) + if missing: + raise RuntimeError(f"provider model catalog is missing configured model(s): {', '.join(missing)}") + return sorted(required_models) + + +def normalize_architecture(value: str) -> str: + normalized = value.strip().lower() + aliases = {"amd64": "x86_64", "x86_64": "x86_64", "arm64": "aarch64", "aarch64": "aarch64"} + if normalized not in aliases: + raise RuntimeError(f"unsupported Docker architecture: {value}") + return aliases[normalized] + + +def capacity_requirement_gb(args: argparse.Namespace, tasks: list[Task]) -> int: + parallel = args.concurrency * args.parallel_max_memory_gb + largest = max(task.memory_gb for task in tasks) + return max(parallel, largest) + args.docker_memory_reserve_gb + + +def shared_preflight(args: argparse.Namespace, tasks: list[Task]) -> dict[str, Any]: + free_bytes = shutil.disk_usage(args.run_root.parent).free + if free_bytes < args.minimum_free_gb * 1024**3: + raise RuntimeError(f"fewer than {args.minimum_free_gb} GiB remain on the run volume") + docker = json.loads( + subprocess.run( + ["docker", "info", "--format", "{{json .}}"], + check=True, + capture_output=True, + text=True, + ).stdout + ) + docker_cpus = int(docker["NCPU"]) + docker_memory_bytes = int(docker["MemTotal"]) + docker_memory_gb = docker_memory_bytes // 1024**3 + docker_architecture = normalize_architecture(str(docker["Architecture"])) + required_memory_gb = capacity_requirement_gb(args, tasks) + if docker_architecture != args.relay_architecture: + raise RuntimeError( + f"Docker architecture {docker_architecture} does not match Relay architecture {args.relay_architecture}" + ) + if args.concurrency > docker_cpus: + raise RuntimeError(f"concurrency {args.concurrency} exceeds Docker CPU count {docker_cpus}") + if required_memory_gb > docker_memory_gb: + raise RuntimeError(f"Phase 2 requires {required_memory_gb} GiB but Docker exposes {docker_memory_gb} GiB") + version = subprocess.run( + [str(args.harbor_bin), "--version"], check=True, capture_output=True, text=True + ).stdout.strip() + if version != "0.18.0": + raise RuntimeError(f"Harbor 0.18.0 is required; {args.harbor_bin} reports {version}") + python_version = subprocess.run( + [ + str(args.python_bin), + "-c", + 'import importlib.metadata; print(importlib.metadata.version("harbor"))', + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if python_version != "0.18.0": + raise RuntimeError(f"Harbor Python 0.18.0 is required; {args.python_bin} provides {python_version}") + probe_url(args.phoenix_url, "Phoenix") + provider_authorization = os.environ.get("SWITCHYARD_PROVIDER_AUTHORIZATION") + if not provider_authorization: + raise RuntimeError("SWITCHYARD_PROVIDER_AUTHORIZATION is unset") + verified_models: set[str] = set() + for provider_url in args.plugin_contract["provider_base_urls"]: + verified_models.update( + verify_provider_catalog( + provider_url, + provider_authorization, + args.plugin_contract["required_models"], + ) + ) + if not args.switchyard_bundle.is_dir(): + raise RuntimeError(f"Switchyard bundle is missing: {args.switchyard_bundle}") + if not args.relay_wheel.is_file(): + raise RuntimeError(f"Relay wheel is missing: {args.relay_wheel}") + return { + "harbor_version": version, + "minimum_free_gb": args.minimum_free_gb, + "disk_free_gb": free_bytes // 1024**3, + "concurrency": args.concurrency, + "parallel_task_memory_gb": args.parallel_max_memory_gb, + "largest_task_memory_gb": max(task.memory_gb for task in tasks), + "docker_memory_reserve_gb": args.docker_memory_reserve_gb, + "required_docker_memory_gb": required_memory_gb, + "docker_memory_gb": docker_memory_gb, + "docker_cpus": docker_cpus, + "docker_architecture": docker_architecture, + "relay_architecture": args.relay_architecture, + "provider_endpoints": args.plugin_contract["provider_base_urls"], + "provider_catalog_models_verified": sorted(verified_models), + "phoenix_endpoint": args.phoenix_url, + "phoenix_reachable": True, + "provider_reachable": True, + "docker_healthy": True, + } + + +def make_plan(args: argparse.Namespace, tasks: list[Task]) -> dict[str, Any]: + manifest = args.switchyard_bundle / "relay-plugin.toml" + library_candidates = sorted(args.switchyard_bundle.glob("libswitchyard_nemo_relay_plugin.*")) + if not manifest.is_file() or len(library_candidates) != 1: + raise ValueError("Switchyard bundle must contain one manifest and one native library") + example_root = args.task_runner.parent + runtime_sources = [ + args.task_runner, + example_root / "run_phase2_cohort.sh", + example_root / "supervise_phase2_cohort.sh", + ] + for relative in ("agents", "config", "scripts"): + runtime_sources.extend( + path + for path in (example_root / relative).rglob("*") + if path.is_file() and path.suffix in {".py", ".sh", ".toml", ".yaml"} + ) + task_definitions = [args.dataset_root / task.name / "task.toml" for task in tasks] + return { + "schema_version": PLAN_SCHEMA_VERSION, + "dataset": args.dataset, + "dataset_root": str(args.dataset_root.resolve()), + "sample_count": args.sample_count, + "canary_task": args.canary_task, + "concurrency": args.concurrency, + "parallel_max_memory_gb": args.parallel_max_memory_gb, + "docker_memory_reserve_gb": args.docker_memory_reserve_gb, + "minimum_free_gb": args.minimum_free_gb, + "phoenix_project": args.phoenix_project, + "evaluation_cohort": args.eval_cohort, + "timeout_multipliers": {"agent": 3, "agent_setup": 6, "environment_build": 6}, + "required_models": args.plugin_contract["required_models"], + "require_cache_hit": args.require_cache_hit, + "routing": { + "strong_model": args.plugin_contract["strong_model"], + "weak_model": args.plugin_contract["weak_model"], + "hermes_caller_model": args.plugin_contract["hermes_caller_model"], + "provider_base_urls": args.plugin_contract["provider_base_urls"], + }, + "inputs": { + "runner_sha256": sha256_file(args.task_runner), + "runtime_sources_sha256": sha256_file_set(example_root, runtime_sources), + "dataset_task_definitions_sha256": sha256_file_set(args.dataset_root, task_definitions), + "phase2_smoke_evidence_sha256": sha256_file(args.smoke_evidence), + "phase2_offline_evidence_sha256": sha256_file(args.offline_evidence), + "plugin_config_template_sha256": sha256_file(args.plugin_config_template), + "relay_wheel_sha256": sha256_file(args.relay_wheel), + "switchyard_manifest_sha256": sha256_file(manifest), + "switchyard_library_sha256": sha256_file(library_candidates[0]), + }, + "tasks": [task.as_json() for task in tasks], + } + + +def load_or_create_plan(path: Path, plan: dict[str, Any]) -> None: + if path.is_file(): + existing = read_json(path) + if existing != plan: + raise ValueError(f"existing Phase 2 plan does not match requested configuration: {path}") + return + write_json(path, plan) + + +def task_record(task: Task, attempt: Path | None) -> dict[str, Any]: + record: dict[str, Any] = task.as_json() + record["status"] = "pending" + record["attempt_count"] = 0 + task_root = attempt.parents[1] if attempt else None + if task_root: + record["attempt_count"] = len([path for path in (task_root / "attempts").iterdir() if path.is_dir()]) + if not attempt: + return record + summary = read_json(attempt / "summary.json") + validation = summary["validation"] + upload = summary["phoenix_upload"] + record.update( + { + "status": "passed", + "successful_attempt": attempt.name, + "attempt_root": str(attempt), + "benchmark_task_passed": validation.get("benchmark_task_passed"), + "direct_result_status": validation.get("direct_result_status"), + "switchyard_decision_count": validation.get("switchyard_decision_count", 0), + "routed_models": validation.get("routed_models", []), + "routed_targets": validation.get("routed_targets", []), + "cache_read_tokens": validation.get("cache_read_tokens", 0), + "cache_write_tokens": validation.get("cache_write_tokens", 0), + "secret_findings": validation.get("secret_findings", []), + "uploaded_spans": upload.get("uploaded_spans", 0), + "phoenix_upload": upload.get("status"), + } + ) + return record + + +def aggregate_summary(args: argparse.Namespace, tasks: list[Task]) -> dict[str, Any]: + records = [] + for task in tasks: + task_root = args.run_root / "tasks" / task.directory_name + records.append(task_record(task, successful_attempt(task_root))) + complete = all(record["status"] == "passed" for record in records) + cache_read_tokens = sum(int(record.get("cache_read_tokens") or 0) for record in records) + observed_models = sorted( + {model for record in records for model in record.get("routed_models", []) if isinstance(model, str)} + ) + missing_models = sorted(set(args.required_model).difference(observed_models)) + secrets_clean = all(not record.get("secret_findings") for record in records if record["status"] == "passed") + gates = { + "task_outputs": {"passed": complete, "completed": sum(r["status"] == "passed" for r in records)}, + "cache_hit": { + "required": args.require_cache_hit, + "passed": not args.require_cache_hit or cache_read_tokens > 0, + "cache_read_tokens": cache_read_tokens, + }, + "route_diversity": { + "required_models": sorted(args.required_model), + "observed_models": observed_models, + "missing_models": missing_models, + "passed": not missing_models, + }, + "secret_scan": {"passed": secrets_clean}, + } + passed = complete and all(gate["passed"] for gate in gates.values()) + return { + "schema_version": SCHEMA_VERSION, + "status": "passed" if passed else "partial", + "dataset": args.dataset, + "phoenix_project": args.phoenix_project, + "evaluation_cohort": args.eval_cohort, + "planned_tasks": len(records), + "completed_tasks": sum(record["status"] == "passed" for record in records), + "benchmark_pass_count": sum(record.get("benchmark_task_passed") is True for record in records), + "benchmark_nonpass_count": sum(record.get("benchmark_task_passed") is False for record in records), + "uploaded_spans": sum(int(record.get("uploaded_spans") or 0) for record in records), + "cohort_gates": gates, + "tasks": records, + } + + +def write_report(root: Path, summary: dict[str, Any]) -> None: + gates = summary["cohort_gates"] + lines = [ + "# Harbor + Hermes + Switchyard Phase 2 cohort", + "", + f"- Status: `{summary['status']}`", + f"- Completed: {summary['completed_tasks']}/{summary['planned_tasks']}", + f"- Benchmark pass/non-pass: {summary['benchmark_pass_count']}/{summary['benchmark_nonpass_count']}", + f"- Uploaded spans: {summary['uploaded_spans']}", + f"- Cache-read tokens: {gates['cache_hit']['cache_read_tokens']}", + f"- Observed provider models: {', '.join(gates['route_diversity']['observed_models']) or 'none'}", + "", + "| # | Task | Memory | Evidence | Benchmark | Attempts | Spans |", + "|---:|---|---:|---|---|---:|---:|", + ] + for task in summary["tasks"]: + benchmark = task.get("benchmark_task_passed") + benchmark_text = "pass" if benchmark is True else "non-pass" if benchmark is False else "pending" + lines.append( + f"| {task['index']:03d} | `{task['name']}` | {task['memory_gb']}G | {task['status']} | " + f"{benchmark_text} | {task['attempt_count']} | {task.get('uploaded_spans', 0)} |" + ) + (root / "report.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + + +class CohortRunner: + def __init__(self, args: argparse.Namespace, tasks: list[Task]): + self.args = args + self.tasks = tasks + self.stop_scheduling = asyncio.Event() + self.summary_lock = asyncio.Lock() + + async def refresh_summary(self) -> None: + async with self.summary_lock: + summary = aggregate_summary(self.args, self.tasks) + write_json(self.args.run_root / "summary.json", summary) + write_report(self.args.run_root, summary) + + async def run_attempt(self, task: Task, attempt_number: int) -> tuple[int, Path, str]: + task_root = self.args.run_root / "tasks" / task.directory_name + attempts_root = task_root / "attempts" + attempts_root.mkdir(mode=0o700, parents=True, exist_ok=True) + attempt = attempts_root / f"{attempt_number:03d}" + log_path = task_root / f"attempt-{attempt_number:03d}.log" + if attempt.exists(): + raise RuntimeError(f"attempt root already exists: {attempt}") + env = os.environ.copy() + env.update( + { + "TASK_NAME": task.name, + "EVAL_PHASE": "phase2", + "TBENCH_DATASET_PATH": str(self.args.dataset_root), + "PHOENIX_PROJECT": self.args.phoenix_project, + "EVAL_COHORT": self.args.eval_cohort, + "PHOENIX_BASE_URL": self.args.phoenix_url, + "SWITCHYARD_BUNDLE": str(self.args.switchyard_bundle), + "RELAY_WHEEL": str(self.args.relay_wheel), + "RELAY_ARCHITECTURE": self.args.relay_architecture, + "PLUGIN_CONFIG_TEMPLATE": str(self.args.plugin_config_template), + "HARBOR_BIN": str(self.args.harbor_bin), + "EVAL_PYTHON": str(self.args.python_bin), + "AGENT_TIMEOUT_MULTIPLIER": "3", + "AGENT_SETUP_TIMEOUT_MULTIPLIER": "6", + "ENVIRONMENT_BUILD_TIMEOUT_MULTIPLIER": "6", + } + ) + with log_path.open("wb") as log: + process = await asyncio.create_subprocess_exec( + str(self.args.task_runner), str(attempt), env=env, stdout=log, stderr=asyncio.subprocess.STDOUT + ) + status = await process.wait() + log_text = log_path.read_text(encoding="utf-8", errors="replace") + return status, attempt, log_text + + async def run_task(self, task: Task) -> bool: + task_root = self.args.run_root / "tasks" / task.directory_name + passed = successful_attempt(task_root) + if passed: + print(f"[phase2] already complete: {task.directory_name}", flush=True) + return True + state_path = task_root / "task-state.json" + if state_path.is_file(): + state = read_json(state_path) + if state.get("status") == "failed" and state.get("failure_class") == "harness_or_integration": + print(f"[phase2] preserved integration blocker: {task.directory_name}", flush=True) + self.stop_scheduling.set() + return False + attempts_root = task_root / "attempts" + existing_attempts = len(list(attempts_root.glob("[0-9][0-9][0-9]"))) + attempt_number = existing_attempts + 1 + infrastructure_failures = 0 + retry_preflight_failures = 0 + needs_retry_preflight = False + while infrastructure_failures < self.args.max_infra_attempts: + if self.stop_scheduling.is_set(): + return False + if needs_retry_preflight: + try: + preflight = shared_preflight(self.args, self.tasks) + except (OSError, RuntimeError, subprocess.SubprocessError, ValueError) as error: + retry_preflight_failures += 1 + write_json( + state_path, + { + "schema_version": TASK_STATE_SCHEMA_VERSION, + "status": "waiting_for_infrastructure", + "task": task.as_json(), + "failure_class": "infrastructure", + "preflight_error": str(error), + "infrastructure_failure_count": infrastructure_failures, + "retry_preflight_failure_count": retry_preflight_failures, + }, + ) + print( + f"[phase2] retry preflight blocked {task.directory_name} " + f"({retry_preflight_failures}/{self.args.max_infra_attempts}): {error}", + flush=True, + ) + await self.refresh_summary() + if retry_preflight_failures >= self.args.max_infra_attempts: + self.stop_scheduling.set() + return False + await asyncio.sleep(self.args.backoff_seconds * (2 ** (retry_preflight_failures - 1))) + continue + write_json(self.args.run_root / "preflight.json", {"status": "passed", **preflight}) + needs_retry_preflight = False + retry_preflight_failures = 0 + print(f"[phase2] starting {task.directory_name} attempt {attempt_number:03d}", flush=True) + status, attempt, log_text = await self.run_attempt(task, attempt_number) + if status == 0 and task_summary_passed(attempt / "summary.json"): + write_json( + state_path, + { + "schema_version": TASK_STATE_SCHEMA_VERSION, + "status": "passed", + "task": task.as_json(), + "successful_attempt": attempt.name, + }, + ) + await self.refresh_summary() + print(f"[phase2] completed {task.directory_name} attempt {attempt_number:03d}", flush=True) + return True + failure_class = classify_attempt_failure(log_text, attempt) + write_json( + state_path, + { + "schema_version": TASK_STATE_SCHEMA_VERSION, + "status": "failed", + "task": task.as_json(), + "latest_attempt": attempt.name, + "failure_class": failure_class, + "exit_code": status, + }, + ) + await self.refresh_summary() + print( + f"[phase2] failed {task.directory_name} attempt {attempt_number:03d}: {failure_class}", + flush=True, + ) + if failure_class != "infrastructure": + self.stop_scheduling.set() + return False + infrastructure_failures += 1 + attempt_number += 1 + needs_retry_preflight = True + if infrastructure_failures < self.args.max_infra_attempts: + await asyncio.sleep(self.args.backoff_seconds * (2 ** (infrastructure_failures - 1))) + self.stop_scheduling.set() + return False + + async def run_parallel_lane(self, tasks: list[Task]) -> bool: + semaphore = asyncio.Semaphore(self.args.concurrency) + + async def guarded(task: Task) -> bool: + async with semaphore: + if self.stop_scheduling.is_set(): + return False + return await self.run_task(task) + + results = await asyncio.gather(*(guarded(task) for task in tasks)) + return all(results) + + async def run(self) -> bool: + preflight = shared_preflight(self.args, self.tasks) + write_json(self.args.run_root / "preflight.json", {"status": "passed", **preflight}) + await self.refresh_summary() + first = self.tasks[0] + if not await self.run_task(first): + return False + parallel = [task for task in self.tasks[1:] if task.memory_gb <= self.args.parallel_max_memory_gb] + serial = [task for task in self.tasks[1:] if task.memory_gb > self.args.parallel_max_memory_gb] + if not await self.run_parallel_lane(parallel): + return False + for task in serial: + if not await self.run_task(task): + return False + await self.refresh_summary() + return aggregate_summary(self.args, self.tasks)["status"] == "passed" + + +def parse_args() -> argparse.Namespace: + example_root = Path(__file__).resolve().parents[1] + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--dataset", default="terminal-bench@2.0") + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--sample-count", type=int, default=89) + parser.add_argument("--exclude-task", action="append", default=[]) + parser.add_argument("--canary-task", default="adaptive-rejection-sampler") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--parallel-max-memory-gb", type=int, default=2) + parser.add_argument("--docker-memory-reserve-gb", type=int, default=4) + parser.add_argument("--max-infra-attempts", type=int, default=3) + parser.add_argument("--backoff-seconds", type=float, default=30) + parser.add_argument("--minimum-free-gb", type=int, default=100) + parser.add_argument("--smoke-evidence", type=Path, required=True) + parser.add_argument("--offline-evidence", type=Path, required=True) + parser.add_argument("--plugin-config-template", type=Path, required=True) + parser.add_argument("--task-runner", type=Path, default=example_root / "run_terminal_bench.sh") + parser.add_argument("--harbor-bin", type=Path, default=example_root / ".venv" / "bin" / "harbor") + parser.add_argument("--python-bin", type=Path, default=example_root / ".venv" / "bin" / "python") + parser.add_argument("--phoenix-url", required=True) + parser.add_argument("--phoenix-project", required=True) + parser.add_argument("--eval-cohort", required=True) + parser.add_argument("--switchyard-bundle", type=Path, required=True) + parser.add_argument("--relay-wheel", type=Path, required=True) + parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), default="x86_64") + parser.add_argument("--require-cache-hit", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--plan-only", action="store_true") + parser.add_argument("--preflight-only", action="store_true") + args = parser.parse_args() + for name in ( + "sample_count", + "concurrency", + "parallel_max_memory_gb", + "docker_memory_reserve_gb", + "max_infra_attempts", + "minimum_free_gb", + ): + if getattr(args, name) <= 0: + parser.error(f"--{name.replace('_', '-')} must be positive") + if not args.run_root.is_absolute(): + parser.error("--run-root must be absolute") + if args.plan_only and args.preflight_only: + parser.error("--plan-only and --preflight-only are mutually exclusive") + return args + + +def main() -> int: + args = parse_args() + args.plugin_contract = plugin_contract(args.plugin_config_template) + args.required_model = args.plugin_contract["required_models"] + validate_smoke_evidence( + args.smoke_evidence, + args.sample_count, + args.dataset_root, + args.concurrency, + args.relay_architecture, + args.relay_wheel, + args.switchyard_bundle, + args.plugin_config_template, + ) + validate_offline_evidence( + args.offline_evidence, + args.relay_architecture, + args.relay_wheel, + args.switchyard_bundle, + args.plugin_config_template, + ) + tasks = discover_tasks(args.dataset_root, args.sample_count, set(args.exclude_task), args.canary_task) + args.run_root.mkdir(mode=0o700, parents=True, exist_ok=True) + lock_path = args.run_root / ".phase2.lock" + with lock_path.open("a+") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise SystemExit(f"another Phase 2 supervisor owns {args.run_root}") from None + plan = make_plan(args, tasks) + load_or_create_plan(args.run_root / "plan.json", plan) + if args.plan_only: + summary = aggregate_summary(args, tasks) + write_json(args.run_root / "summary.json", summary) + write_report(args.run_root, summary) + print(json.dumps(plan, indent=2)) + return 0 + if args.preflight_only: + preflight = shared_preflight(args, tasks) + write_json(args.run_root / "preflight.json", {"status": "passed", **preflight}) + summary = aggregate_summary(args, tasks) + write_json(args.run_root / "summary.json", summary) + write_report(args.run_root, summary) + print(json.dumps({"status": "passed", "preflight": preflight}, indent=2)) + return 0 + passed = asyncio.run(CohortRunner(args, tasks).run()) + summary = aggregate_summary(args, tasks) + print(json.dumps(summary, indent=2)) + if passed: + return 0 + states = [read_json(path) for path in (args.run_root / "tasks").glob("*/task-state.json") if path.is_file()] + if any(state.get("failure_class") == "harness_or_integration" for state in states): + return 20 + return 75 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh new file mode 100755 index 000000000..39ccb5407 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +set +x + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +env_file="${PHASE2_ENV_FILE:-${1:-}}" +if [[ -z "$env_file" || "$env_file" != /* ]]; then + echo "PHASE2_ENV_FILE must be an absolute path" >&2 + exit 2 +fi + +"$example_root/scripts/validate_phase2_environment.sh" "$env_file" +set -a +# shellcheck disable=SC1090 +source "$env_file" +set +a +set +x + +mkdir -p "$PHASE2_RUN_ROOT" +chmod 0700 "$PHASE2_RUN_ROOT" +exec "$example_root/supervise_phase2_cohort.sh" "$PHASE2_RUN_ROOT" \ + >>"$PHASE2_RUN_ROOT/supervisor.log" 2>&1 diff --git a/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py new file mode 100755 index 000000000..6d9252e1e --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise all Phase 2 dataset and runtime wiring without provider work.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import importlib.metadata +import json +import socket +import subprocess +import sys +import tempfile +import tomllib +from pathlib import Path +from typing import Any +from unittest.mock import patch + +from harbor.job import Job +from harbor.models.job.config import DatasetConfig, JobConfig +from harbor.models.task.task import Task as HarborTask +from harbor.models.trial.config import AgentConfig, EnvironmentConfig + +SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-smoke.v1" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def combined_digest(root: Path, paths: list[Path]) -> str: + digest = hashlib.sha256() + for path in sorted(paths): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + digest.update(sha256_file(path).encode()) + digest.update(b"\n") + return digest.hexdigest() + + +def parse_memory_gb(task: HarborTask) -> int: + memory_mb = task.config.environment.memory_mb or 2048 + if memory_mb % 1024: + raise ValueError(f"unsupported memory for {task.name}: {memory_mb} MB") + return memory_mb // 1024 + + +async def validate_local_dataset( + dataset_root: Path, + expected_count: int, + jobs_dir: Path, + concurrency: int, + relay_architecture: str, + authorization_file: Path, +) -> tuple[list[dict[str, Any]], int]: + network_attempts = 0 + + def deny_network(*_args: Any, **_kwargs: Any) -> None: + nonlocal network_attempts + network_attempts += 1 + raise AssertionError("local dataset smoke attempted network access") + + dataset = DatasetConfig(path=dataset_root) + with ( + patch("socket.create_connection", side_effect=deny_network), + patch.object(socket.socket, "connect", side_effect=deny_network), + patch( + "harbor.registry.client.factory.RegistryClientFactory.create", + side_effect=deny_network, + ), + ): + configs = await dataset.get_task_configs() + if len(configs) != expected_count: + raise ValueError(f"Harbor resolved {len(configs)} tasks; expected {expected_count}") + task_records: list[dict[str, Any]] = [] + expected_names = sorted(path.parent.name for path in dataset_root.glob("*/task.toml")) + resolved_names = sorted(config.get_local_path().name for config in configs) + if resolved_names != expected_names: + raise ValueError("Harbor local dataset resolution did not preserve the exported task set") + for name in resolved_names: + selected = await DatasetConfig( + path=dataset_root, + task_names=[name], + n_tasks=1, + ).get_task_configs() + if len(selected) != 1 or selected[0].get_local_path().name != name: + raise ValueError(f"Harbor did not uniquely select local task {name}") + task = HarborTask(selected[0].get_local_path()) + if not task.instruction.strip(): + raise ValueError(f"Harbor loaded an empty instruction for {name}") + test_path = task.paths.discovered_test_path + if not test_path.is_file(): + raise ValueError(f"Harbor did not discover a verifier test for {name}") + task_records.append( + { + "name": name, + "memory_gb": parse_memory_gb(task), + "instruction_path": task.paths.instruction_path.relative_to(dataset_root).as_posix(), + "verifier_path": test_path.relative_to(dataset_root).as_posix(), + "task_toml_sha256": sha256_file(task.paths.config_path), + "instruction_sha256": sha256_file(task.paths.instruction_path), + "test_sha256": sha256_file(test_path), + "has_steps": task.has_steps, + } + ) + + job = await Job.create( + JobConfig( + job_name="phase2-all-task-smoke", + jobs_dir=jobs_dir, + n_attempts=1, + n_concurrent_trials=concurrency, + agent_timeout_multiplier=3, + agent_setup_timeout_multiplier=6, + environment_build_timeout_multiplier=6, + agents=[ + AgentConfig( + import_path="harbor_hermes_agent:HarborHermesAgent", + model_name="openai/ollama-route-stub", + kwargs={ + "repository_url": "https://github.com/bbednarski9/hermes-agent.git", + "repository_ref": "feat/relay-native-plugin-init", + "commit": "efb63e714abc436af88af9b0d6734751c199aa6d", + "relay_config_path": "/smoke/runtime/plugins.toml", + "switchyard_bundle_dir": "/smoke/runtime/switchyard-plugin", + "relay_wheel_path": "/smoke/runtime/nemo-relay.whl", + "relay_architecture": relay_architecture, + }, + env={ + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "http://127.0.0.1:9/v1", + }, + ) + ], + environment=EnvironmentConfig( + mounts=[ + { + "type": "bind", + "source": str(authorization_file), + "target": "/run/secrets/switchyard-provider-authorization", + "read_only": True, + "bind": {"create_host_path": False}, + } + ] + ), + datasets=[dataset], + artifacts=["/logs/agent/direct-hermes"], + ) + ) + if len(job) != expected_count: + raise ValueError(f"Harbor constructed {len(job)} trials; expected {expected_count}") + return task_records, network_attempts + + +def validate_cli_local_projection( + harbor_bin: Path, dataset_root: Path, task_name: str, authorization_file: Path +) -> None: + mounts = json.dumps( + [ + { + "type": "bind", + "source": str(authorization_file), + "target": "/run/secrets/switchyard-provider-authorization", + "read_only": True, + "bind": {"create_host_path": False}, + } + ], + separators=(",", ":"), + ) + result = subprocess.run( + [ + str(harbor_bin), + "run", + "--path", + str(dataset_root), + "--include-task-name", + task_name, + "--n-tasks", + "1", + "--agent", + "harbor_hermes_agent:HarborHermesAgent", + "--model", + "openai/ollama-route-stub", + "--mounts", + mounts, + "--print-config", + ], + check=True, + capture_output=True, + text=True, + ) + config = json.loads(result.stdout) + datasets = config.get("datasets") + if not isinstance(datasets, list) or len(datasets) != 1: + raise ValueError("Harbor CLI did not render one local dataset") + dataset = datasets[0] + if ( + Path(dataset.get("path", "")).resolve() != dataset_root + or dataset.get("task_names") != [task_name] + or dataset.get("n_tasks") != 1 + or dataset.get("name") is not None + ): + raise ValueError("Harbor CLI local dataset projection is incorrect") + if config.get("environment", {}).get("mounts") != json.loads(mounts): + raise ValueError("Harbor CLI did not preserve the protected authorization mount") + + +def validate_relay_runtime( + example_root: Path, + temporary_root: Path, + switchyard_bundle: Path, + relay_wheel: Path, + relay_architecture: str, + plugin_config_template: Path, +) -> dict[str, Any]: + run_root = temporary_root / "runtime-smoke" + subprocess.run( + [ + sys.executable, + str(example_root / "scripts" / "prepare_runtime.py"), + "--run-root", + str(run_root), + "--switchyard-bundle", + str(switchyard_bundle), + "--relay-wheel", + str(relay_wheel), + "--relay-architecture", + relay_architecture, + "--plugin-config-template", + str(plugin_config_template), + "--openinference-endpoint", + "http://127.0.0.1:4318/v1/traces", + "--phoenix-project", + "phase2-smoke", + "--eval-cohort", + "phase2-smoke", + ], + check=True, + stdout=subprocess.DEVNULL, + ) + compatibility_path = run_root / "artifacts" / "harbor-hermes-compatibility.json" + subprocess.run( + [ + sys.executable, + str(example_root / "scripts" / "verify_harbor_hermes_compat.py"), + "--bridge", + str(example_root / "agents" / "harbor_hermes_agent.py"), + "--relay-config", + str(run_root / "runtime" / "plugins.toml"), + "--output", + str(compatibility_path), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + provenance = json.loads((run_root / "runtime" / "provenance.json").read_text()) + compatibility = json.loads(compatibility_path.read_text()) + with (run_root / "runtime" / "plugins.toml").open("rb") as stream: + relay_config = tomllib.load(stream) + with (switchyard_bundle / "relay-plugin.toml").open("rb") as stream: + switchyard_manifest = tomllib.load(stream) + dynamic_plugins = relay_config.get("plugins", {}).get("dynamic", []) + components = {component["kind"]: component for component in relay_config.get("components", [])} + plugin_id = switchyard_manifest.get("plugin", {}).get("id") + if ( + provenance.get("nemo_relay", {}).get("version") != "0.7.0" + or provenance.get("nemo_relay", {}).get("wheel_sha256") != sha256_file(relay_wheel) + or provenance.get("switchyard", {}).get("library_sha256") is None + or compatibility.get("status") != "passed" + or len(dynamic_plugins) != 1 + or dynamic_plugins[0].get("manifest") != "/opt/relay-plugins/nvidia.switchyard/relay-plugin.toml" + or plugin_id != "nvidia.switchyard" + or components.get("observability", {}).get("config", {}).get("version") != 3 + ): + raise ValueError("Relay/Hermes/Switchyard smoke wiring did not pass") + return { + "status": "passed", + "relay_version": "0.7.0", + "relay_wheel_sha256": provenance["nemo_relay"]["wheel_sha256"], + "switchyard_library_sha256": provenance["switchyard"]["library_sha256"], + "plugin_config_template_sha256": provenance["plugin_config_template_sha256"], + "relay_config_sha256": provenance["relay_config_sha256"], + "relay_architecture": provenance["nemo_relay"]["architecture"], + "routing": provenance["routing"], + "dynamic_plugin_id": plugin_id, + "observability_version": 3, + "compatibility_status": compatibility["status"], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset-root", type=Path, required=True) + parser.add_argument("--expected-count", type=int, default=89) + parser.add_argument("--harbor-bin", type=Path, required=True) + parser.add_argument("--switchyard-bundle", type=Path, required=True) + parser.add_argument("--relay-wheel", type=Path, required=True) + parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), required=True) + parser.add_argument("--plugin-config-template", type=Path, required=True) + parser.add_argument("--concurrency", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + example_root = Path(__file__).resolve().parents[1] + dataset_root = args.dataset_root.expanduser().resolve(strict=True) + if args.concurrency <= 0: + raise ValueError("concurrency must be positive") + if importlib.metadata.version("harbor") != "0.18.0": + raise RuntimeError("Phase 2 smoke requires Harbor 0.18.0") + if ( + subprocess.run([str(args.harbor_bin), "--version"], check=True, capture_output=True, text=True).stdout.strip() + != "0.18.0" + ): + raise RuntimeError("Phase 2 smoke Harbor CLI is not 0.18.0") + + with tempfile.TemporaryDirectory(prefix="harbor-phase2-smoke-") as directory: + temporary_root = Path(directory) + authorization_file = temporary_root / "switchyard-provider-authorization" + authorization_file.write_text("Bearer offline-placeholder", encoding="utf-8") + authorization_file.chmod(0o600) + task_records, network_attempts = asyncio.run( + validate_local_dataset( + dataset_root, + args.expected_count, + temporary_root / "jobs", + args.concurrency, + args.relay_architecture, + authorization_file, + ) + ) + validate_cli_local_projection(args.harbor_bin, dataset_root, task_records[0]["name"], authorization_file) + relay_runtime = validate_relay_runtime( + example_root, + temporary_root, + args.switchyard_bundle.expanduser().resolve(strict=True), + args.relay_wheel.expanduser().resolve(strict=True), + args.relay_architecture, + args.plugin_config_template.expanduser().resolve(strict=True), + ) + + task_tomls = [dataset_root / record["name"] / "task.toml" for record in task_records] + result = { + "schema_version": SCHEMA_VERSION, + "status": "passed", + "harbor_version": "0.18.0", + "dataset_name": dataset_root.name, + "task_count": len(task_records), + "dataset_task_definitions_sha256": combined_digest(dataset_root, task_tomls), + "registry_network_attempts": network_attempts, + "local_cli_projection": "passed", + "job_trial_count": len(task_records), + "concurrency": args.concurrency, + "relay_architecture": args.relay_architecture, + "relay_runtime": relay_runtime, + "memory_lanes": { + "2G": sum(record["memory_gb"] == 2 for record in task_records), + "4G": sum(record["memory_gb"] == 4 for record in task_records), + "8G": sum(record["memory_gb"] == 8 for record in task_records), + }, + "tasks": task_records, + } + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({key: value for key, value in result.items() if key != "tasks"}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/validate_parallel_isolation.py b/examples/harbor-hermes-switchyard/scripts/validate_parallel_isolation.py new file mode 100755 index 000000000..f0491185c --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/validate_parallel_isolation.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate that concurrent Phase 1 tasks used isolated runtime state.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = "harbor-hermes-switchyard.parallel-isolation.v1" + + +def read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"expected a JSON object: {path}") + return value + + +def validate(run_roots: list[Path]) -> dict[str, Any]: + if len(run_roots) < 2: + raise ValueError("parallel isolation requires at least two task roots") + records: list[dict[str, Any]] = [] + for root in run_roots: + resolved = root.resolve(strict=True) + summary = read_json(resolved / "summary.json") + if summary.get("status") != "passed": + raise ValueError(f"task summary did not pass: {resolved}") + artifacts = Path(summary.get("artifacts", "")).resolve(strict=True) + receipt = read_json(artifacts / "direct-hermes-receipt.json") + provenance = read_json(resolved / "runtime" / "provenance.json") + cleanup = receipt.get("cleanup") or {} + if not all(cleanup.get(key) is True for key in ("plugin_host_closed", "exporters_flushed")): + raise ValueError(f"task did not close plugin/exporter lifecycle: {resolved}") + records.append( + { + "task": summary.get("task_name"), + "run_root": str(resolved), + "artifact_root": str(artifacts), + "job_name": summary.get("job_name"), + "session_handle": receipt.get("session_handle"), + "relay_config_sha256": provenance.get("relay_config_sha256"), + "phoenix_project": provenance.get("phoenix_project"), + "evaluation_cohort": provenance.get("eval_cohort"), + "relay_wheel_sha256": provenance.get("nemo_relay", {}).get("wheel_sha256"), + "switchyard_library_sha256": provenance.get("switchyard", {}).get("library_sha256"), + } + ) + distinct_fields = ( + "run_root", + "artifact_root", + "job_name", + "session_handle", + "relay_config_sha256", + "phoenix_project", + "evaluation_cohort", + ) + for field in distinct_fields: + values = [record.get(field) for record in records] + if any(not value for value in values) or len(set(values)) != len(values): + raise ValueError(f"parallel tasks did not have distinct {field} values") + shared_fields = ("relay_wheel_sha256", "switchyard_library_sha256") + for field in shared_fields: + values = [record.get(field) for record in records] + if any(not value for value in values) or len(set(values)) != 1: + raise ValueError(f"parallel tasks did not use one immutable {field}") + return { + "schema_version": SCHEMA_VERSION, + "status": "passed", + "task_count": len(records), + "distinct_fields": list(distinct_fields), + "shared_input_fields": list(shared_fields), + "tasks": records, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = validate(args.run_root) + args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh new file mode 100755 index 000000000..901b5f2d1 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail +set +x + +env_file="${1:-}" +if [[ -z "$env_file" || "$env_file" != /* || ! -f "$env_file" ]]; then + echo "usage: $0 /absolute/phase2-run.env" >&2 + exit 2 +fi +if mode="$(stat -f '%Lp' "$env_file" 2>/dev/null)"; then + : +else + mode="$(stat -c '%a' "$env_file")" +fi +if [[ "$mode" != "600" ]]; then + echo "Phase 2 environment file must have mode 0600: $env_file" >&2 + exit 2 +fi +if grep -Eq '^(INFERENCE_SECRETS_FILE|NV_INFERENCEHUB_ENDPOINT|NV_INFERENCEHUB_KEY|STRONG_MODEL|WEAK_MODEL|UPSTREAM_BASE_URL|UPSTREAM_AUTH_ENV)=' "$env_file"; then + echo "legacy secret or provider-routing overrides are not supported in Phase 2" >&2 + exit 2 +fi + +set -a +# shellcheck disable=SC1090 +source "$env_file" +set +a + +required_values=( + EXAMPLE_ROOT PHASE2_RUN_ID PHASE2_RUN_ROOT PHASE2_ADMISSION_ROOT + HARBOR_BIN EVAL_PYTHON TBENCH_DATASET_PATH SWITCHYARD_BUNDLE RELAY_WHEEL + RELAY_ARCHITECTURE PLUGIN_CONFIG_TEMPLATE PHASE2_SMOKE_EVIDENCE + PHASE2_OFFLINE_EVIDENCE PHOENIX_BASE_URL PHOENIX_PROJECT EVAL_COHORT + TBENCH_SAMPLE_COUNT TBENCH_CANARY_TASK TBENCH_CONCURRENCY + TBENCH_PARALLEL_MAX_MEMORY_GB TBENCH_DOCKER_MEMORY_RESERVE_GB + TBENCH_MINIMUM_FREE_GB SWITCHYARD_PROVIDER_AUTHORIZATION +) +for name in "${required_values[@]}"; do + if [[ -z "${!name:-}" ]]; then + echo "required Phase 2 variable is unset: $name" >&2 + exit 2 + fi +done +for name in EXAMPLE_ROOT PHASE2_RUN_ROOT PHASE2_ADMISSION_ROOT HARBOR_BIN EVAL_PYTHON \ + TBENCH_DATASET_PATH SWITCHYARD_BUNDLE RELAY_WHEEL PLUGIN_CONFIG_TEMPLATE \ + PHASE2_SMOKE_EVIDENCE PHASE2_OFFLINE_EVIDENCE; do + if [[ "${!name}" != /* ]]; then + echo "Phase 2 path must be absolute: $name" >&2 + exit 2 + fi +done +for name in TBENCH_SAMPLE_COUNT TBENCH_CONCURRENCY TBENCH_PARALLEL_MAX_MEMORY_GB \ + TBENCH_DOCKER_MEMORY_RESERVE_GB TBENCH_MINIMUM_FREE_GB; do + if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then + echo "Phase 2 capacity value must be a positive integer: $name" >&2 + exit 2 + fi +done +case "$RELAY_ARCHITECTURE" in + x86_64|aarch64) ;; + *) echo "RELAY_ARCHITECTURE must be x86_64 or aarch64" >&2; exit 2 ;; +esac +for path in "$EXAMPLE_ROOT" "$TBENCH_DATASET_PATH" "$SWITCHYARD_BUNDLE"; do + [[ -d "$path" ]] || { echo "required Phase 2 directory is missing" >&2; exit 2; } +done +for path in "$HARBOR_BIN" "$EVAL_PYTHON" "$RELAY_WHEEL" "$PLUGIN_CONFIG_TEMPLATE"; do + [[ -f "$path" ]] || { echo "required Phase 2 file is missing" >&2; exit 2; } +done + +"$EVAL_PYTHON" - "$PLUGIN_CONFIG_TEMPLATE" <<'PY' +import sys +import tomllib + +with open(sys.argv[1], "rb") as stream: + config = tomllib.load(stream) +plugins = config.get("plugins", {}).get("dynamic", []) +if len(plugins) != 1: + raise SystemExit("plugin config must contain one dynamic plugin") +targets = plugins[0].get("config", {}).get("targets", {}) +if set(targets) != {"strong", "weak"}: + raise SystemExit("plugin config must contain strong and weak targets") +for target in targets.values(): + if target.get("header_env") != {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"}: + raise SystemExit("plugin config must reference SWITCHYARD_PROVIDER_AUTHORIZATION") +PY + +echo "Phase 2 environment validation passed (secret values withheld)" diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py index fc55d12db..b2a877405 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_run.py +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -43,6 +43,30 @@ def read_benchmark_passed(value: dict[str, Any]) -> bool | None: return None +def validate_harbor_job_config(job_dir: Path) -> tuple[dict[str, float], list[str]]: + """Validate the timeout multipliers serialized by Harbor for this job.""" + errors: list[str] = [] + path = job_dir / "config.json" + expected = { + "agent_timeout_multiplier": 3.0, + "agent_setup_timeout_multiplier": 6.0, + "environment_build_timeout_multiplier": 6.0, + } + if not path.is_file(): + return {}, [f"missing Harbor job config: {path}"] + config = read_json(path) + observed: dict[str, float] = {} + for key, expected_value in expected.items(): + value = config.get(key) + if not isinstance(value, (int, float)) or isinstance(value, bool): + errors.append(f"Harbor job config has no numeric {key}") + continue + observed[key] = float(value) + if observed[key] != expected_value: + errors.append(f"Harbor job config {key}={observed[key]} does not match required {expected_value}") + return observed, errors + + def contained_files(root: Path) -> list[Path]: resolved_root = root.resolve(strict=True) files: list[Path] = [] @@ -89,11 +113,111 @@ def scan_secrets(files: Iterable[Path], values: list[bytes]) -> list[str]: return sorted(set(findings)) -def read_atof(path: Path) -> tuple[int, list[str], list[str], list[str]]: +def validate_receipt_provenance(receipt: dict[str, Any], provenance: dict[str, Any]) -> list[str]: + errors: list[str] = [] + dependencies = receipt.get("dependencies", {}) + relay = dependencies.get("nemo_relay", {}) + hermes = dependencies.get("hermes", {}) + switchyard = dependencies.get("switchyard", {}) + provenance_relay = provenance.get("nemo_relay", {}) + provenance_hermes = provenance.get("hermes", {}) + provenance_switchyard = provenance.get("switchyard", {}) + if relay.get("version") != "0.7.0": + errors.append("receipt did not record nemo-relay==0.7.0") + if relay.get("wheel_sha256") != provenance_relay.get("wheel_sha256"): + errors.append("Relay wheel digest does not match runtime provenance") + if receipt.get("relay_config_sha256") != provenance.get("relay_config_sha256"): + errors.append("Relay config digest does not match runtime provenance") + if hermes.get("commit") != provenance_hermes.get("commit"): + errors.append("Hermes commit does not match runtime provenance") + if switchyard.get("commit") != provenance_switchyard.get("commit"): + errors.append("Switchyard commit does not match runtime provenance") + if switchyard.get("manifest_sha256") != provenance_switchyard.get("manifest_sha256"): + errors.append("Switchyard manifest digest does not match runtime provenance") + if switchyard.get("library_sha256") != provenance_switchyard.get("library_sha256"): + errors.append("Switchyard library digest does not match runtime provenance") + if receipt.get("dynamic_plugin_ids") != ["nvidia.switchyard"]: + errors.append("receipt did not record only nvidia.switchyard") + if receipt.get("activation_mode") != "relay_standard_dynamic": + errors.append("receipt did not record standard dynamic activation") + if receipt.get("routing_contract") != { + "relay_outer_lifecycle": True, + "execution_intercept_owner": "nvidia.switchyard", + "provider_http_client_owner": "switchyard-llm-client", + "separate_switchyard_service": False, + }: + errors.append("receipt did not record the expected Relay/Switchyard ownership contract") + cleanup = receipt.get("cleanup", {}) + if not cleanup.get("plugin_host_closed") or not cleanup.get("exporters_flushed"): + errors.append("receipt did not prove plugin close and exporter flush") + return errors + + +def _otlp_attribute_value(attribute: dict[str, Any]) -> Any: + value = attribute.get("value") + if not isinstance(value, dict): + return None + for key in ("stringValue", "boolValue", "intValue", "doubleValue"): + if key in value: + return value[key] + return None + + +def inspect_openinference(path: Path) -> dict[str, Any]: + documents = 0 + spans = 0 + span_kinds: set[str] = set() + scope_names: set[str] = set() + lineage_spans = 0 + resource_attributes: dict[str, set[Any]] = {} + with path.open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + if not line.strip(): + continue + payload = json.loads(line) + if not isinstance(payload, dict): + raise ValueError(f"OpenInference line {line_number} is not an object") + documents += 1 + for resource_span in payload.get("resourceSpans", []): + for attribute in resource_span.get("resource", {}).get("attributes", []): + key = attribute.get("key") + value = _otlp_attribute_value(attribute) + if isinstance(key, str) and value is not None: + resource_attributes.setdefault(key, set()).add(value) + for scope_span in resource_span.get("scopeSpans", []): + scope_name = scope_span.get("scope", {}).get("name") + if isinstance(scope_name, str) and scope_name: + scope_names.add(scope_name) + for span in scope_span.get("spans", []): + spans += 1 + attributes = { + attribute.get("key"): _otlp_attribute_value(attribute) + for attribute in span.get("attributes", []) + if isinstance(attribute, dict) + } + kind = attributes.get("openinference.span.kind") + if isinstance(kind, str) and kind: + span_kinds.add(kind) + if attributes.get("nemo_relay.uuid") and attributes.get("nemo_relay.scope_type"): + lineage_spans += 1 + return { + "documents": documents, + "spans": spans, + "span_kinds": sorted(span_kinds), + "scope_names": sorted(scope_names), + "lineage_spans": lineage_spans, + "resource_attributes": {key: sorted(values, key=str) for key, values in sorted(resource_attributes.items())}, + } + + +def inspect_atof(path: Path) -> dict[str, Any]: count = 0 marks: list[str] = [] models: list[str] = [] targets: list[str] = [] + cache_read_tokens = 0 + cache_write_tokens = 0 + decision_count = 0 with path.open(encoding="utf-8") as stream: for line_number, line in enumerate(stream, 1): if not line.strip(): @@ -105,6 +229,8 @@ def read_atof(path: Path) -> tuple[int, list[str], list[str], list[str]]: name = payload.get("name") if isinstance(name, str) and name.startswith("switchyard.routing."): marks.append(name) + if name == "switchyard.routing.decision": + decision_count += 1 for container in (payload.get("data"), payload.get("metadata")): if isinstance(container, dict): for key in ("model", "selected_model", "target_model"): @@ -114,7 +240,41 @@ def read_atof(path: Path) -> tuple[int, list[str], list[str], list[str]]: value = container.get("selected_target") if isinstance(value, str) and value: targets.append(value) - return count, sorted(set(marks)), sorted(set(models)), sorted(set(targets)) + profile = payload.get("category_profile") + response = profile.get("annotated_response") if isinstance(profile, dict) else None + data = payload.get("data") + if isinstance(response, dict): + model = response.get("model") + if isinstance(model, str) and model: + models.append(model) + if name == "openai.chat_completions" and payload.get("scope_category") == "end": + model = data.get("model") if isinstance(data, dict) else None + if isinstance(model, str) and model: + models.append(model) + if name == "llm.chunk" and isinstance(data, dict): + usage = data.get("usage") + if isinstance(usage, dict): + cache_read = usage.get("cache_read_tokens") + cache_write = usage.get("cache_write_tokens") + if isinstance(cache_read, int) and not isinstance(cache_read, bool): + cache_read_tokens += cache_read + if isinstance(cache_write, int) and not isinstance(cache_write, bool): + cache_write_tokens += cache_write + return { + "count": count, + "marks": sorted(set(marks)), + "models": sorted(set(models)), + "targets": sorted(set(targets)), + "decision_count": decision_count, + "cache_read_tokens": cache_read_tokens, + "cache_write_tokens": cache_write_tokens, + } + + +def read_atof(path: Path) -> tuple[int, list[str], list[str], list[str]]: + """Retain the Phase 1 tuple API while Phase 2 consumes richer evidence.""" + evidence = inspect_atof(path) + return evidence["count"], evidence["marks"], evidence["models"], evidence["targets"] def main() -> int: @@ -175,32 +335,55 @@ def main() -> int: errors.append("missing runtime provenance") if receipt: - dependencies = receipt.get("dependencies", {}) - relay = dependencies.get("nemo_relay", {}) - hermes = dependencies.get("hermes", {}) - switchyard = dependencies.get("switchyard", {}) - if relay.get("version") != "0.7.0": - errors.append("receipt did not record nemo-relay==0.7.0") - if relay.get("wheel_sha256") != provenance.get("nemo_relay", {}).get("wheel_sha256"): - errors.append("Relay wheel digest does not match runtime provenance") - if hermes.get("commit") != provenance.get("hermes", {}).get("commit"): - errors.append("Hermes commit does not match runtime provenance") - if switchyard.get("commit") != provenance.get("switchyard", {}).get("commit"): - errors.append("Switchyard commit does not match runtime provenance") - if receipt.get("dynamic_plugin_ids") != ["nvidia.switchyard"]: - errors.append("receipt did not record only nvidia.switchyard") - if receipt.get("activation_mode") != "relay_standard_dynamic": - errors.append("receipt did not record standard dynamic activation") - cleanup = receipt.get("cleanup", {}) - if not cleanup.get("plugin_host_closed") or not cleanup.get("exporters_flushed"): - errors.append("receipt did not prove plugin close and exporter flush") + errors.extend(validate_receipt_provenance(receipt, provenance)) + + openinference_evidence = { + "documents": 0, + "spans": 0, + "span_kinds": [], + "scope_names": [], + "lineage_spans": 0, + "resource_attributes": {}, + } + if args.openinference.is_file() and args.openinference.stat().st_size > 0: + try: + openinference_evidence = inspect_openinference(args.openinference) + except Exception as error: + errors.append(f"invalid OpenInference OTLP artifact: {error}") + if openinference_evidence["documents"] == 0 or openinference_evidence["spans"] == 0: + errors.append("OpenInference artifact contains no spans") + if not {"AGENT", "LLM"}.issubset(openinference_evidence["span_kinds"]): + errors.append("OpenInference artifact does not contain both AGENT and LLM span kinds") + if openinference_evidence["scope_names"] != ["harbor-hermes-switchyard"]: + errors.append("OpenInference instrumentation scope does not match the example") + if openinference_evidence["lineage_spans"] != openinference_evidence["spans"]: + errors.append("OpenInference spans are missing Relay UUID or scope-type lineage") + expected_resources = { + "openinference.project.name": provenance.get("phoenix_project"), + "evaluation.cohort": provenance.get("eval_cohort"), + "service.name": "harbor-hermes-switchyard", + "service.namespace": "nemo-relay-examples", + } + for key, expected in expected_resources.items(): + if openinference_evidence["resource_attributes"].get(key) != [expected]: + errors.append(f"OpenInference resource attribute {key!r} does not match runtime provenance") event_count = 0 routing_marks: list[str] = [] routed_models: list[str] = [] routed_targets: list[str] = [] + switchyard_decision_count = 0 + cache_read_tokens = 0 + cache_write_tokens = 0 if required["atof"].is_file(): - event_count, routing_marks, routed_models, routed_targets = read_atof(required["atof"]) + atof_evidence = inspect_atof(required["atof"]) + event_count = atof_evidence["count"] + routing_marks = atof_evidence["marks"] + routed_models = atof_evidence["models"] + routed_targets = atof_evidence["targets"] + switchyard_decision_count = atof_evidence["decision_count"] + cache_read_tokens = atof_evidence["cache_read_tokens"] + cache_write_tokens = atof_evidence["cache_write_tokens"] if event_count == 0: errors.append("ATOF artifact is empty") if not routing_marks: @@ -212,6 +395,16 @@ def main() -> int: errors.append(f"ATOF artifact selected unexpected targets: {unexpected_targets}") caller_model = provenance.get("routing", {}).get("hermes_caller_model") + target_models = { + "strong": provenance.get("routing", {}).get("strong_model"), + "weak": provenance.get("routing", {}).get("weak_model"), + } + routed_models = sorted( + { + *routed_models, + *(target_models[target] for target in routed_targets if target_models.get(target)), + } + ) if caller_model and caller_model in routed_models: errors.append("Hermes caller stub appeared as a routed provider model") @@ -237,8 +430,11 @@ def main() -> int: errors.append(f"secret scan found {len(findings)} persisted value(s)") harbor_results: list[Path] = [] + harbor_timeout_multipliers: dict[str, float] = {} benchmark_passed: bool | None = None if args.harbor_job_dir: + harbor_timeout_multipliers, timeout_errors = validate_harbor_job_config(args.harbor_job_dir) + errors.extend(timeout_errors) harbor_results = [ path for path in sorted(args.harbor_job_dir.glob("**/result.json")) if is_trial_result(read_json(path)) ] @@ -254,12 +450,17 @@ def main() -> int: "errors": errors, "direct_result_status": result.get("status"), "harbor_trial_count": len(harbor_results) if args.harbor_job_dir else None, + "harbor_timeout_multipliers": harbor_timeout_multipliers, "benchmark_task_passed": benchmark_passed, "atof_event_count": event_count, "atif_trajectory_count": len(atif_files), + "openinference": openinference_evidence, "switchyard_routing_marks": routing_marks, + "switchyard_decision_count": switchyard_decision_count, "routed_models": routed_models, "routed_targets": routed_targets, + "cache_read_tokens": cache_read_tokens, + "cache_write_tokens": cache_write_tokens, "secret_values_scanned": len(secret_values), "secret_findings": findings, } diff --git a/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh b/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh new file mode 100755 index 000000000..d2aeb308a --- /dev/null +++ b/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +run_root="${1:-}" +if [[ -z "$run_root" || "$run_root" != /* ]]; then + echo "usage: $0 /absolute/phase2-run-root [coordinator options]" >&2 + exit 2 +fi +shift + +child_pid="" +terminate_group() { + if [[ -z "$child_pid" ]]; then + return + fi + kill -TERM -- "-$child_pid" >/dev/null 2>&1 || true + for _ in {1..10}; do + if ! kill -0 -- "-$child_pid" >/dev/null 2>&1; then + return + fi + sleep 1 + done + kill -KILL -- "-$child_pid" >/dev/null 2>&1 || true +} +terminate() { + terminate_group + [[ -z "$child_pid" ]] || wait "$child_pid" >/dev/null 2>&1 || true + exit 143 +} +trap terminate INT TERM + +backoff_seconds="${PHASE2_SUPERVISOR_BACKOFF_SECONDS:-60}" +maximum_backoff_seconds="${PHASE2_SUPERVISOR_MAX_BACKOFF_SECONDS:-900}" +if [[ ! "$backoff_seconds" =~ ^[1-9][0-9]*$ || ! "$maximum_backoff_seconds" =~ ^[1-9][0-9]*$ ]]; then + echo "Phase 2 supervisor backoffs must be positive integers" >&2 + exit 2 +fi + +while true; do + python_bin="${EVAL_PYTHON:-$example_root/.venv/bin/python}" + "$python_bin" "$example_root/scripts/exec_process_group.py" \ + "$example_root/run_phase2_cohort.sh" "$run_root" "$@" & + child_pid=$! + wait "$child_pid" + status=$? + terminate_group + child_pid="" + case "$status" in + 0) + exit 0 + ;; + 20) + echo "[phase2-supervisor] stopping on preserved harness/integration blocker" >&2 + exit 20 + ;; + *) + echo "[phase2-supervisor] coordinator exited $status; resuming in ${backoff_seconds}s" >&2 + sleep "$backoff_seconds" + if ((backoff_seconds < maximum_backoff_seconds)); then + backoff_seconds=$((backoff_seconds * 2)) + if ((backoff_seconds > maximum_backoff_seconds)); then + backoff_seconds=$maximum_backoff_seconds + fi + fi + ;; + esac +done diff --git a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py index 4c244d3db..39dcbd18d 100644 --- a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py @@ -34,7 +34,7 @@ def make_args(tmp_path: Path, *, error_type: str = "") -> argparse.Namespace: relay_wheel_sha256="a" * 64, hermes_repository="https://github.com/bbednarski9/hermes-agent.git", hermes_commit="efb63e714abc436af88af9b0d6734751c199aa6d", - switchyard_commit="8293936a0f5758aa1a782639d485b8b8948cf03e", + switchyard_commit="5d9d3292d6154e44d50295d0d4a3fd4f144f2528", session_handle="phase1-session", started_at=1.0, error_type=error_type, @@ -119,3 +119,30 @@ def test_empty_session_uses_bounded_quiet_cli_output(tmp_path: Path, monkeypatch assert result["status"] == "completed" assert result["session_id"] == "cli-session" assert result["final_response"] == "completed\nanswer" + + +def test_failed_agent_output_is_not_promoted_to_a_completed_response(tmp_path: Path, monkeypatch) -> None: + module = load_finalizer() + root = tmp_path / "artifacts" + root.mkdir() + session = tmp_path / "hermes-session.jsonl" + session.write_text("", encoding="utf-8") + log = tmp_path / "hermes.txt" + log.write_text( + "session_id: failed-session\nAPI call failed after 3 retries: provider returned HTTP 400\n", + encoding="utf-8", + ) + monkeypatch.setattr(module, "HERMES_SESSION", session) + monkeypatch.setattr(module, "HERMES_LOG", log) + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.0") + + args = make_args(tmp_path, error_type="NonZeroAgentExitCodeError") + module.initialize(args, root) + module.complete(args, root) + + result = json.loads((root / "direct-hermes-result.json").read_text()) + completion = json.loads((root / "completion.json").read_text()) + assert result["status"] == "failed" + assert result["final_response"] is None + assert result["error"] == {"type": "NonZeroAgentExitCodeError", "phase": "agent"} + assert completion["status"] == "failed" diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py index a6ccf9235..c55479d4e 100644 --- a/examples/harbor-hermes-switchyard/tests/test_config_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_config_contract.py @@ -3,25 +3,26 @@ from __future__ import annotations +import asyncio +import importlib.util import re +import sys import tomllib from pathlib import Path +from unittest.mock import AsyncMock, patch + +from harbor.agents.installed.hermes import Hermes EXAMPLE_ROOT = Path(__file__).resolve().parents[1] def render_template(**values: str) -> dict: - text = (EXAMPLE_ROOT / "config" / "relay.toml.in").read_text(encoding="utf-8") + text = (EXAMPLE_ROOT / "config" / "plugins.toml.in").read_text(encoding="utf-8") defaults = { - "STRONG_MODEL": "phase1-test-strong", - "WEAK_MODEL": "phase1-test-weak", - "HERMES_CALLER_MODEL": "ollama-route-stub", "HERMES_COMMIT": "efb63e714abc436af88af9b0d6734751c199aa6d", "OPENINFERENCE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", "PHOENIX_PROJECT": "phase1-test", "EVAL_COHORT": "phase1-test", - "UPSTREAM_BASE_URL": "http://127.0.0.1:8000/v1", - "UPSTREAM_AUTH_ENV": "SWITCHYARD_PROVIDER_AUTHORIZATION", } defaults.update(values) for key, value in defaults.items(): @@ -47,7 +48,6 @@ def test_config_uses_static_schema_v3_and_one_standard_dynamic_plugin() -> None: "weak_target": "weak", "strong_target": "strong", "base_threshold": 0.5, - "min_confidence": 0.0, "recent_turn_window": 0, "session_affinity": True, "message_hash_fallback": True, @@ -55,9 +55,8 @@ def test_config_uses_static_schema_v3_and_one_standard_dynamic_plugin() -> None: assert plugin["config"]["default_targets"] == {"openai_chat": "strong"} assert set(plugin["config"]["targets"]) == {"strong", "weak"} for target in plugin["config"]["targets"].values(): - assert target["header_env"] == { - "authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION" - } + assert target["header_env"] == {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"} + assert target["drop_caller_extra_body"] is True def test_config_contains_no_literal_provider_headers_or_credentials() -> None: @@ -76,20 +75,41 @@ def walk(value: object) -> None: def test_pricing_does_not_duplicate_relay_generated_aliases() -> None: - config = render_template( - STRONG_MODEL="namespace/strong", - WEAK_MODEL="namespace/weak", - ) + config = render_template() entries = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] - assert [entry["model_id"] for entry in entries] == ["namespace/strong", "namespace/weak"] + assert [entry["model_id"] for entry in entries] == [ + "aws/anthropic/bedrock-claude-opus-4-6", + "aws/anthropic/bedrock-claude-sonnet-4-6", + ] assert all("aliases" not in entry for entry in entries) +def test_pricing_uses_nonzero_claude_46_list_rates() -> None: + config = render_template() + entries = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] + rates = {entry["model_id"]: entry["rates"] for entry in entries} + assert rates["aws/anthropic/bedrock-claude-opus-4-6"] == { + "input_per_million": 5.0, + "output_per_million": 25.0, + "cache_read_per_million": 0.5, + "cache_write_per_million": 6.25, + } + assert rates["aws/anthropic/bedrock-claude-sonnet-4-6"] == { + "input_per_million": 3.0, + "output_per_million": 15.0, + "cache_read_per_million": 0.3, + "cache_write_per_million": 3.75, + } + + def test_switchyard_models_are_distinct_from_fail_closed_hermes_caller() -> None: config = render_template() plugin = config["plugins"]["dynamic"][0] provider_models = {target["model"] for target in plugin["config"]["targets"].values()} - assert provider_models == {"phase1-test-strong", "phase1-test-weak"} + assert provider_models == { + "aws/anthropic/bedrock-claude-opus-4-6", + "aws/anthropic/bedrock-claude-sonnet-4-6", + } observability = next(item for item in config["components"] if item["kind"] == "observability") assert observability["config"]["atif"]["model_name"] == "ollama-route-stub" assert "ollama-route-stub" not in provider_models @@ -105,9 +125,75 @@ def test_task_runner_defaults_to_production_x86_64_architecture() -> None: def test_task_runner_defaults_to_inference_hub_tiers_and_fail_closed_caller() -> None: runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") - assert "aws/anthropic/bedrock-claude-opus-4-6" in runner - assert "aws/anthropic/bedrock-claude-sonnet-4-6" in runner - assert 'hermes_caller_model="${HERMES_CALLER_MODEL:-ollama-route-stub}"' in runner + assert "STRONG_MODEL" not in runner + assert "WEAK_MODEL" not in runner + assert 'plugin_config_template="${PLUGIN_CONFIG_TEMPLATE:-$example_root/config/plugins.toml.in}"' in runner assert '--model "openai/$hermes_caller_model"' in runner assert 'fail_closed_openai_base_url="http://127.0.0.1:9/v1"' in runner assert '--ae "OPENAI_BASE_URL=$fail_closed_openai_base_url"' in runner + + +def test_task_runner_projects_provider_authorization_by_read_only_mount() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + assert '--ae "$upstream_auth_env=' not in runner + assert 'host_temporary_root="$(cd "${TMPDIR:-/tmp}" && pwd -P)"' in runner + assert '"$(dirname "$run_root")/.phase2-secret.' not in runner + assert '"$run_root/"*)' in runner + assert 'provider_authorization_target="/run/secrets/switchyard-provider-authorization"' in runner + assert '"read_only": True' in runner + assert '"bind": {"create_host_path": False}' in runner + assert '--mounts "$mounts_json"' in runner + assert "'OPENAI_API_KEY=${OPENAI_API_KEY}'" in runner + + +def test_agent_reads_provider_authorization_inside_container_only() -> None: + path = EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py" + spec = importlib.util.spec_from_file_location("phase2_secret_agent", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + agent = object.__new__(module.HarborHermesAgent) + agent._load_provider_authorization = True + with patch.object(Hermes, "exec_as_agent", new_callable=AsyncMock) as parent: + asyncio.run(agent.exec_as_agent(object(), "hermes --yolo chat", env={"SAFE": "value"})) + command = parent.await_args.args[-1] + assert "cat -- /run/secrets/switchyard-provider-authorization" in command + assert 'export SWITCHYARD_PROVIDER_AUTHORIZATION="$(cat --' in command + assert parent.await_args.kwargs["env"] == {"SAFE": "value"} + + +def test_agent_install_retries_transient_apt_failures() -> None: + agent = (EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py").read_text(encoding="utf-8") + assert "for attempt in 1 2 3; do " in agent + assert "apt-get update && apt-get install -y --no-install-recommends " in agent + assert "sleep $((attempt * 5))" in agent + + +def test_phase2_environment_template_consolidates_secret_without_legacy_file() -> None: + template = (EXAMPLE_ROOT / "phase2-run.env.example").read_text(encoding="utf-8") + assert "SWITCHYARD_PROVIDER_AUTHORIZATION='Bearer replace-with-provider-token'" in template + assert "INFERENCE_SECRETS_FILE" not in template + assert "NV_INFERENCEHUB_KEY" not in template + assert "NV_INFERENCEHUB_ENDPOINT" not in template + assert "STRONG_MODEL" not in template + assert "WEAK_MODEL" not in template + assert "UPSTREAM_BASE_URL" not in template + assert "phase2-run.env" in (EXAMPLE_ROOT / ".gitignore").read_text(encoding="utf-8") + + +def test_phase2_readme_uses_admissions_instead_of_phase1_regressions() -> None: + readme = (EXAMPLE_ROOT / "README.md").read_text(encoding="utf-8") + assert "run_phase1_regressions.sh" not in readme + assert "PHASE1_EVIDENCE_ROOT" not in readme + assert "INFERENCE_SECRETS_FILE" not in readme + assert "all-89 no-token admission" in readme.lower() + assert "Docker offline runtime admission" in readme + + +def test_phase2_runner_requires_and_uses_the_local_dataset_export() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + assert 'tbench_dataset_path="${TBENCH_DATASET_PATH:-}"' in runner + assert 'dataset_args=(--path "$tbench_dataset_path")' in runner + assert '"${dataset_args[@]}"' in runner + assert "Phase 2 requires TBENCH_DATASET_PATH" in runner diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py new file mode 100644 index 000000000..aa25f1c59 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -0,0 +1,427 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from io import BytesIO +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def load_coordinator(): + path = EXAMPLE_ROOT / "scripts" / "run_phase2_cohort.py" + spec = importlib.util.spec_from_file_location("phase2_coordinator", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def load_isolation_validator(): + path = EXAMPLE_ROOT / "scripts" / "validate_parallel_isolation.py" + spec = importlib.util.spec_from_file_location("parallel_isolation_validator", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_task(dataset: Path, name: str, memory: str) -> None: + task = dataset / name + task.mkdir(parents=True) + (task / "task.toml").write_text(f'[environment]\nmemory = "{memory}"\n', encoding="utf-8") + (task / "instruction.md").write_text(f"instruction for {name}\n", encoding="utf-8") + (task / "tests").mkdir() + (task / "tests" / "test.sh").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + + +def write_passed_attempt( + root: Path, + task, + *, + model: str, + cache_read: int, + benchmark_passed: bool, +) -> None: + attempt = root / "tasks" / task.directory_name / "attempts" / "001" + attempt.mkdir(parents=True) + summary = { + "status": "passed", + "validation": { + "status": "passed", + "benchmark_task_passed": benchmark_passed, + "direct_result_status": "completed", + "switchyard_decision_count": 2, + "routed_models": [model], + "routed_targets": ["strong"], + "cache_read_tokens": cache_read, + "cache_write_tokens": 0, + "secret_findings": [], + }, + "phoenix_upload": {"status": "passed", "uploaded_spans": 10}, + } + (attempt / "summary.json").write_text(json.dumps(summary), encoding="utf-8") + + +def cohort_args(root: Path) -> argparse.Namespace: + return argparse.Namespace( + run_root=root, + dataset="terminal-bench@2.0", + phoenix_project="phase2-project", + eval_cohort="phase2-cohort", + required_model=["sonnet", "opus"], + require_cache_hit=True, + ) + + +def test_task_discovery_places_explicit_canary_before_lexical_lane(tmp_path: Path) -> None: + module = load_coordinator() + write_task(tmp_path, "task-z", "4G") + write_task(tmp_path, "task-a", "2G") + tasks = module.discover_tasks(tmp_path, 2, set(), "task-z") + assert [(task.index, task.name, task.memory_gb) for task in tasks] == [ + (1, "task-z", 4), + (2, "task-a", 2), + ] + + +def test_failed_attempt_is_preserved_and_passed_attempt_wins(tmp_path: Path) -> None: + module = load_coordinator() + task = module.Task(1, "task", 2) + attempts = tmp_path / task.directory_name / "attempts" + failed = attempts / "001" + failed.mkdir(parents=True) + (failed / "summary.json").write_text('{"status":"failed"}', encoding="utf-8") + passed = attempts / "002" + passed.mkdir() + (passed / "summary.json").write_text( + json.dumps( + { + "status": "passed", + "validation": {"status": "passed"}, + "phoenix_upload": {"status": "passed"}, + } + ), + encoding="utf-8", + ) + assert module.successful_attempt(tmp_path / task.directory_name) == passed + assert failed.is_dir() + + +def test_cohort_summary_requires_completion_cache_routes_and_secret_scan(tmp_path: Path) -> None: + module = load_coordinator() + tasks = [module.Task(1, "one", 2), module.Task(2, "two", 2)] + write_passed_attempt(tmp_path, tasks[0], model="sonnet", cache_read=12, benchmark_passed=True) + write_passed_attempt(tmp_path, tasks[1], model="opus", cache_read=0, benchmark_passed=False) + summary = module.aggregate_summary(cohort_args(tmp_path), tasks) + assert summary["status"] == "passed" + assert summary["completed_tasks"] == 2 + assert summary["benchmark_pass_count"] == 1 + assert summary["benchmark_nonpass_count"] == 1 + assert summary["cohort_gates"]["cache_hit"]["cache_read_tokens"] == 12 + assert summary["cohort_gates"]["route_diversity"]["observed_models"] == ["opus", "sonnet"] + + +def test_cohort_summary_blocks_missing_route_even_when_tasks_pass(tmp_path: Path) -> None: + module = load_coordinator() + tasks = [module.Task(1, "one", 2)] + write_passed_attempt(tmp_path, tasks[0], model="opus", cache_read=12, benchmark_passed=True) + summary = module.aggregate_summary(cohort_args(tmp_path), tasks) + assert summary["status"] == "partial" + assert summary["cohort_gates"]["route_diversity"]["missing_models"] == ["sonnet"] + + +def test_failure_classifier_retries_only_known_infrastructure_failures() -> None: + module = load_coordinator() + assert module.classify_failure("TLS handshake timeout contacting registry-1.docker.io") == "infrastructure" + assert module.classify_failure("ConnectError: Error getting dataset terminal-bench@2.0") == "infrastructure" + assert module.classify_failure("Command failed (exit 100): apt-get update && apt-get install") == "infrastructure" + assert module.classify_failure("receipt did not prove plugin close") == "harness_or_integration" + + +def test_failure_classifier_reads_nested_harbor_logs(tmp_path: Path) -> None: + module = load_coordinator() + attempt = tmp_path / "attempt" + nested = attempt / "jobs" / "task" / "trial.log" + nested.parent.mkdir(parents=True) + nested.write_text( + 'failed to do request: Head "https://registry-1.docker.io/v2/library/debian/manifests/13.0-slim": ' + "context deadline exceeded\n", + encoding="utf-8", + ) + + assert module.classify_attempt_failure("expected one direct Hermes result, found 0", attempt) == "infrastructure" + + +def test_smoke_evidence_is_bound_to_exact_local_dataset(tmp_path: Path) -> None: + module = load_coordinator() + dataset = tmp_path / "dataset" + write_task(dataset, "task-a", "2G") + write_task(dataset, "task-b", "4G") + task_tomls = sorted(dataset.glob("*/task.toml")) + relay_wheel = tmp_path / "relay.whl" + relay_wheel.write_bytes(b"relay") + bundle = tmp_path / "bundle" + bundle.mkdir() + library = bundle / "libswitchyard_nemo_relay_plugin.so" + library.write_bytes(b"switchyard") + plugin_template = EXAMPLE_ROOT / "config" / "plugins.toml.in" + evidence_path = tmp_path / "smoke.json" + evidence = { + "schema_version": "harbor-hermes-switchyard.phase2-smoke.v1", + "status": "passed", + "task_count": 2, + "dataset_task_definitions_sha256": module.sha256_file_set(dataset, task_tomls), + "registry_network_attempts": 0, + "concurrency": 3, + "relay_architecture": "aarch64", + "relay_runtime": { + "status": "passed", + "relay_wheel_sha256": module.sha256_file(relay_wheel), + "switchyard_library_sha256": module.sha256_file(library), + "plugin_config_template_sha256": module.sha256_file(plugin_template), + }, + "tasks": [ + { + "name": name, + "instruction_path": f"{name}/instruction.md", + "verifier_path": f"{name}/tests/test.sh", + "task_toml_sha256": module.sha256_file(dataset / name / "task.toml"), + "instruction_sha256": module.sha256_file(dataset / name / "instruction.md"), + "test_sha256": module.sha256_file(dataset / name / "tests" / "test.sh"), + } + for name in ("task-a", "task-b") + ], + } + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + module.validate_smoke_evidence(evidence_path, 2, dataset, 3, "aarch64", relay_wheel, bundle, plugin_template) + + (dataset / "task-b" / "task.toml").write_text('[environment]\nmemory = "8G"\n', encoding="utf-8") + try: + module.validate_smoke_evidence(evidence_path, 2, dataset, 3, "aarch64", relay_wheel, bundle, plugin_template) + except ValueError: + pass + else: + raise AssertionError("stale smoke evidence accepted a changed dataset") + + (dataset / "task-b" / "task.toml").write_text('[environment]\nmemory = "4G"\n', encoding="utf-8") + (dataset / "task-a" / "instruction.md").write_text("changed instruction\n", encoding="utf-8") + try: + module.validate_smoke_evidence(evidence_path, 2, dataset, 3, "aarch64", relay_wheel, bundle, plugin_template) + except ValueError: + pass + else: + raise AssertionError("stale smoke evidence accepted a changed instruction") + (dataset / "task-a" / "instruction.md").write_text("instruction for task-a\n", encoding="utf-8") + + try: + module.validate_smoke_evidence(evidence_path, 2, dataset, 4, "aarch64", relay_wheel, bundle, plugin_template) + except ValueError: + pass + else: + raise AssertionError("smoke evidence accepted changed concurrency") + + +def test_parallel_isolation_requires_distinct_state_and_shared_inputs(tmp_path: Path) -> None: + module = load_isolation_validator() + roots = [] + for index, task in enumerate(("one", "two"), 1): + root = tmp_path / task + artifacts = root / "artifacts" + artifacts.mkdir(parents=True) + (root / "runtime").mkdir() + (root / "summary.json").write_text( + json.dumps( + { + "status": "passed", + "task_name": task, + "job_name": f"job-{task}", + "artifacts": str(artifacts), + } + ), + encoding="utf-8", + ) + (artifacts / "direct-hermes-receipt.json").write_text( + json.dumps( + { + "session_handle": f"session-{task}", + "cleanup": {"plugin_host_closed": True, "exporters_flushed": True}, + } + ), + encoding="utf-8", + ) + (root / "runtime" / "provenance.json").write_text( + json.dumps( + { + "relay_config_sha256": f"config-{index}", + "phoenix_project": f"project-{index}", + "eval_cohort": f"cohort-{index}", + "nemo_relay": {"wheel_sha256": "relay"}, + "switchyard": {"library_sha256": "switchyard"}, + } + ), + encoding="utf-8", + ) + roots.append(root) + result = module.validate(roots) + assert result["status"] == "passed" + assert result["task_count"] == 2 + + +def test_durable_supervisor_owns_the_coordinator_process_group() -> None: + supervisor = (EXAMPLE_ROOT / "supervise_phase2_cohort.sh").read_text(encoding="utf-8") + assert "scripts/exec_process_group.py" in supervisor + assert 'kill -TERM -- "-$child_pid"' in supervisor + assert 'kill -KILL -- "-$child_pid"' in supervisor + + +def test_tmux_launcher_projects_only_the_protected_file_path() -> None: + launcher = (EXAMPLE_ROOT / "scripts" / "launch_phase2_tmux.sh").read_text(encoding="utf-8") + child = (EXAMPLE_ROOT / "scripts" / "run_phase2_from_env.sh").read_text(encoding="utf-8") + assert '-e "PHASE2_ENV_FILE=$env_file"' in launcher + assert 'source "$env_file"' not in launcher + assert "tmux has-session" in launcher + assert 'source "$env_file"' in child + assert "set +x" in child + assert "supervisor.log" in child + + +def test_phase2_launcher_is_local_dataset_only() -> None: + launcher = (EXAMPLE_ROOT / "run_phase2_cohort.sh").read_text(encoding="utf-8") + assert 'dataset_root="${TBENCH_DATASET_PATH:-$dataset_export_root/$dataset_name}"' in launcher + assert "datasets download" not in launcher + assert "never downloads or resolves a dataset through the Harbor registry" in launcher + assert '--smoke-evidence "$smoke_evidence"' in launcher + assert '--offline-evidence "$offline_evidence"' in launcher + assert "PHASE1_EVIDENCE_ROOT" not in launcher + assert "INFERENCE_SECRETS_FILE" not in launcher + + +def test_plugin_contract_owns_routes_and_authorization_name() -> None: + module = load_coordinator() + contract = module.plugin_contract(EXAMPLE_ROOT / "config" / "plugins.toml.in") + assert contract["strong_model"] == "aws/anthropic/bedrock-claude-opus-4-6" + assert contract["weak_model"] == "aws/anthropic/bedrock-claude-sonnet-4-6" + assert contract["hermes_caller_model"] == "ollama-route-stub" + assert contract["provider_base_urls"] == ["https://inference-api.nvidia.com/v1"] + + +def test_provider_catalog_requires_every_configured_route_without_persisting_authorization() -> None: + module = load_coordinator() + + class Response(BytesIO): + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + models = ["provider/sonnet", "provider/opus"] + response = Response(json.dumps({"data": [{"id": model} for model in models]}).encode()) + with patch.object(module.urllib.request, "urlopen", return_value=response) as request: + assert module.verify_provider_catalog("https://provider.example/v1", "Bearer secret", models) == sorted(models) + projected = request.call_args.args[0] + assert projected.full_url == "https://provider.example/v1/models" + assert projected.headers["Authorization"] == "Bearer secret" + + missing = Response(json.dumps({"data": [{"id": models[0]}]}).encode()) + with patch.object(module.urllib.request, "urlopen", return_value=missing): + try: + module.verify_provider_catalog("https://provider.example/v1", "Bearer secret", models) + except RuntimeError as error: + assert models[1] in str(error) + assert "Bearer secret" not in str(error) + else: + raise AssertionError("provider catalog accepted a missing configured model") + + +def test_capacity_requirement_covers_parallel_lane_and_largest_serial_task() -> None: + module = load_coordinator() + args = argparse.Namespace(concurrency=6, parallel_max_memory_gb=2, docker_memory_reserve_gb=4) + tasks = [module.Task(1, "canary", 2), module.Task(2, "large", 8)] + assert module.capacity_requirement_gb(args, tasks) == 16 + args.concurrency = 2 + assert module.capacity_requirement_gb(args, tasks) == 12 + assert module.normalize_architecture("amd64") == "x86_64" + assert module.normalize_architecture("arm64") == "aarch64" + + +def test_preflight_hard_rejects_capacity_above_docker_memory(tmp_path: Path) -> None: + module = load_coordinator() + args = argparse.Namespace( + run_root=tmp_path / "run", + minimum_free_gb=100, + concurrency=4, + parallel_max_memory_gb=2, + docker_memory_reserve_gb=4, + relay_architecture="aarch64", + ) + docker_info = json.dumps({"NCPU": 8, "MemTotal": 11 * 1024**3, "Architecture": "arm64"}) + completed = SimpleNamespace(stdout=docker_info) + tasks = [module.Task(1, "canary", 2)] + with ( + patch.object(module.shutil, "disk_usage", return_value=SimpleNamespace(free=200 * 1024**3)), + patch.object(module.subprocess, "run", return_value=completed), + ): + try: + module.shared_preflight(args, tasks) + except RuntimeError as error: + assert "requires 12 GiB" in str(error) + else: + raise AssertionError("unsafe Docker memory capacity was accepted") + + +def test_existing_plan_is_immutable(tmp_path: Path) -> None: + module = load_coordinator() + path = tmp_path / "plan.json" + module.load_or_create_plan(path, {"concurrency": 4}) + module.load_or_create_plan(path, {"concurrency": 4}) + try: + module.load_or_create_plan(path, {"concurrency": 6}) + except ValueError: + pass + else: + raise AssertionError("existing plan accepted changed concurrency") + + +def test_offline_evidence_is_bound_to_runtime_inputs(tmp_path: Path) -> None: + module = load_coordinator() + relay_wheel = tmp_path / "relay.whl" + relay_wheel.write_bytes(b"relay") + bundle = tmp_path / "bundle" + bundle.mkdir() + library = bundle / "libswitchyard_nemo_relay_plugin.so" + library.write_bytes(b"switchyard") + plugin_template = EXAMPLE_ROOT / "config" / "plugins.toml.in" + evidence_path = tmp_path / "offline.json" + evidence = { + "schema_version": "harbor-hermes-switchyard.phase2-offline-admission.v1", + "status": "passed", + "hermes_commit": module.EXPECTED_HERMES_COMMIT, + "relay_architecture": "x86_64", + "relay_wheel_sha256": module.sha256_file(relay_wheel), + "switchyard_library_sha256": module.sha256_file(library), + "plugin_config_template_sha256": module.sha256_file(plugin_template), + "provider_requests": 4, + "surviving_shutdown_threads": [], + } + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + module.validate_offline_evidence(evidence_path, "x86_64", relay_wheel, bundle, plugin_template) + evidence["relay_architecture"] = "aarch64" + evidence_path.write_text(json.dumps(evidence), encoding="utf-8") + try: + module.validate_offline_evidence(evidence_path, "x86_64", relay_wheel, bundle, plugin_template) + except ValueError: + pass + else: + raise AssertionError("offline evidence accepted a changed architecture") diff --git a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py index 58c166c6e..248630b10 100644 --- a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py @@ -31,6 +31,24 @@ def test_harbor_018_numeric_reward_is_normalized() -> None: assert module.read_benchmark_passed({"verifier_result": {"rewards": {"reward": 1.0}}}) is True +def test_harbor_timeout_multipliers_are_validated(tmp_path: Path) -> None: + module = load_validator() + config = { + "agent_timeout_multiplier": 3.0, + "agent_setup_timeout_multiplier": 6.0, + "environment_build_timeout_multiplier": 6.0, + } + (tmp_path / "config.json").write_text(json.dumps(config), encoding="utf-8") + observed, errors = module.validate_harbor_job_config(tmp_path) + assert observed == config + assert errors == [] + + config["agent_timeout_multiplier"] = 1.0 + (tmp_path / "config.json").write_text(json.dumps(config), encoding="utf-8") + _, errors = module.validate_harbor_job_config(tmp_path) + assert errors == ["Harbor job config agent_timeout_multiplier=1.0 does not match required 3.0"] + + def test_atof_reader_extracts_switchyard_selected_targets(tmp_path: Path) -> None: module = load_validator() path = tmp_path / "trajectory.atof.jsonl" @@ -53,6 +71,37 @@ def test_atof_reader_extracts_switchyard_selected_targets(tmp_path: Path) -> Non assert targets == ["strong", "weak"] +def test_atof_inspection_extracts_provider_models_and_cache_usage(tmp_path: Path) -> None: + module = load_validator() + path = tmp_path / "trajectory.atof.jsonl" + events = [ + { + "name": "switchyard.routing.decision", + "data": {"selected_target": "weak"}, + }, + { + "name": "openai.chat_completions", + "category_profile": {"annotated_response": {"model": "aws/anthropic/bedrock-claude-sonnet-4-6"}}, + "data": {"model": "aws/anthropic/bedrock-claude-sonnet-4-6"}, + }, + { + "name": "llm.chunk", + "data": {"usage": {"cache_read_tokens": 120, "cache_write_tokens": 30}}, + }, + ] + path.write_text("\n".join(json.dumps(event) for event in events) + "\n") + evidence = module.inspect_atof(path) + assert evidence == { + "count": 3, + "marks": ["switchyard.routing.decision"], + "models": ["aws/anthropic/bedrock-claude-sonnet-4-6"], + "targets": ["weak"], + "decision_count": 1, + "cache_read_tokens": 120, + "cache_write_tokens": 30, + } + + def test_secret_scan_finds_raw_key_within_artifact(tmp_path: Path) -> None: module = load_validator() artifact = tmp_path / "artifact.log" @@ -60,3 +109,82 @@ def test_secret_scan_finds_raw_key_within_artifact(tmp_path: Path) -> None: assert module.scan_secrets([artifact], [b"Bearer raw-provider-key", b"raw-provider-key"]) == [ "artifact.log:secret[1]" ] + + +def test_receipt_provenance_validation_covers_every_staged_digest() -> None: + module = load_validator() + provenance = { + "nemo_relay": {"wheel_sha256": "relay"}, + "hermes": {"commit": "hermes"}, + "switchyard": { + "commit": "switchyard", + "manifest_sha256": "manifest", + "library_sha256": "library", + }, + "relay_config_sha256": "config", + } + receipt = { + "activation_mode": "relay_standard_dynamic", + "dynamic_plugin_ids": ["nvidia.switchyard"], + "relay_config_sha256": "config", + "dependencies": { + "nemo_relay": {"version": "0.7.0", "wheel_sha256": "relay"}, + "hermes": {"commit": "hermes"}, + "switchyard": { + "commit": "switchyard", + "manifest_sha256": "manifest", + "library_sha256": "library", + }, + }, + "routing_contract": { + "relay_outer_lifecycle": True, + "execution_intercept_owner": "nvidia.switchyard", + "provider_http_client_owner": "switchyard-llm-client", + "separate_switchyard_service": False, + }, + "cleanup": {"plugin_host_closed": True, "exporters_flushed": True}, + } + assert module.validate_receipt_provenance(receipt, provenance) == [] + receipt["dependencies"]["switchyard"]["library_sha256"] = "tampered" + assert module.validate_receipt_provenance(receipt, provenance) == [ + "Switchyard library digest does not match runtime provenance" + ] + + +def test_openinference_inspection_extracts_semantic_and_lineage_evidence(tmp_path: Path) -> None: + module = load_validator() + artifact = tmp_path / "openinference.jsonl" + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "openinference.project.name", "value": {"stringValue": "project"}}, + {"key": "evaluation.cohort", "value": {"stringValue": "cohort"}}, + ] + }, + "scopeSpans": [ + { + "scope": {"name": "harbor-hermes-switchyard"}, + "spans": [ + { + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "LLM"}}, + {"key": "nemo_relay.uuid", "value": {"stringValue": "uuid"}}, + {"key": "nemo_relay.scope_type", "value": {"stringValue": "llm"}}, + ] + } + ], + } + ], + } + ] + } + artifact.write_text(json.dumps(payload) + "\n", encoding="utf-8") + evidence = module.inspect_openinference(artifact) + assert evidence["documents"] == 1 + assert evidence["spans"] == 1 + assert evidence["span_kinds"] == ["LLM"] + assert evidence["scope_names"] == ["harbor-hermes-switchyard"] + assert evidence["lineage_spans"] == 1 + assert evidence["resource_attributes"]["openinference.project.name"] == ["project"] From 33370a199a271c6726b59884df1cf40aa92262e4 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 6 Aug 2026 08:37:22 -0600 Subject: [PATCH 06/34] docs(examples): organize Harbor runbook steps --- examples/harbor-hermes-switchyard/README.md | 30 ++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 808de23c0..852cf1386 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -3,10 +3,10 @@ This example runs one complete Terminal-Bench 2.0 cohort through Harbor and Hermes. Hermes owns an in-process NeMo Relay 0.7.0 runtime; Relay loads the Switchyard native plugin, and Switchyard selects and calls the configured -provider route. Phase 2 is one resumable 89-task cohort. Multi-cohort execution -and result aggregation belong to Phase 3 and are intentionally out of scope. +provider route. It runs one resumable 89-task cohort. Multi-cohort execution +and result aggregation are intentionally out of scope for this example. -## Pinned inputs +## 1. Pinned inputs | Dependency | Input used by this example | |---|---| @@ -19,7 +19,7 @@ Every source checkout is detached and verified. The Hermes installer is followed by `uv sync --frozen`, then the selected Relay 0.7.0 wheel is force-installed without dependencies and verified by digest. -## Request and lifecycle ownership +## 2. Request and lifecycle ownership There is no Switchyard service in this topology: @@ -39,7 +39,7 @@ The adapter can be removed after is upstream and Harbor can install an immutable compatible Hermes revision while projecting the Relay configuration and plugin bundle. -## Configuration ownership +## 3. Configuration ownership The two configuration files have deliberately different responsibilities: @@ -72,7 +72,7 @@ unserved `openai/ollama-route-stub` identity and projects a dead local OpenAI endpoint. If Switchyard is bypassed, the request fails closed instead of reaching a provider. -## Host prerequisites +## 4. Host prerequisites - Linux or macOS, Bash, Python 3.11+, Docker, and `tmux`; - a local, immutable Terminal-Bench 2.0 dataset export containing 89 tasks; @@ -107,7 +107,7 @@ chmod 0600 /absolute/private/phase2-run.env The validator reports names and paths only. It rejects legacy secret-file variables and never renders or prints the authorization value. -## Phase 2 admission and runbook +## 5. Prepare and validate a cohort run Run every stage with the same immutable inputs. If the dataset, concurrency, architecture, Relay wheel, Switchyard library, plugin template, or Hermes @@ -124,7 +124,7 @@ set +a set +x ``` -### 1. All-89 no-token admission +## 6. Verify the complete dataset without provider tokens This loads and uniquely selects all tasks, hashes their instructions and verifiers, expands the complete Harbor job graph, denies registry/provider @@ -148,7 +148,7 @@ chmod 0700 "$PHASE2_ADMISSION_ROOT" The passed evidence binds task names, task/instruction/verifier hashes, concurrency, architecture, Relay wheel, Switchyard library, and plugin config. -### 2. Docker offline runtime admission +## 7. Verify the offline container runtime Prepare a fresh admission root with test-only structured overrides. Production model, URL, and routing values remain owned by `plugins.toml.in`; these flags @@ -184,7 +184,7 @@ observability, pinned-library loading, and clean shutdown. Its evidence binds the same Hermes commit, Relay wheel, Switchyard library, architecture, and plugin template consumed by the cohort. -### 3. Immutable plan +## 8. Create the immutable run plan The first command writes `plan.json`; any later invocation with different immutable inputs is refused. Choose concurrency before this point. @@ -197,7 +197,7 @@ immutable inputs is refused. Choose concurrency before this point. and upload result opens the parallel lane even when its benchmark reward is a non-pass. -### 4. Capacity and network preflight +## 9. Check capacity and provider availability ```bash "$EXAMPLE_ROOT/run_phase2_cohort.sh" "$PHASE2_RUN_ROOT" --preflight-only @@ -218,7 +218,7 @@ It also rejects less than the configured free-disk minimum (100G by default), concurrency above Docker's CPU count, and an architecture mismatch. The defaults are a 2G parallel lane and 4G Docker reserve. -### 5. Durable launch under tmux +## 10. Launch and resume the cohort Exit the secret-bearing admission shell first. From a shell where the protected file has **not** been sourced, start one detached supervisor. Only the file path @@ -264,7 +264,7 @@ Agent setup also retries transient `apt-get` failures three times locally; exhausted package-manager failures are classified as infrastructure and remain subject to the cohort's bounded retry limit. -## Completion gates +## 11. Completion gates A task is complete only when both its `validation.json` and `phoenix-upload.json` have `status=passed`. A benchmark @@ -279,5 +279,5 @@ The cohort passes only when: - `summary.json.status` is `passed`. `report.md` is regenerated after each completed attempt and is safe for -progress review. Phase 3 multiple-run orchestration and aggregated reports are -not part of this runbook. +progress review. Running multiple cohorts and aggregating their reports are not +part of this runbook. From 85c101112006b941f6fe80e69b26d9dd118f7c67 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 6 Aug 2026 08:40:48 -0600 Subject: [PATCH 07/34] refactor(examples): rename cohort environment interface --- examples/harbor-hermes-switchyard/.gitignore | 2 +- examples/harbor-hermes-switchyard/README.md | 30 +++++++++---------- .../run_phase2_cohort.sh | 6 ++-- .../scripts/launch_phase2_tmux.sh | 4 +-- .../scripts/run_phase2_from_env.sh | 12 ++++---- .../scripts/validate_phase2_environment.sh | 12 ++++---- .../supervise_phase2_cohort.sh | 4 +-- ...example => terminal-bench-run.env.example} | 10 +++---- .../tests/test_config_contract.py | 4 +-- .../tests/test_phase2_cohort.py | 2 +- 10 files changed, 43 insertions(+), 43 deletions(-) rename examples/harbor-hermes-switchyard/{phase2-run.env.example => terminal-bench-run.env.example} (78%) diff --git a/examples/harbor-hermes-switchyard/.gitignore b/examples/harbor-hermes-switchyard/.gitignore index bedbb2773..66d960511 100644 --- a/examples/harbor-hermes-switchyard/.gitignore +++ b/examples/harbor-hermes-switchyard/.gitignore @@ -1,2 +1,2 @@ -phase2-run.env +terminal-bench-run.env phase2-run.*.env diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 852cf1386..774018484 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -43,8 +43,8 @@ while projecting the Relay configuration and plugin bundle. The two configuration files have deliberately different responsibilities: -- `phase2-run.env.example` is copied to an untracked, mode-`0600` - `phase2-run.env`. It contains per-machine paths, the run identity, Phoenix +- `terminal-bench-run.env.example` is copied to an untracked, mode-`0600` + `terminal-bench-run.env`. It contains per-machine paths, the run identity, Phoenix destination, manually selected capacity, and the real `SWITCHYARD_PROVIDER_AUTHORIZATION` header. - `config/plugins.toml.in` is checked in and non-secret. It is the only source @@ -99,9 +99,9 @@ placeholder, including the complete provider Authorization header. Do not source this file into the interactive shell used to start `tmux`. ```bash -cp phase2-run.env.example /absolute/private/phase2-run.env -chmod 0600 /absolute/private/phase2-run.env -./scripts/validate_phase2_environment.sh /absolute/private/phase2-run.env +cp terminal-bench-run.env.example /absolute/private/terminal-bench-run.env +chmod 0600 /absolute/private/terminal-bench-run.env +./scripts/validate_phase2_environment.sh /absolute/private/terminal-bench-run.env ``` The validator reports names and paths only. It rejects legacy secret-file @@ -119,7 +119,7 @@ For the commands below, enter a short-lived shell with tracing disabled: ```bash set +x set -a -source /absolute/private/phase2-run.env +source /absolute/private/terminal-bench-run.env set +a set +x ``` @@ -131,8 +131,8 @@ verifiers, expands the complete Harbor job graph, denies registry/provider access, and renders the runtime. It starts neither Docker nor an agent. ```bash -mkdir -p "$PHASE2_ADMISSION_ROOT" -chmod 0700 "$PHASE2_ADMISSION_ROOT" +mkdir -p "$TERMINAL_BENCH_ADMISSION_ROOT" +chmod 0700 "$TERMINAL_BENCH_ADMISSION_ROOT" "$EVAL_PYTHON" "$EXAMPLE_ROOT/scripts/smoke_phase2_dataset.py" \ --dataset-root "$TBENCH_DATASET_PATH" \ --expected-count 89 \ @@ -142,7 +142,7 @@ chmod 0700 "$PHASE2_ADMISSION_ROOT" --relay-wheel "$RELAY_WHEEL" \ --relay-architecture "$RELAY_ARCHITECTURE" \ --plugin-config-template "$PLUGIN_CONFIG_TEMPLATE" \ - --output "$PHASE2_SMOKE_EVIDENCE" + --output "$TERMINAL_BENCH_SMOKE_EVIDENCE" ``` The passed evidence binds task names, task/instruction/verifier hashes, @@ -155,7 +155,7 @@ model, URL, and routing values remain owned by `plugins.toml.in`; these flags exist only to point this closed offline test at its fake endpoints. ```bash -OFFLINE_ROOT="$PHASE2_ADMISSION_ROOT/offline-runtime" +OFFLINE_ROOT="$TERMINAL_BENCH_ADMISSION_ROOT/offline-runtime" "$EVAL_PYTHON" "$EXAMPLE_ROOT/scripts/prepare_runtime.py" \ --run-root "$OFFLINE_ROOT" \ --switchyard-bundle "$SWITCHYARD_BUNDLE" \ @@ -175,7 +175,7 @@ case "$RELAY_ARCHITECTURE" in *) echo "unsupported architecture" >&2; return 2 ;; esac "$EXAMPLE_ROOT/scripts/run_offline_compatibility_smoke.sh" \ - "$OFFLINE_ROOT" "$PHASE2_OFFLINE_EVIDENCE" + "$OFFLINE_ROOT" "$TERMINAL_BENCH_OFFLINE_EVIDENCE" ``` This performs real Hermes→Relay→Switchyard calls against local fake provider @@ -190,7 +190,7 @@ The first command writes `plan.json`; any later invocation with different immutable inputs is refused. Choose concurrency before this point. ```bash -"$EXAMPLE_ROOT/run_phase2_cohort.sh" "$PHASE2_RUN_ROOT" --plan-only +"$EXAMPLE_ROOT/run_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" --plan-only ``` `adaptive-rejection-sampler` is always first and serial. A passed validation @@ -200,7 +200,7 @@ non-pass. ## 9. Check capacity and provider availability ```bash -"$EXAMPLE_ROOT/run_phase2_cohort.sh" "$PHASE2_RUN_ROOT" --preflight-only +"$EXAMPLE_ROOT/run_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" --preflight-only ``` Preflight authenticates to each configured provider's model catalog and @@ -228,7 +228,7 @@ disabled and persists output below the run root. ```bash exit # only when returning from the short-lived admission shell above ./scripts/launch_phase2_tmux.sh \ - /absolute/private/phase2-run.env \ + /absolute/private/terminal-bench-run.env \ harbor-hermes-switchyard-phase2-run-1 ``` @@ -251,7 +251,7 @@ tmux send-keys -t harbor-hermes-switchyard-phase2-run-1 C-c # After the old session exits, resume the same immutable root. ./scripts/launch_phase2_tmux.sh \ - /absolute/private/phase2-run.env \ + /absolute/private/terminal-bench-run.env \ harbor-hermes-switchyard-phase2-run-1 ``` diff --git a/examples/harbor-hermes-switchyard/run_phase2_cohort.sh b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh index ebf4441dd..0afa1225e 100755 --- a/examples/harbor-hermes-switchyard/run_phase2_cohort.sh +++ b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh @@ -5,7 +5,7 @@ set -euo pipefail example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -run_root="${1:-${PHASE2_RUN_ROOT:-}}" +run_root="${1:-${TERMINAL_BENCH_RUN_ROOT:-}}" if [[ -z "$run_root" || "$run_root" != /* ]]; then echo "usage: $0 /absolute/phase2-run-root [coordinator options]" >&2 exit 2 @@ -18,8 +18,8 @@ dataset_export_root="${TBENCH_DATASET_EXPORT_ROOT:-$(dirname "$run_root")/harbor dataset_root="${TBENCH_DATASET_PATH:-$dataset_export_root/$dataset_name}" harbor_bin="${HARBOR_BIN:-$example_root/.venv/bin/harbor}" python_bin="${EVAL_PYTHON:-$example_root/.venv/bin/python}" -smoke_evidence="${PHASE2_SMOKE_EVIDENCE:-}" -offline_evidence="${PHASE2_OFFLINE_EVIDENCE:-}" +smoke_evidence="${TERMINAL_BENCH_SMOKE_EVIDENCE:-}" +offline_evidence="${TERMINAL_BENCH_OFFLINE_EVIDENCE:-}" phoenix_url="${PHOENIX_BASE_URL:-}" phoenix_project="${PHOENIX_PROJECT:-}" eval_cohort="${EVAL_COHORT:-}" diff --git a/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh index 072907a6c..caef0d389 100755 --- a/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh +++ b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh @@ -8,7 +8,7 @@ example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" env_file="${1:-}" session="${2:-}" if [[ -z "$env_file" || "$env_file" != /* || -z "$session" ]]; then - echo "usage: $0 /absolute/phase2-run.env tmux-session-name" >&2 + echo "usage: $0 /absolute/terminal-bench-run.env tmux-session-name" >&2 exit 2 fi if [[ ! "$session" =~ ^[A-Za-z0-9_.-]+$ ]]; then @@ -25,6 +25,6 @@ fi # The caller must not source the protected file. Only its path is projected # into the detached session; run_phase2_from_env.sh sources it with xtrace off. tmux new-session -d -s "$session" \ - -e "PHASE2_ENV_FILE=$env_file" \ + -e "TERMINAL_BENCH_ENV_FILE=$env_file" \ "$example_root/scripts/run_phase2_from_env.sh" echo "started tmux session: $session" diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh index 39ccb5407..ba1f5a41b 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh @@ -6,9 +6,9 @@ set -euo pipefail set +x example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -env_file="${PHASE2_ENV_FILE:-${1:-}}" +env_file="${TERMINAL_BENCH_ENV_FILE:-${1:-}}" if [[ -z "$env_file" || "$env_file" != /* ]]; then - echo "PHASE2_ENV_FILE must be an absolute path" >&2 + echo "TERMINAL_BENCH_ENV_FILE must be an absolute path" >&2 exit 2 fi @@ -19,7 +19,7 @@ source "$env_file" set +a set +x -mkdir -p "$PHASE2_RUN_ROOT" -chmod 0700 "$PHASE2_RUN_ROOT" -exec "$example_root/supervise_phase2_cohort.sh" "$PHASE2_RUN_ROOT" \ - >>"$PHASE2_RUN_ROOT/supervisor.log" 2>&1 +mkdir -p "$TERMINAL_BENCH_RUN_ROOT" +chmod 0700 "$TERMINAL_BENCH_RUN_ROOT" +exec "$example_root/supervise_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" \ + >>"$TERMINAL_BENCH_RUN_ROOT/supervisor.log" 2>&1 diff --git a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh index 901b5f2d1..56408ec52 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh +++ b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh @@ -7,7 +7,7 @@ set +x env_file="${1:-}" if [[ -z "$env_file" || "$env_file" != /* || ! -f "$env_file" ]]; then - echo "usage: $0 /absolute/phase2-run.env" >&2 + echo "usage: $0 /absolute/terminal-bench-run.env" >&2 exit 2 fi if mode="$(stat -f '%Lp' "$env_file" 2>/dev/null)"; then @@ -30,10 +30,10 @@ source "$env_file" set +a required_values=( - EXAMPLE_ROOT PHASE2_RUN_ID PHASE2_RUN_ROOT PHASE2_ADMISSION_ROOT + EXAMPLE_ROOT TERMINAL_BENCH_RUN_ID TERMINAL_BENCH_RUN_ROOT TERMINAL_BENCH_ADMISSION_ROOT HARBOR_BIN EVAL_PYTHON TBENCH_DATASET_PATH SWITCHYARD_BUNDLE RELAY_WHEEL - RELAY_ARCHITECTURE PLUGIN_CONFIG_TEMPLATE PHASE2_SMOKE_EVIDENCE - PHASE2_OFFLINE_EVIDENCE PHOENIX_BASE_URL PHOENIX_PROJECT EVAL_COHORT + RELAY_ARCHITECTURE PLUGIN_CONFIG_TEMPLATE TERMINAL_BENCH_SMOKE_EVIDENCE + TERMINAL_BENCH_OFFLINE_EVIDENCE PHOENIX_BASE_URL PHOENIX_PROJECT EVAL_COHORT TBENCH_SAMPLE_COUNT TBENCH_CANARY_TASK TBENCH_CONCURRENCY TBENCH_PARALLEL_MAX_MEMORY_GB TBENCH_DOCKER_MEMORY_RESERVE_GB TBENCH_MINIMUM_FREE_GB SWITCHYARD_PROVIDER_AUTHORIZATION @@ -44,9 +44,9 @@ for name in "${required_values[@]}"; do exit 2 fi done -for name in EXAMPLE_ROOT PHASE2_RUN_ROOT PHASE2_ADMISSION_ROOT HARBOR_BIN EVAL_PYTHON \ +for name in EXAMPLE_ROOT TERMINAL_BENCH_RUN_ROOT TERMINAL_BENCH_ADMISSION_ROOT HARBOR_BIN EVAL_PYTHON \ TBENCH_DATASET_PATH SWITCHYARD_BUNDLE RELAY_WHEEL PLUGIN_CONFIG_TEMPLATE \ - PHASE2_SMOKE_EVIDENCE PHASE2_OFFLINE_EVIDENCE; do + TERMINAL_BENCH_SMOKE_EVIDENCE TERMINAL_BENCH_OFFLINE_EVIDENCE; do if [[ "${!name}" != /* ]]; then echo "Phase 2 path must be absolute: $name" >&2 exit 2 diff --git a/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh b/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh index d2aeb308a..7c9dbdae6 100755 --- a/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh +++ b/examples/harbor-hermes-switchyard/supervise_phase2_cohort.sh @@ -33,8 +33,8 @@ terminate() { } trap terminate INT TERM -backoff_seconds="${PHASE2_SUPERVISOR_BACKOFF_SECONDS:-60}" -maximum_backoff_seconds="${PHASE2_SUPERVISOR_MAX_BACKOFF_SECONDS:-900}" +backoff_seconds="${TERMINAL_BENCH_SUPERVISOR_BACKOFF_SECONDS:-60}" +maximum_backoff_seconds="${TERMINAL_BENCH_SUPERVISOR_MAX_BACKOFF_SECONDS:-900}" if [[ ! "$backoff_seconds" =~ ^[1-9][0-9]*$ || ! "$maximum_backoff_seconds" =~ ^[1-9][0-9]*$ ]]; then echo "Phase 2 supervisor backoffs must be positive integers" >&2 exit 2 diff --git a/examples/harbor-hermes-switchyard/phase2-run.env.example b/examples/harbor-hermes-switchyard/terminal-bench-run.env.example similarity index 78% rename from examples/harbor-hermes-switchyard/phase2-run.env.example rename to examples/harbor-hermes-switchyard/terminal-bench-run.env.example index 8b0596d68..305b5424d 100644 --- a/examples/harbor-hermes-switchyard/phase2-run.env.example +++ b/examples/harbor-hermes-switchyard/terminal-bench-run.env.example @@ -4,9 +4,9 @@ # Copy this file outside the checkout, chmod 0600, and replace every /absolute # placeholder. Never commit the populated file. EXAMPLE_ROOT=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard -PHASE2_RUN_ID=harbor-hermes-switchyard-phase2-run-1 -PHASE2_RUN_ROOT=/absolute/path/to/phase2-runs/harbor-hermes-switchyard-phase2-run-1 -PHASE2_ADMISSION_ROOT=/absolute/path/to/phase2-admission +TERMINAL_BENCH_RUN_ID=harbor-hermes-switchyard-phase2-run-1 +TERMINAL_BENCH_RUN_ROOT=/absolute/path/to/phase2-runs/harbor-hermes-switchyard-phase2-run-1 +TERMINAL_BENCH_ADMISSION_ROOT=/absolute/path/to/phase2-admission HARBOR_BIN=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/harbor EVAL_PYTHON=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/python @@ -16,8 +16,8 @@ RELAY_WHEEL=/absolute/path/to/nemo_relay-0.7.0-platform-wheel.whl RELAY_ARCHITECTURE=x86_64 PLUGIN_CONFIG_TEMPLATE=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/config/plugins.toml.in -PHASE2_SMOKE_EVIDENCE=/absolute/path/to/phase2-admission/all-89-smoke.json -PHASE2_OFFLINE_EVIDENCE=/absolute/path/to/phase2-admission/offline-admission.json +TERMINAL_BENCH_SMOKE_EVIDENCE=/absolute/path/to/phase2-admission/all-89-smoke.json +TERMINAL_BENCH_OFFLINE_EVIDENCE=/absolute/path/to/phase2-admission/offline-admission.json PHOENIX_BASE_URL=https://your-phoenix-endpoint PHOENIX_PROJECT=harbor-hermes-switchyard-phase2-run-1 EVAL_COHORT=harbor-hermes-switchyard-phase2-run-1 diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py index c55479d4e..176e7efa7 100644 --- a/examples/harbor-hermes-switchyard/tests/test_config_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_config_contract.py @@ -171,7 +171,7 @@ def test_agent_install_retries_transient_apt_failures() -> None: def test_phase2_environment_template_consolidates_secret_without_legacy_file() -> None: - template = (EXAMPLE_ROOT / "phase2-run.env.example").read_text(encoding="utf-8") + template = (EXAMPLE_ROOT / "terminal-bench-run.env.example").read_text(encoding="utf-8") assert "SWITCHYARD_PROVIDER_AUTHORIZATION='Bearer replace-with-provider-token'" in template assert "INFERENCE_SECRETS_FILE" not in template assert "NV_INFERENCEHUB_KEY" not in template @@ -179,7 +179,7 @@ def test_phase2_environment_template_consolidates_secret_without_legacy_file() - assert "STRONG_MODEL" not in template assert "WEAK_MODEL" not in template assert "UPSTREAM_BASE_URL" not in template - assert "phase2-run.env" in (EXAMPLE_ROOT / ".gitignore").read_text(encoding="utf-8") + assert "terminal-bench-run.env" in (EXAMPLE_ROOT / ".gitignore").read_text(encoding="utf-8") def test_phase2_readme_uses_admissions_instead_of_phase1_regressions() -> None: diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index aa25f1c59..883660728 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -286,7 +286,7 @@ def test_durable_supervisor_owns_the_coordinator_process_group() -> None: def test_tmux_launcher_projects_only_the_protected_file_path() -> None: launcher = (EXAMPLE_ROOT / "scripts" / "launch_phase2_tmux.sh").read_text(encoding="utf-8") child = (EXAMPLE_ROOT / "scripts" / "run_phase2_from_env.sh").read_text(encoding="utf-8") - assert '-e "PHASE2_ENV_FILE=$env_file"' in launcher + assert '-e "TERMINAL_BENCH_ENV_FILE=$env_file"' in launcher assert 'source "$env_file"' not in launcher assert "tmux has-session" in launcher assert 'source "$env_file"' in child From 75cc6862080ddfda90bab3b312e9cfb5299a1db8 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 6 Aug 2026 08:43:46 -0600 Subject: [PATCH 08/34] refactor(examples): use standard environment template name --- ...rminal-bench-run.env.example => .env.example} | 0 examples/harbor-hermes-switchyard/.gitignore | 3 +-- examples/harbor-hermes-switchyard/README.md | 16 ++++++++-------- .../scripts/launch_phase2_tmux.sh | 2 +- .../scripts/validate_phase2_environment.sh | 2 +- .../tests/test_config_contract.py | 4 ++-- 6 files changed, 13 insertions(+), 14 deletions(-) rename examples/harbor-hermes-switchyard/{terminal-bench-run.env.example => .env.example} (100%) diff --git a/examples/harbor-hermes-switchyard/terminal-bench-run.env.example b/examples/harbor-hermes-switchyard/.env.example similarity index 100% rename from examples/harbor-hermes-switchyard/terminal-bench-run.env.example rename to examples/harbor-hermes-switchyard/.env.example diff --git a/examples/harbor-hermes-switchyard/.gitignore b/examples/harbor-hermes-switchyard/.gitignore index 66d960511..4c49bd78f 100644 --- a/examples/harbor-hermes-switchyard/.gitignore +++ b/examples/harbor-hermes-switchyard/.gitignore @@ -1,2 +1 @@ -terminal-bench-run.env -phase2-run.*.env +.env diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 774018484..2ae4bf38d 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -43,8 +43,8 @@ while projecting the Relay configuration and plugin bundle. The two configuration files have deliberately different responsibilities: -- `terminal-bench-run.env.example` is copied to an untracked, mode-`0600` - `terminal-bench-run.env`. It contains per-machine paths, the run identity, Phoenix +- `.env.example` is copied to an untracked, mode-`0600` + `.env`. It contains per-machine paths, the run identity, Phoenix destination, manually selected capacity, and the real `SWITCHYARD_PROVIDER_AUTHORIZATION` header. - `config/plugins.toml.in` is checked in and non-secret. It is the only source @@ -99,9 +99,9 @@ placeholder, including the complete provider Authorization header. Do not source this file into the interactive shell used to start `tmux`. ```bash -cp terminal-bench-run.env.example /absolute/private/terminal-bench-run.env -chmod 0600 /absolute/private/terminal-bench-run.env -./scripts/validate_phase2_environment.sh /absolute/private/terminal-bench-run.env +cp .env.example /absolute/private/.env +chmod 0600 /absolute/private/.env +./scripts/validate_phase2_environment.sh /absolute/private/.env ``` The validator reports names and paths only. It rejects legacy secret-file @@ -119,7 +119,7 @@ For the commands below, enter a short-lived shell with tracing disabled: ```bash set +x set -a -source /absolute/private/terminal-bench-run.env +source /absolute/private/.env set +a set +x ``` @@ -228,7 +228,7 @@ disabled and persists output below the run root. ```bash exit # only when returning from the short-lived admission shell above ./scripts/launch_phase2_tmux.sh \ - /absolute/private/terminal-bench-run.env \ + /absolute/private/.env \ harbor-hermes-switchyard-phase2-run-1 ``` @@ -251,7 +251,7 @@ tmux send-keys -t harbor-hermes-switchyard-phase2-run-1 C-c # After the old session exits, resume the same immutable root. ./scripts/launch_phase2_tmux.sh \ - /absolute/private/terminal-bench-run.env \ + /absolute/private/.env \ harbor-hermes-switchyard-phase2-run-1 ``` diff --git a/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh index caef0d389..d9f22dcc4 100755 --- a/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh +++ b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh @@ -8,7 +8,7 @@ example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" env_file="${1:-}" session="${2:-}" if [[ -z "$env_file" || "$env_file" != /* || -z "$session" ]]; then - echo "usage: $0 /absolute/terminal-bench-run.env tmux-session-name" >&2 + echo "usage: $0 /absolute/.env tmux-session-name" >&2 exit 2 fi if [[ ! "$session" =~ ^[A-Za-z0-9_.-]+$ ]]; then diff --git a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh index 56408ec52..a2aed79fe 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh +++ b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh @@ -7,7 +7,7 @@ set +x env_file="${1:-}" if [[ -z "$env_file" || "$env_file" != /* || ! -f "$env_file" ]]; then - echo "usage: $0 /absolute/terminal-bench-run.env" >&2 + echo "usage: $0 /absolute/.env" >&2 exit 2 fi if mode="$(stat -f '%Lp' "$env_file" 2>/dev/null)"; then diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py index 176e7efa7..060c9c06d 100644 --- a/examples/harbor-hermes-switchyard/tests/test_config_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_config_contract.py @@ -171,7 +171,7 @@ def test_agent_install_retries_transient_apt_failures() -> None: def test_phase2_environment_template_consolidates_secret_without_legacy_file() -> None: - template = (EXAMPLE_ROOT / "terminal-bench-run.env.example").read_text(encoding="utf-8") + template = (EXAMPLE_ROOT / ".env.example").read_text(encoding="utf-8") assert "SWITCHYARD_PROVIDER_AUTHORIZATION='Bearer replace-with-provider-token'" in template assert "INFERENCE_SECRETS_FILE" not in template assert "NV_INFERENCEHUB_KEY" not in template @@ -179,7 +179,7 @@ def test_phase2_environment_template_consolidates_secret_without_legacy_file() - assert "STRONG_MODEL" not in template assert "WEAK_MODEL" not in template assert "UPSTREAM_BASE_URL" not in template - assert "terminal-bench-run.env" in (EXAMPLE_ROOT / ".gitignore").read_text(encoding="utf-8") + assert ".env" in (EXAMPLE_ROOT / ".gitignore").read_text(encoding="utf-8") def test_phase2_readme_uses_admissions_instead_of_phase1_regressions() -> None: From 24d1cb86efb60e2a7b86609cb08e59eddb0aaf28 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 6 Aug 2026 08:57:50 -0600 Subject: [PATCH 09/34] feat(examples): make cohort canary optional --- .../harbor-hermes-switchyard/.env.example | 2 ++ examples/harbor-hermes-switchyard/README.md | 20 +++++++++-- .../run_phase2_cohort.sh | 4 ++- .../scripts/run_phase2_cohort.py | 33 ++++++++++++------- .../tests/test_phase2_cohort.py | 11 +++++++ 5 files changed, 55 insertions(+), 15 deletions(-) diff --git a/examples/harbor-hermes-switchyard/.env.example b/examples/harbor-hermes-switchyard/.env.example index 305b5424d..ca0bae780 100644 --- a/examples/harbor-hermes-switchyard/.env.example +++ b/examples/harbor-hermes-switchyard/.env.example @@ -23,6 +23,8 @@ PHOENIX_PROJECT=harbor-hermes-switchyard-phase2-run-1 EVAL_COHORT=harbor-hermes-switchyard-phase2-run-1 TBENCH_SAMPLE_COUNT=89 +# Default: run this task alone before opening the full cohort lanes. +# Set an explicitly blank value to begin the full cohort immediately. TBENCH_CANARY_TASK=adaptive-rejection-sampler TBENCH_CONCURRENCY=4 TBENCH_PARALLEL_MAX_MEMORY_GB=2 diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 2ae4bf38d..5e80d8c84 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -193,9 +193,23 @@ immutable inputs is refused. Choose concurrency before this point. "$EXAMPLE_ROOT/run_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" --plan-only ``` -`adaptive-rejection-sampler` is always first and serial. A passed validation -and upload result opens the parallel lane even when its benchmark reward is a -non-pass. +**Optional canary-first scheduling.** + +The default `TBENCH_CANARY_TASK=adaptive-rejection-sampler` runs that one real +task first. A passed validation and upload result opens the parallel lane even +when its benchmark reward is a non-pass. This is a conservative production +check, not a separate command: launching the full cohort on a fresh run root +automatically starts the canary and then continues with the remaining tasks. + +To skip that one-task checkpoint, set an explicitly blank value in `.env`: + +```bash +TBENCH_CANARY_TASK= +``` + +With the canary disabled, the full cohort starts immediately. Task 1 remains +the first selected task, but it is scheduled in the normal parallel or serial +lane rather than running alone first. ## 9. Check capacity and provider availability diff --git a/examples/harbor-hermes-switchyard/run_phase2_cohort.sh b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh index 0afa1225e..c614e19fd 100755 --- a/examples/harbor-hermes-switchyard/run_phase2_cohort.sh +++ b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh @@ -28,7 +28,9 @@ relay_wheel="${RELAY_WHEEL:-}" relay_architecture="${RELAY_ARCHITECTURE:-x86_64}" plugin_config_template="${PLUGIN_CONFIG_TEMPLATE:-$example_root/config/plugins.toml.in}" sample_count="${TBENCH_SAMPLE_COUNT:-89}" -canary_task="${TBENCH_CANARY_TASK:-adaptive-rejection-sampler}" +# An explicitly blank value disables canary-first scheduling. An unset value +# keeps the conservative default. +canary_task="${TBENCH_CANARY_TASK-adaptive-rejection-sampler}" concurrency="${TBENCH_CONCURRENCY:-4}" parallel_memory_gb="${TBENCH_PARALLEL_MAX_MEMORY_GB:-2}" docker_memory_reserve_gb="${TBENCH_DOCKER_MEMORY_RESERVE_GB:-4}" diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index e50e56cde..e4421238a 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -105,7 +105,7 @@ def parse_memory_gb(value: object, task_name: str) -> int: return int(match.group(1)) -def discover_tasks(dataset_root: Path, sample_count: int, excluded: set[str], canary_task: str) -> list[Task]: +def discover_tasks(dataset_root: Path, sample_count: int, excluded: set[str], canary_task: str | None) -> list[Task]: if not dataset_root.is_dir(): raise ValueError(f"dataset root is not a directory: {dataset_root}") discovered: list[tuple[str, int]] = [] @@ -122,10 +122,13 @@ def discover_tasks(dataset_root: Path, sample_count: int, excluded: set[str], ca selected = discovered[:sample_count] if len(selected) != sample_count: raise ValueError(f"dataset contains {len(selected)} selectable tasks; requested {sample_count}") - canaries = [item for item in selected if item[0] == canary_task] - if len(canaries) != 1: - raise ValueError(f"canary task is not uniquely selectable: {canary_task}") - ordered = canaries + [item for item in selected if item[0] != canary_task] + if canary_task: + canaries = [item for item in selected if item[0] == canary_task] + if len(canaries) != 1: + raise ValueError(f"canary task is not uniquely selectable: {canary_task}") + ordered = canaries + [item for item in selected if item[0] != canary_task] + else: + ordered = selected return [Task(index, name, memory) for index, (name, memory) in enumerate(ordered, 1)] @@ -779,11 +782,14 @@ async def run(self) -> bool: preflight = shared_preflight(self.args, self.tasks) write_json(self.args.run_root / "preflight.json", {"status": "passed", **preflight}) await self.refresh_summary() - first = self.tasks[0] - if not await self.run_task(first): - return False - parallel = [task for task in self.tasks[1:] if task.memory_gb <= self.args.parallel_max_memory_gb] - serial = [task for task in self.tasks[1:] if task.memory_gb > self.args.parallel_max_memory_gb] + remaining = self.tasks + if self.args.canary_task: + first = self.tasks[0] + if not await self.run_task(first): + return False + remaining = self.tasks[1:] + parallel = [task for task in remaining if task.memory_gb <= self.args.parallel_max_memory_gb] + serial = [task for task in remaining if task.memory_gb > self.args.parallel_max_memory_gb] if not await self.run_parallel_lane(parallel): return False for task in serial: @@ -801,7 +807,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--dataset-root", type=Path, required=True) parser.add_argument("--sample-count", type=int, default=89) parser.add_argument("--exclude-task", action="append", default=[]) - parser.add_argument("--canary-task", default="adaptive-rejection-sampler") + parser.add_argument( + "--canary-task", + default="", + help="task to run alone before the cohort; pass an empty value to disable the canary", + ) parser.add_argument("--concurrency", type=int, default=4) parser.add_argument("--parallel-max-memory-gb", type=int, default=2) parser.add_argument("--docker-memory-reserve-gb", type=int, default=4) @@ -862,6 +872,7 @@ def main() -> int: args.switchyard_bundle, args.plugin_config_template, ) + args.canary_task = args.canary_task or None tasks = discover_tasks(args.dataset_root, args.sample_count, set(args.exclude_task), args.canary_task) args.run_root.mkdir(mode=0o700, parents=True, exist_ok=True) lock_path = args.run_root / ".phase2.lock" diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 883660728..584e47d28 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -93,6 +93,17 @@ def test_task_discovery_places_explicit_canary_before_lexical_lane(tmp_path: Pat ] +def test_task_discovery_without_a_canary_preserves_dataset_order(tmp_path: Path) -> None: + module = load_coordinator() + write_task(tmp_path, "task-z", "4G") + write_task(tmp_path, "task-a", "2G") + tasks = module.discover_tasks(tmp_path, 2, set(), None) + assert [(task.index, task.name, task.memory_gb) for task in tasks] == [ + (1, "task-a", 2), + (2, "task-z", 4), + ] + + def test_failed_attempt_is_preserved_and_passed_attempt_wins(tmp_path: Path) -> None: module = load_coordinator() task = module.Task(1, "task", 2) From 9dc0fb9243dd525f2dee4cc0e20339fa27dc5281 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 6 Aug 2026 09:04:20 -0600 Subject: [PATCH 10/34] chore(examples): remove obsolete smoke helpers --- .../run_terminal_bench.sh | 2 +- .../scripts/native_plugin_loader_smoke.py | 43 --------- ...egressions.sh => run_regression_smokes.sh} | 11 ++- .../scripts/validate_parallel_isolation.py | 93 ------------------- .../tests/test_config_contract.py | 4 +- .../tests/test_phase2_cohort.py | 55 ----------- 6 files changed, 9 insertions(+), 199 deletions(-) delete mode 100644 examples/harbor-hermes-switchyard/scripts/native_plugin_loader_smoke.py rename examples/harbor-hermes-switchyard/scripts/{run_phase1_regressions.sh => run_regression_smokes.sh} (86%) delete mode 100755 examples/harbor-hermes-switchyard/scripts/validate_parallel_isolation.py diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index 471b262e8..454fa236d 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -15,7 +15,7 @@ eval_cohort="${EVAL_COHORT:-harbor-hermes-switchyard-phase1}" default_harbor_bin="$example_root/.venv/bin/harbor" default_python_bin="$example_root/.venv/bin/python" harbor_bin="${HARBOR_BIN:-$default_harbor_bin}" -python_bin="${EVAL_PYTHON:-${PHASE1_PYTHON:-$default_python_bin}}" +python_bin="${EVAL_PYTHON:-$default_python_bin}" expected_harbor_version="0.18.0" eval_phase="${EVAL_PHASE:-phase1}" tbench_dataset_path="${TBENCH_DATASET_PATH:-}" diff --git a/examples/harbor-hermes-switchyard/scripts/native_plugin_loader_smoke.py b/examples/harbor-hermes-switchyard/scripts/native_plugin_loader_smoke.py deleted file mode 100644 index 9936fc202..000000000 --- a/examples/harbor-hermes-switchyard/scripts/native_plugin_loader_smoke.py +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Load and close the configured native plugin without starting Hermes.""" - -from __future__ import annotations - -import argparse -import asyncio -import json -from pathlib import Path - -from nemo_relay import plugin - - -async def exercise(config: Path) -> dict[str, object]: - specs = plugin.load_dynamic_plugin_activation_specs(config) - if len(specs) != 1 or specs[0].plugin_id != "nvidia.switchyard": - raise AssertionError("expected one nvidia.switchyard activation spec") - host = await plugin.initialize_with_dynamic_plugins( - {"version": 1, "components": []}, - specs, - ) - try: - report = host.report - if not host.is_active: - raise AssertionError("dynamic plugin host did not become active") - return report.to_dict() if hasattr(report, "to_dict") else report - finally: - await host.close() - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--plugins", type=Path, required=True) - args = parser.parse_args() - report = asyncio.run(exercise(args.plugins.resolve())) - print(json.dumps(report, indent=2, sort_keys=True)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase1_regressions.sh b/examples/harbor-hermes-switchyard/scripts/run_regression_smokes.sh similarity index 86% rename from examples/harbor-hermes-switchyard/scripts/run_phase1_regressions.sh rename to examples/harbor-hermes-switchyard/scripts/run_regression_smokes.sh index a9583f709..a904448c5 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase1_regressions.sh +++ b/examples/harbor-hermes-switchyard/scripts/run_regression_smokes.sh @@ -25,20 +25,21 @@ tasks=( ) for task in "${tasks[@]}"; do - echo "Running Phase 1 regression: $task" + echo "Running regression smoke: $task" run_root="$regression_root/$task" inject=false if [[ "$task" == "circuit-fibsqrt" ]]; then inject=true fi TASK_NAME="$task" \ - PHOENIX_PROJECT="${PHOENIX_PROJECT:-harbor-hermes-switchyard-phase1}-$task" \ - EVAL_COHORT="${EVAL_COHORT:-harbor-hermes-switchyard-phase1}-$task" \ + PHOENIX_PROJECT="${PHOENIX_PROJECT:-harbor-hermes-switchyard-regression}-$task" \ + EVAL_COHORT="${EVAL_COHORT:-harbor-hermes-switchyard-regression}-$task" \ + EVAL_PHASE="regression" \ INJECT_POST_RESPONSE_FAILURE="$inject" \ "$example_root/run_terminal_bench.sh" "$run_root" done -python_bin="${PHASE1_PYTHON:-python3}" +python_bin="${EVAL_PYTHON:-python3}" "$python_bin" - "$regression_root" "${tasks[@]}" <<'PY' import json import pathlib @@ -57,7 +58,7 @@ for task in tasks: summaries.append(summary) result = { - "schema_version": "harbor-hermes-switchyard.phase1-regressions.v1", + "schema_version": "harbor-hermes-switchyard.regression-smokes.v1", "status": "passed", "planned": len(tasks), "completed": len(summaries), diff --git a/examples/harbor-hermes-switchyard/scripts/validate_parallel_isolation.py b/examples/harbor-hermes-switchyard/scripts/validate_parallel_isolation.py deleted file mode 100755 index f0491185c..000000000 --- a/examples/harbor-hermes-switchyard/scripts/validate_parallel_isolation.py +++ /dev/null @@ -1,93 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Validate that concurrent Phase 1 tasks used isolated runtime state.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any - -SCHEMA_VERSION = "harbor-hermes-switchyard.parallel-isolation.v1" - - -def read_json(path: Path) -> dict[str, Any]: - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict): - raise ValueError(f"expected a JSON object: {path}") - return value - - -def validate(run_roots: list[Path]) -> dict[str, Any]: - if len(run_roots) < 2: - raise ValueError("parallel isolation requires at least two task roots") - records: list[dict[str, Any]] = [] - for root in run_roots: - resolved = root.resolve(strict=True) - summary = read_json(resolved / "summary.json") - if summary.get("status") != "passed": - raise ValueError(f"task summary did not pass: {resolved}") - artifacts = Path(summary.get("artifacts", "")).resolve(strict=True) - receipt = read_json(artifacts / "direct-hermes-receipt.json") - provenance = read_json(resolved / "runtime" / "provenance.json") - cleanup = receipt.get("cleanup") or {} - if not all(cleanup.get(key) is True for key in ("plugin_host_closed", "exporters_flushed")): - raise ValueError(f"task did not close plugin/exporter lifecycle: {resolved}") - records.append( - { - "task": summary.get("task_name"), - "run_root": str(resolved), - "artifact_root": str(artifacts), - "job_name": summary.get("job_name"), - "session_handle": receipt.get("session_handle"), - "relay_config_sha256": provenance.get("relay_config_sha256"), - "phoenix_project": provenance.get("phoenix_project"), - "evaluation_cohort": provenance.get("eval_cohort"), - "relay_wheel_sha256": provenance.get("nemo_relay", {}).get("wheel_sha256"), - "switchyard_library_sha256": provenance.get("switchyard", {}).get("library_sha256"), - } - ) - distinct_fields = ( - "run_root", - "artifact_root", - "job_name", - "session_handle", - "relay_config_sha256", - "phoenix_project", - "evaluation_cohort", - ) - for field in distinct_fields: - values = [record.get(field) for record in records] - if any(not value for value in values) or len(set(values)) != len(values): - raise ValueError(f"parallel tasks did not have distinct {field} values") - shared_fields = ("relay_wheel_sha256", "switchyard_library_sha256") - for field in shared_fields: - values = [record.get(field) for record in records] - if any(not value for value in values) or len(set(values)) != 1: - raise ValueError(f"parallel tasks did not use one immutable {field}") - return { - "schema_version": SCHEMA_VERSION, - "status": "passed", - "task_count": len(records), - "distinct_fields": list(distinct_fields), - "shared_input_fields": list(shared_fields), - "tasks": records, - } - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--run-root", type=Path, action="append", required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - result = validate(args.run_root) - args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") - print(json.dumps(result, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py index 060c9c06d..463700429 100644 --- a/examples/harbor-hermes-switchyard/tests/test_config_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_config_contract.py @@ -182,9 +182,9 @@ def test_phase2_environment_template_consolidates_secret_without_legacy_file() - assert ".env" in (EXAMPLE_ROOT / ".gitignore").read_text(encoding="utf-8") -def test_phase2_readme_uses_admissions_instead_of_phase1_regressions() -> None: +def test_readme_uses_admissions_instead_of_regression_smokes() -> None: readme = (EXAMPLE_ROOT / "README.md").read_text(encoding="utf-8") - assert "run_phase1_regressions.sh" not in readme + assert "run_regression_smokes.sh" not in readme assert "PHASE1_EVIDENCE_ROOT" not in readme assert "INFERENCE_SECRETS_FILE" not in readme assert "all-89 no-token admission" in readme.lower() diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 584e47d28..7ad06db3d 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -25,15 +25,6 @@ def load_coordinator(): return module -def load_isolation_validator(): - path = EXAMPLE_ROOT / "scripts" / "validate_parallel_isolation.py" - spec = importlib.util.spec_from_file_location("parallel_isolation_validator", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - def write_task(dataset: Path, name: str, memory: str) -> None: task = dataset / name task.mkdir(parents=True) @@ -241,52 +232,6 @@ def test_smoke_evidence_is_bound_to_exact_local_dataset(tmp_path: Path) -> None: raise AssertionError("smoke evidence accepted changed concurrency") -def test_parallel_isolation_requires_distinct_state_and_shared_inputs(tmp_path: Path) -> None: - module = load_isolation_validator() - roots = [] - for index, task in enumerate(("one", "two"), 1): - root = tmp_path / task - artifacts = root / "artifacts" - artifacts.mkdir(parents=True) - (root / "runtime").mkdir() - (root / "summary.json").write_text( - json.dumps( - { - "status": "passed", - "task_name": task, - "job_name": f"job-{task}", - "artifacts": str(artifacts), - } - ), - encoding="utf-8", - ) - (artifacts / "direct-hermes-receipt.json").write_text( - json.dumps( - { - "session_handle": f"session-{task}", - "cleanup": {"plugin_host_closed": True, "exporters_flushed": True}, - } - ), - encoding="utf-8", - ) - (root / "runtime" / "provenance.json").write_text( - json.dumps( - { - "relay_config_sha256": f"config-{index}", - "phoenix_project": f"project-{index}", - "eval_cohort": f"cohort-{index}", - "nemo_relay": {"wheel_sha256": "relay"}, - "switchyard": {"library_sha256": "switchyard"}, - } - ), - encoding="utf-8", - ) - roots.append(root) - result = module.validate(roots) - assert result["status"] == "passed" - assert result["task_count"] == 2 - - def test_durable_supervisor_owns_the_coordinator_process_group() -> None: supervisor = (EXAMPLE_ROOT / "supervise_phase2_cohort.sh").read_text(encoding="utf-8") assert "scripts/exec_process_group.py" in supervisor From 765961401b2968184684c4a2c8728d4a75dc43af Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 6 Aug 2026 09:23:05 -0600 Subject: [PATCH 11/34] fix(examples): snapshot phase 2 runtime --- examples/harbor-hermes-switchyard/README.md | 14 +- .../scripts/run_phase2_from_env.sh | 7 +- .../scripts/stage_phase2_runtime.py | 129 ++++++++++++++++++ .../tests/test_phase2_cohort.py | 29 ++++ 4 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 examples/harbor-hermes-switchyard/scripts/stage_phase2_runtime.py diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 5e80d8c84..64311b37b 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -126,7 +126,7 @@ set +x ## 6. Verify the complete dataset without provider tokens -This loads and uniquely selects all tasks, hashes their instructions and +This all-89 no-token admission loads and uniquely selects all tasks, hashes their instructions and verifiers, expands the complete Harbor job graph, denies registry/provider access, and renders the runtime. It starts neither Docker nor an agent. @@ -150,7 +150,8 @@ concurrency, architecture, Relay wheel, Switchyard library, and plugin config. ## 7. Verify the offline container runtime -Prepare a fresh admission root with test-only structured overrides. Production +The Docker offline runtime admission uses a fresh admission root with test-only +structured overrides. Production model, URL, and routing values remain owned by `plugins.toml.in`; these flags exist only to point this closed offline test at its fake endpoints. @@ -238,6 +239,10 @@ Exit the secret-bearing admission shell first. From a shell where the protected file has **not** been sourced, start one detached supervisor. Only the file path is placed in the tmux server environment; the child sources it with xtrace disabled and persists output below the run root. +Before the supervisor starts, the launcher copies only the plan-bound harness +sources into `runtime-harness/` below the run root and verifies their aggregate +hash. Retries execute this snapshot, so later checkout changes cannot alter an +active cohort. No environment file or secret is copied into the snapshot. ```bash exit # only when returning from the short-lived admission shell above @@ -263,8 +268,9 @@ jq '{status,completed_tasks,planned_tasks,benchmark_pass_count,benchmark_nonpass # Graceful interruption. tmux send-keys -t harbor-hermes-switchyard-phase2-run-1 C-c -# After the old session exits, resume the same immutable root. -./scripts/launch_phase2_tmux.sh \ +# After the old session exits, resume from the run-bound snapshot. This also +# works after a checkout update or host reboot. +/absolute/path/to/phase2-run-root/runtime-harness/scripts/launch_phase2_tmux.sh \ /absolute/private/.env \ harbor-hermes-switchyard-phase2-run-1 ``` diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh index ba1f5a41b..5e50e7bb9 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh @@ -21,5 +21,10 @@ set +x mkdir -p "$TERMINAL_BENCH_RUN_ROOT" chmod 0700 "$TERMINAL_BENCH_RUN_ROOT" -exec "$example_root/supervise_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" \ +runtime_harness="$TERMINAL_BENCH_RUN_ROOT/runtime-harness" +"$EVAL_PYTHON" "$example_root/scripts/stage_phase2_runtime.py" \ + --source "$example_root" \ + --destination "$runtime_harness" \ + --plan "$TERMINAL_BENCH_RUN_ROOT/plan.json" +exec "$runtime_harness/supervise_phase2_cohort.sh" "$TERMINAL_BENCH_RUN_ROOT" \ >>"$TERMINAL_BENCH_RUN_ROOT/supervisor.log" 2>&1 diff --git a/examples/harbor-hermes-switchyard/scripts/stage_phase2_runtime.py b/examples/harbor-hermes-switchyard/scripts/stage_phase2_runtime.py new file mode 100644 index 000000000..4bb70cf75 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/stage_phase2_runtime.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stage and verify the immutable Phase 2 runtime harness.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +from pathlib import Path + +RUNTIME_SUFFIXES = {".py", ".sh", ".toml", ".yaml"} +RUNTIME_TOP_LEVEL = ( + "run_terminal_bench.sh", + "run_phase2_cohort.sh", + "supervise_phase2_cohort.sh", +) +RUNTIME_DIRECTORIES = ("agents", "config", "scripts") + + +def runtime_files(root: Path) -> list[Path]: + files = [root / name for name in RUNTIME_TOP_LEVEL] + for relative in RUNTIME_DIRECTORIES: + files.extend( + path + for path in (root / relative).rglob("*") + if path.is_file() and path.suffix in RUNTIME_SUFFIXES + ) + missing = [path for path in files if not path.is_file()] + if missing: + raise FileNotFoundError(f"runtime source is missing: {missing[0]}") + return sorted(files) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def runtime_digest(root: Path, files: list[Path]) -> str: + digest = hashlib.sha256() + for path in sorted(files): + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(b"\0") + digest.update(file_sha256(path).encode()) + digest.update(b"\n") + return digest.hexdigest() + + +def expected_digest(plan_path: Path) -> str: + plan = json.loads(plan_path.read_text(encoding="utf-8")) + value = plan.get("inputs", {}).get("runtime_sources_sha256") + if not isinstance(value, str) or not value: + raise ValueError("plan does not contain inputs.runtime_sources_sha256") + return value + + +def verify_runtime(root: Path, expected: str) -> tuple[str, int]: + files = runtime_files(root) + observed = runtime_digest(root, files) + if observed != expected: + raise ValueError(f"runtime harness hash mismatch: expected {expected}, observed {observed}") + return observed, len(files) + + +def stage_runtime(source: Path, destination: Path, plan_path: Path) -> dict[str, object]: + source = source.resolve() + destination = destination.resolve() + expected = expected_digest(plan_path) + if destination.is_dir(): + observed, file_count = verify_runtime(destination, expected) + return {"status": "verified", "runtime_sources_sha256": observed, "file_count": file_count} + if destination.exists(): + raise ValueError(f"runtime harness destination is not a directory: {destination}") + + files = runtime_files(source) + observed = runtime_digest(source, files) + if observed != expected: + raise ValueError(f"live runtime differs from immutable plan: expected {expected}, observed {observed}") + + temporary = destination.with_name(f"{destination.name}.tmp-{os.getpid()}") + if temporary.exists(): + shutil.rmtree(temporary) + temporary.mkdir(mode=0o700, parents=False) + try: + for path in files: + target = temporary / path.relative_to(source) + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copy2(path, target) + (temporary / "snapshot.json").write_text( + json.dumps( + { + "schema_version": "harbor-hermes-switchyard.phase2-runtime-snapshot.v1", + "runtime_sources_sha256": observed, + "file_count": len(files), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + temporary.replace(destination) + except BaseException: + shutil.rmtree(temporary, ignore_errors=True) + raise + return {"status": "staged", "runtime_sources_sha256": observed, "file_count": len(files)} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--plan", type=Path, required=True) + args = parser.parse_args() + result = stage_runtime(args.source, args.destination, args.plan) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 7ad06db3d..774afdd1c 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -247,9 +247,38 @@ def test_tmux_launcher_projects_only_the_protected_file_path() -> None: assert "tmux has-session" in launcher assert 'source "$env_file"' in child assert "set +x" in child + assert "stage_phase2_runtime.py" in child + assert 'runtime_harness="$TERMINAL_BENCH_RUN_ROOT/runtime-harness"' in child + assert 'exec "$runtime_harness/supervise_phase2_cohort.sh"' in child assert "supervisor.log" in child +def test_runtime_snapshot_remains_bound_after_checkout_changes(tmp_path: Path) -> None: + import importlib.util + + script = EXAMPLE_ROOT / "scripts" / "stage_phase2_runtime.py" + spec = importlib.util.spec_from_file_location("stage_phase2_runtime", script) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + source = tmp_path / "source" + for relative in ("agents", "config", "scripts"): + (source / relative).mkdir(parents=True) + for relative in module.RUNTIME_TOP_LEVEL: + (source / relative).write_text(f"{relative}\n", encoding="utf-8") + (source / "scripts" / "helper.py").write_text("VALUE = 1\n", encoding="utf-8") + digest = module.runtime_digest(source, module.runtime_files(source)) + plan = tmp_path / "plan.json" + plan.write_text(json.dumps({"inputs": {"runtime_sources_sha256": digest}}), encoding="utf-8") + destination = tmp_path / "runtime-harness" + + assert module.stage_runtime(source, destination, plan)["status"] == "staged" + (source / "scripts" / "helper.py").write_text("VALUE = 2\n", encoding="utf-8") + assert module.stage_runtime(source, destination, plan)["status"] == "verified" + assert (destination / "scripts" / "helper.py").read_text(encoding="utf-8") == "VALUE = 1\n" + + def test_phase2_launcher_is_local_dataset_only() -> None: launcher = (EXAMPLE_ROOT / "run_phase2_cohort.sh").read_text(encoding="utf-8") assert 'dataset_root="${TBENCH_DATASET_PATH:-$dataset_export_root/$dataset_name}"' in launcher From bb427f9f1903a45d7e91d17f8410fe411f309b4f Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Thu, 6 Aug 2026 21:45:14 -0600 Subject: [PATCH 12/34] feat(example): harden Harbor phase 2 cohort setup Signed-off-by: Bryan Bednarski --- .../harbor-hermes-switchyard/.env.example | 7 + examples/harbor-hermes-switchyard/README.md | 2 +- .../agents/harbor_hermes_agent.py | 170 ++++- .../run_phase2_cohort.sh | 8 + .../run_terminal_bench.sh | 49 +- .../scripts/build_hermetic_runtime.py | 263 +++++++ .../scripts/build_switchyard_plugin.sh | 6 +- .../scripts/prepare_runtime.py | 2 +- .../scripts/run_phase2_cohort.py | 239 +++++++ .../scripts/run_regression_smokes.sh | 71 -- .../scripts/run_setup_admission.py | 649 ++++++++++++++++++ .../scripts/validate_phase2_environment.sh | 8 +- .../tests/test_agent_result_contract.py | 2 +- .../tests/test_config_contract.py | 199 ------ .../tests/test_phase2_cohort.py | 54 ++ .../tests/test_setup_admission.py | 338 +++++++++ 16 files changed, 1778 insertions(+), 289 deletions(-) create mode 100755 examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py delete mode 100755 examples/harbor-hermes-switchyard/scripts/run_regression_smokes.sh create mode 100755 examples/harbor-hermes-switchyard/scripts/run_setup_admission.py delete mode 100644 examples/harbor-hermes-switchyard/tests/test_config_contract.py create mode 100644 examples/harbor-hermes-switchyard/tests/test_setup_admission.py diff --git a/examples/harbor-hermes-switchyard/.env.example b/examples/harbor-hermes-switchyard/.env.example index ca0bae780..657142803 100644 --- a/examples/harbor-hermes-switchyard/.env.example +++ b/examples/harbor-hermes-switchyard/.env.example @@ -7,6 +7,9 @@ EXAMPLE_ROOT=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard TERMINAL_BENCH_RUN_ID=harbor-hermes-switchyard-phase2-run-1 TERMINAL_BENCH_RUN_ROOT=/absolute/path/to/phase2-runs/harbor-hermes-switchyard-phase2-run-1 TERMINAL_BENCH_ADMISSION_ROOT=/absolute/path/to/phase2-admission +# Automatically populated with a content-addressed Hermes runtime. Users do +# not pre-build task images or this runtime. +TERMINAL_BENCH_BOOTSTRAP_ROOT=/absolute/path/to/phase2-admission/bootstrap HARBOR_BIN=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/harbor EVAL_PYTHON=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/python @@ -27,6 +30,10 @@ TBENCH_SAMPLE_COUNT=89 # Set an explicitly blank value to begin the full cohort immediately. TBENCH_CANARY_TASK=adaptive-rejection-sampler TBENCH_CONCURRENCY=4 +# Environment preparation is intentionally quieter than provider execution. +TBENCH_SETUP_CONCURRENCY=2 +TBENCH_SETUP_BATCH_SIZE=89 +TBENCH_SETUP_MAX_INFRA_ATTEMPTS=4 TBENCH_PARALLEL_MAX_MEMORY_GB=2 TBENCH_DOCKER_MEMORY_RESERVE_GB=4 TBENCH_MINIMUM_FREE_GB=100 diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 64311b37b..48d81359b 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -12,7 +12,7 @@ and result aggregation are intentionally out of scope for this example. |---|---| | NeMo Relay | Released `nemo-relay==0.7.0` platform wheel, installed by digest rather than from this source checkout. | | Hermes | `bbednarski9/hermes-agent`, detached commit `efb63e714abc436af88af9b0d6734751c199aa6d` from PR #77915. | -| Switchyard | `bbednarski9/Switchyard`, detached commit `5d9d3292d6154e44d50295d0d4a3fd4f144f2528` from PR #270. | +| Switchyard | `bbednarski9/Switchyard`, detached commit `8daac03edf8544144833af1fd009b3da737715bc` from PR #270. | | Harbor | `harbor==0.18.0`, local export of dataset `terminal-bench@2.0`. | Every source checkout is detached and verified. The Hermes installer is diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py index 6f11aef58..9a83ac80f 100644 --- a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -12,6 +12,7 @@ from __future__ import annotations import hashlib +import json import re import shlex import time @@ -30,9 +31,15 @@ _DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" _DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" _DEFAULT_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" -_DEFAULT_SWITCHYARD_COMMIT = "5d9d3292d6154e44d50295d0d4a3fd4f144f2528" +_DEFAULT_SWITCHYARD_COMMIT = "8daac03edf8544144833af1fd009b3da737715bc" _ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]*") _PROVIDER_AUTHORIZATION_FILE = "/run/secrets/switchyard-provider-authorization" +_HERMETIC_RUNTIME_ROOT = "/opt/hermes-runtime" +_HERMETIC_RUNTIME_SCHEMA = "harbor-hermes-switchyard.hermetic-runtime.v1" +_HERMETIC_CA_BUNDLE_RELATIVE = Path("hermes-agent-src/venv/lib/python3.11/site-packages/certifi/cacert.pem") +_HERMETIC_CA_BUNDLE = f"{_HERMETIC_RUNTIME_ROOT}/{_HERMETIC_CA_BUNDLE_RELATIVE.as_posix()}" +_HERMETIC_RUNTIME_READY_ATTEMPTS = 6 +_HERMETIC_RUNTIME_READY_DELAY_SECONDS = 2 def _require_full_sha(value: str, name: str) -> str: @@ -57,6 +64,73 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _load_hermetic_runtime( + path: Path, + *, + expected_digest: str, + hermes_commit: str, + relay_wheel_sha256: str, + relay_architecture: str, +) -> dict[str, Any]: + marker = path / "payload.json" + if not marker.is_file(): + raise FileNotFoundError(marker) + payload = json.loads(marker.read_text(encoding="utf-8")) + expected = { + "schema_version": _HERMETIC_RUNTIME_SCHEMA, + "status": "passed", + "content_sha256": expected_digest, + "hermes_commit": hermes_commit, + "relay_wheel_sha256": relay_wheel_sha256, + "relay_architecture": relay_architecture, + } + mismatches = { + key: {"expected": value, "actual": payload.get(key)} + for key, value in expected.items() + if payload.get(key) != value + } + if mismatches: + raise ValueError(f"hermetic runtime metadata mismatch: {mismatches}") + required = ( + path / "bin" / "hermes", + path / "bin" / "python", + path / "bin" / "uv", + path / "hermes-agent-src" / "venv", + path / _HERMETIC_CA_BUNDLE_RELATIVE, + ) + missing = [str(candidate) for candidate in required if not candidate.exists()] + if missing: + raise FileNotFoundError(f"hermetic runtime is incomplete: {missing}") + return payload + + +def _hermetic_runtime_readiness_command( + runtime_root: str = _HERMETIC_RUNTIME_ROOT, + *, + attempts: int = _HERMETIC_RUNTIME_READY_ATTEMPTS, + delay_seconds: int = _HERMETIC_RUNTIME_READY_DELAY_SECONDS, +) -> str: + """Return a bounded probe that executes both nested runtime entrypoints.""" + if attempts < 1: + raise ValueError("attempts must be positive") + if delay_seconds < 0: + raise ValueError("delay_seconds cannot be negative") + runtime = shlex.quote(runtime_root) + attempt_numbers = " ".join(str(attempt) for attempt in range(1, attempts + 1)) + return ( + "runtime_ready=1; " + f"for attempt in {attempt_numbers}; do " + f'if {runtime}/bin/python -c "import importlib.metadata as m; ' + "assert m.version('nemo-relay') == '0.7.0'\" " + f"&& {runtime}/bin/hermes version; then " + "runtime_ready=0; break; " + "else runtime_ready=$?; fi; " + f'if [ "$attempt" -lt {attempts} ]; then sleep {delay_seconds}; fi; ' + "done; " + '[ "$runtime_ready" -eq 0 ] || exit "$runtime_ready"; ' + ) + + def _verify_elf_architecture(path: Path, architecture: str) -> None: with path.open("rb") as stream: header = stream.read(20) @@ -234,6 +308,8 @@ def __init__( switchyard_commit: str = _DEFAULT_SWITCHYARD_COMMIT, artifact_root: str = "/logs/agent/direct-hermes", inject_post_response_failure: bool = False, + hermetic_runtime_dir: str | None = None, + hermetic_runtime_sha256: str | None = None, **kwargs: Any, ) -> None: self.repository_url = _require_public_https_git_url(repository_url) @@ -252,6 +328,8 @@ def __init__( self.relay_wheel_path = Path(relay_wheel_path).expanduser().resolve() self.artifact_root = artifact_root.rstrip("/") self.inject_post_response_failure = inject_post_response_failure + self.hermetic_runtime_dir: Path | None = None + self.hermetic_runtime_sha256: str | None = None self._load_provider_authorization = False if not self.artifact_root.startswith("/logs/agent/"): raise ValueError("artifact_root must be an absolute child of /logs/agent") @@ -278,6 +356,21 @@ def __init__( self.switchyard_library = libraries[0] _verify_elf_architecture(self.switchyard_library, relay_architecture) + if (hermetic_runtime_dir is None) != (hermetic_runtime_sha256 is None): + raise ValueError("hermetic_runtime_dir and hermetic_runtime_sha256 must be supplied together") + if hermetic_runtime_dir is not None and hermetic_runtime_sha256 is not None: + runtime_dir = Path(hermetic_runtime_dir).expanduser().resolve() + runtime_digest = _require_sha256(hermetic_runtime_sha256, "hermetic_runtime_sha256") + _load_hermetic_runtime( + runtime_dir, + expected_digest=runtime_digest, + hermes_commit=self.commit, + relay_wheel_sha256=self.relay_wheel_sha256, + relay_architecture=self.relay_architecture, + ) + self.hermetic_runtime_dir = runtime_dir + self.hermetic_runtime_sha256 = runtime_digest + self._example_root = Path(__file__).resolve().parents[1] self._finalizer_path = self._example_root / "scripts" / "finalize_artifacts.py" if not self._finalizer_path.is_file(): @@ -296,6 +389,15 @@ async def exec_as_agent( cwd: str | None = None, timeout_sec: int | None = None, ) -> Any: + if self.hermetic_runtime_dir is not None: + ca_bundle = shlex.quote(_HERMETIC_CA_BUNDLE) + command = ( + f"test -r {ca_bundle}; " + f"export SSL_CERT_FILE={ca_bundle}; " + f"export REQUESTS_CA_BUNDLE={ca_bundle}; " + f"export CURL_CA_BUNDLE={ca_bundle}; " + f"{command}" + ) if self._load_provider_authorization: secret_file = shlex.quote(_PROVIDER_AUTHORIZATION_FILE) command = ( @@ -314,6 +416,31 @@ async def exec_as_agent( @override async def install(self, environment: BaseEnvironment) -> None: + if self.hermetic_runtime_dir is not None: + runtime = shlex.quote(_HERMETIC_RUNTIME_ROOT) + await self.exec_as_agent( + environment, + command=( + "set -euo pipefail; " + f"test -r {runtime}/payload.json; " + f"test -x {runtime}/bin/hermes; " + f"test -x {runtime}/bin/python; " + f"test -x {runtime}/bin/uv; " + f"test -r {shlex.quote(_HERMETIC_CA_BUNDLE)}; " + f"{_hermetic_runtime_readiness_command()}" + "rm -rf /tmp/hermes-agent-src; " + f"ln -s {runtime}/hermes-agent-src /tmp/hermes-agent-src; " + 'mkdir -p /tmp/hermes/bin "$HOME/.local/bin"; ' + f'ln -sf {runtime}/bin/hermes "$HOME/.local/bin/hermes"; ' + f"ln -sf {runtime}/bin/uv /tmp/hermes/bin/uv; " + f"if test -x {runtime}/bin/rg; then " + f'ln -sf {runtime}/bin/rg "$HOME/.local/bin/rg"; fi; ' + 'export PATH="$HOME/.local/bin:$PATH"' + ), + timeout_sec=90, + ) + return + await self.exec_as_root( environment, command=( @@ -371,22 +498,47 @@ async def setup(self, environment: BaseEnvironment) -> None: await environment.upload_file(self.relay_config_path, "/tmp/hermes/relay/plugins.toml") relay_wheel = f"/opt/relay-wheels/{self.relay_wheel_path.name}" await environment.upload_file(self.relay_wheel_path, relay_wheel) + if self.hermetic_runtime_dir is None: + relay_install = ( + "/tmp/hermes/bin/uv pip install " + "--python /tmp/hermes-agent-src/venv/bin/python " + f"--force-reinstall --no-deps {shlex.quote(relay_wheel)}; " + "/tmp/hermes-agent-src/venv/bin/python" + ) + else: + relay_install = f"{_HERMETIC_RUNTIME_ROOT}/bin/python" await self.exec_as_agent( environment, command=( "set -euo pipefail; " f"test \"$(sha256sum {shlex.quote(relay_wheel)} | cut -d' ' -f1)\" = " f"{shlex.quote(self.relay_wheel_sha256)}; " - "/tmp/hermes/bin/uv pip install " - "--python /tmp/hermes-agent-src/venv/bin/python " - f"--force-reinstall --no-deps {shlex.quote(relay_wheel)}; " - "/tmp/hermes-agent-src/venv/bin/python -c " - "\"import importlib.metadata as m; assert m.version('nemo-relay') == '0.7.0'\"" + f'{relay_install} -c "import importlib.metadata as m; ' + "assert m.version('nemo-relay') == '0.7.0'\"" ), timeout_sec=120, ) await environment.upload_dir(self.switchyard_bundle_dir, "/opt/relay-plugins/nvidia.switchyard") await environment.upload_file(self._finalizer_path, "/installed-agent/finalize_artifacts.py") + probe_python = ( + f"{_HERMETIC_RUNTIME_ROOT}/bin/python" + if self.hermetic_runtime_dir is not None + else "/tmp/hermes-agent-src/venv/bin/python" + ) + switchyard_library = f"/opt/relay-plugins/nvidia.switchyard/{self.switchyard_library.name}" + await self.exec_as_agent( + environment, + command=( + f"{probe_python} -c " + + shlex.quote( + "import ctypes, importlib.metadata as m; " + "assert m.version('nemo-relay') == '0.7.0'; " + f"library = ctypes.CDLL({switchyard_library!r}); " + "assert getattr(library, 'nemo_relay_register_plugin')" + ) + ), + timeout_sec=30, + ) await self.exec_as_agent( environment, command=self._finalizer_command("initialize"), @@ -402,7 +554,11 @@ def _finalizer_command( error_type: str = "", ) -> str: arguments = [ - "/tmp/hermes-agent-src/venv/bin/python", + ( + f"{_HERMETIC_RUNTIME_ROOT}/bin/python" + if self.hermetic_runtime_dir is not None + else "/tmp/hermes-agent-src/venv/bin/python" + ), "/installed-agent/finalize_artifacts.py", mode, "--artifact-root", diff --git a/examples/harbor-hermes-switchyard/run_phase2_cohort.sh b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh index c614e19fd..708ae888f 100755 --- a/examples/harbor-hermes-switchyard/run_phase2_cohort.sh +++ b/examples/harbor-hermes-switchyard/run_phase2_cohort.sh @@ -32,9 +32,13 @@ sample_count="${TBENCH_SAMPLE_COUNT:-89}" # keeps the conservative default. canary_task="${TBENCH_CANARY_TASK-adaptive-rejection-sampler}" concurrency="${TBENCH_CONCURRENCY:-4}" +setup_concurrency="${TBENCH_SETUP_CONCURRENCY:-2}" +setup_batch_size="${TBENCH_SETUP_BATCH_SIZE:-89}" +setup_max_infra_attempts="${TBENCH_SETUP_MAX_INFRA_ATTEMPTS:-4}" parallel_memory_gb="${TBENCH_PARALLEL_MAX_MEMORY_GB:-2}" docker_memory_reserve_gb="${TBENCH_DOCKER_MEMORY_RESERVE_GB:-4}" minimum_free_gb="${TBENCH_MINIMUM_FREE_GB:-100}" +bootstrap_root="${TERMINAL_BENCH_BOOTSTRAP_ROOT:-${TERMINAL_BENCH_ADMISSION_ROOT:-$(dirname "$run_root")}/bootstrap}" for required in \ "$harbor_bin" \ @@ -74,6 +78,9 @@ exec "$python_bin" "$example_root/scripts/run_phase2_cohort.py" \ --sample-count "$sample_count" \ --canary-task "$canary_task" \ --concurrency "$concurrency" \ + --setup-concurrency "$setup_concurrency" \ + --setup-batch-size "$setup_batch_size" \ + --setup-max-infra-attempts "$setup_max_infra_attempts" \ --parallel-max-memory-gb "$parallel_memory_gb" \ --docker-memory-reserve-gb "$docker_memory_reserve_gb" \ --minimum-free-gb "$minimum_free_gb" \ @@ -89,4 +96,5 @@ exec "$python_bin" "$example_root/scripts/run_phase2_cohort.py" \ --switchyard-bundle "$switchyard_bundle" \ --relay-wheel "$relay_wheel" \ --relay-architecture "$relay_architecture" \ + --bootstrap-root "$bootstrap_root" \ "$@" diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index 454fa236d..279c2c60d 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -28,6 +28,9 @@ agent_setup_timeout_multiplier="${AGENT_SETUP_TIMEOUT_MULTIPLIER:-6}" environment_build_timeout_multiplier="${ENVIRONMENT_BUILD_TIMEOUT_MULTIPLIER:-6}" collector_image="${OTEL_COLLECTOR_IMAGE:-otel/opentelemetry-collector-contrib:0.135.0}" inject_post_response_failure="${INJECT_POST_RESPONSE_FAILURE:-false}" +hermetic_runtime_dir="${HERMETIC_RUNTIME_DIR:-}" +hermetic_runtime_sha256="${HERMETIC_RUNTIME_SHA256:-}" +harbor_force_build="${HARBOR_FORCE_BUILD:-true}" if [[ -z "$run_root" || "$run_root" != /* ]]; then echo "usage: $0 /absolute/new-run-root" >&2 @@ -86,6 +89,25 @@ if [[ "$relay_architecture" != "x86_64" && "$relay_architecture" != "aarch64" ]] echo "RELAY_ARCHITECTURE must be x86_64 or aarch64" >&2 exit 2 fi +if [[ ( -n "$hermetic_runtime_dir" && -z "$hermetic_runtime_sha256" ) || \ + ( -z "$hermetic_runtime_dir" && -n "$hermetic_runtime_sha256" ) ]]; then + echo "HERMETIC_RUNTIME_DIR and HERMETIC_RUNTIME_SHA256 must be supplied together" >&2 + exit 2 +fi +if [[ -n "$hermetic_runtime_dir" ]]; then + if [[ "$hermetic_runtime_dir" != /* || ! -f "$hermetic_runtime_dir/payload.json" ]]; then + echo "HERMETIC_RUNTIME_DIR must be an absolute prepared runtime directory" >&2 + exit 2 + fi + if [[ ! "$hermetic_runtime_sha256" =~ ^[0-9a-f]{64}$ ]]; then + echo "HERMETIC_RUNTIME_SHA256 must be a lowercase SHA-256 digest" >&2 + exit 2 + fi +fi +if [[ "$harbor_force_build" != "true" && "$harbor_force_build" != "false" ]]; then + echo "HARBOR_FORCE_BUILD must be true or false" >&2 + exit 2 +fi docker info >/dev/null curl --fail --silent --show-error \ @@ -127,14 +149,23 @@ chmod 0600 "$provider_authorization_file" provider_authorization_target="/run/secrets/switchyard-provider-authorization" mounts_json="$($python_bin -c ' import json, sys -print(json.dumps([{ +mounts = [{ "type": "bind", "source": sys.argv[1], "target": sys.argv[2], "read_only": True, "bind": {"create_host_path": False}, -}], separators=(",", ":"))) -' "$provider_authorization_file" "$provider_authorization_target")" +}] +if sys.argv[3]: + mounts.append({ + "type": "bind", + "source": sys.argv[3], + "target": "/opt/hermes-runtime", + "read_only": True, + "bind": {"create_host_path": False}, + }) +print(json.dumps(mounts, separators=(",", ":"))) +' "$provider_authorization_file" "$provider_authorization_target" "$hermetic_runtime_dir")" if [[ -z "$switchyard_bundle" ]]; then temporary_build="$(mktemp -d "$(dirname "$run_root")/.switchyard-build.XXXXXX")" @@ -212,6 +243,16 @@ elif [[ "$inject_post_response_failure" != "false" ]]; then echo "INJECT_POST_RESPONSE_FAILURE must be true or false" >&2 exit 2 fi +if [[ -n "$hermetic_runtime_dir" ]]; then + agent_kwargs+=( + --ak "hermetic_runtime_dir=$hermetic_runtime_dir" + --ak "hermetic_runtime_sha256=$hermetic_runtime_sha256" + ) +fi +harbor_build_args=() +if [[ "$harbor_force_build" == "true" ]]; then + harbor_build_args+=(--force-build) +fi ( "$harbor_bin" run \ "${dataset_args[@]}" \ @@ -242,7 +283,7 @@ fi --agent-timeout-multiplier "$agent_timeout_multiplier" \ --agent-setup-timeout-multiplier "$agent_setup_timeout_multiplier" \ --environment-build-timeout-multiplier "$environment_build_timeout_multiplier" \ - --force-build \ + "${harbor_build_args[@]}" \ --yes ) >"$run_root/harbor.log" 2>&1 diff --git a/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py b/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py new file mode 100755 index 000000000..6236ef890 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build one reusable, provider-free Hermes runtime for Phase 2 setup. + +The coordinator invokes this content-addressed materialization automatically +when it is absent. Network access is confined to this one step; task containers +consume the resulting directory through a read-only bind mount and perform no +apt, Git, or Python package installation. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +import time +from pathlib import Path + +SCHEMA_VERSION = "harbor-hermes-switchyard.hermetic-runtime.v1" +DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" +DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" +DEFAULT_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" +UV_VERSION = "0.11.16" +PYTHON_VERSION = "3.11.13" +BUILDER_IMAGE = "python:3.11-bullseye" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def sha256_tree(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()): + if path.name == "payload.json": + continue + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + digest.update(path.stat().st_mode.to_bytes(4, "big")) + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def run(command: list[str], *, attempts: int = 4, **kwargs: object) -> None: + for attempt in range(1, attempts + 1): + try: + subprocess.run(command, check=True, **kwargs) + return + except subprocess.CalledProcessError: + if attempt == attempts: + raise + time.sleep(5 * (2 ** (attempt - 1))) + + +def materialize_source( + destination: Path, + *, + repository: str, + repository_ref: str, + commit: str, +) -> None: + clone = destination.parent / "clone" + for attempt in range(1, 5): + shutil.rmtree(clone, ignore_errors=True) + try: + run( + [ + "git", + "clone", + "--no-tags", + "--filter=blob:none", + "--branch", + repository_ref, + repository, + str(clone), + ], + attempts=1, + ) + break + except subprocess.CalledProcessError: + if attempt == 4: + raise + time.sleep(5 * (2 ** (attempt - 1))) + run(["git", "-C", str(clone), "fetch", "--depth", "1", "origin", commit]) + run(["git", "-C", str(clone), "checkout", "--detach", commit]) + actual = subprocess.check_output( + ["git", "-C", str(clone), "rev-parse", "HEAD"], text=True + ).strip() + if actual != commit: + raise RuntimeError(f"Hermes checkout mismatch: expected {commit}, got {actual}") + shutil.copytree(clone, destination, ignore=shutil.ignore_patterns(".git")) + + +def build_payload( + output: Path, + *, + source: Path, + relay_wheel: Path, + platform: str, +) -> None: + script = r''' +set -euo pipefail +python -m pip install --no-cache-dir "uv==${UV_VERSION}" + +mkdir -p /opt/hermes-runtime/bin /opt/hermes-runtime/lib +cp -a /source /opt/hermes-runtime/hermes-agent-src +cp /usr/local/bin/uv /opt/hermes-runtime/bin/uv + +/usr/local/bin/uv python install "${PYTHON_VERSION}" \ + --install-dir /opt/hermes-runtime/python --no-bin --compile-bytecode +python_bin="$(find /opt/hermes-runtime/python -type f -path '*/bin/python3.11' -print -quit)" +test -n "$python_bin" + +UV_PROJECT_ENVIRONMENT=/opt/hermes-runtime/hermes-agent-src/venv \ +UV_PYTHON_DOWNLOADS=never \ + /usr/local/bin/uv sync --frozen --extra all \ + --project /opt/hermes-runtime/hermes-agent-src --python "$python_bin" +/usr/local/bin/uv pip install \ + --python /opt/hermes-runtime/hermes-agent-src/venv/bin/python \ + --force-reinstall --no-deps "/input/${RELAY_WHEEL_NAME}" + +cat > /opt/hermes-runtime/bin/python <<'EOF' +#!/bin/sh +set -eu +export LD_LIBRARY_PATH="/opt/hermes-runtime/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +exec /opt/hermes-runtime/hermes-agent-src/venv/bin/python "$@" +EOF +cat > /opt/hermes-runtime/bin/hermes <<'EOF' +#!/bin/sh +set -eu +export LD_LIBRARY_PATH="/opt/hermes-runtime/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +exec /opt/hermes-runtime/hermes-agent-src/venv/bin/hermes "$@" +EOF +chmod 0755 /opt/hermes-runtime/bin/python /opt/hermes-runtime/bin/hermes \ + /opt/hermes-runtime/bin/uv + +/opt/hermes-runtime/bin/hermes version +/opt/hermes-runtime/bin/python -c \ + 'import importlib.metadata as m; assert m.version("nemo-relay") == "0.7.0"' +''' + env = os.environ.copy() + env.update({"UV_VERSION": UV_VERSION, "PYTHON_VERSION": PYTHON_VERSION}) + run( + [ + "docker", + "run", + "--rm", + "--platform", + platform, + "--env", + f"UV_VERSION={UV_VERSION}", + "--env", + f"PYTHON_VERSION={PYTHON_VERSION}", + "--env", + f"RELAY_WHEEL_NAME={relay_wheel.name}", + "--volume", + f"{source}:/source:ro", + "--volume", + f"{relay_wheel}:/input/{relay_wheel.name}:ro", + "--volume", + f"{output}:/opt/hermes-runtime", + BUILDER_IMAGE, + "bash", + "-lc", + script, + ], + attempts=1, + env=env, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--relay-wheel", type=Path, required=True) + parser.add_argument("--relay-architecture", choices=("x86_64", "aarch64"), required=True) + parser.add_argument("--hermes-repository", default=DEFAULT_HERMES_REPOSITORY) + parser.add_argument("--hermes-ref", default=DEFAULT_HERMES_REF) + parser.add_argument("--hermes-commit", default=DEFAULT_HERMES_COMMIT) + args = parser.parse_args() + + output = args.output.expanduser().resolve() + relay_wheel = args.relay_wheel.expanduser().resolve(strict=True) + if output.exists(): + raise FileExistsError(f"output already exists: {output}") + expected_arch = args.relay_architecture + if "manylinux" not in relay_wheel.name or expected_arch not in relay_wheel.name: + raise ValueError(f"Relay wheel does not target Linux {expected_arch}: {relay_wheel.name}") + platform = {"x86_64": "linux/amd64", "aarch64": "linux/arm64"}[expected_arch] + + output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary_output = output.with_name(f".{output.name}.building") + if temporary_output.exists(): + raise FileExistsError(f"stale temporary output exists: {temporary_output}") + temporary_output.mkdir(mode=0o700) + try: + with tempfile.TemporaryDirectory( + prefix="hermes-source-", dir=output.parent + ) as temporary: + source = Path(temporary) / "source" + materialize_source( + source, + repository=args.hermes_repository, + repository_ref=args.hermes_ref, + commit=args.hermes_commit, + ) + for attempt in range(1, 5): + try: + build_payload( + temporary_output, + source=source, + relay_wheel=relay_wheel, + platform=platform, + ) + break + except subprocess.CalledProcessError: + if attempt == 4: + raise + shutil.rmtree(temporary_output) + temporary_output.mkdir(mode=0o700) + time.sleep(5 * (2 ** (attempt - 1))) + content_sha256 = sha256_tree(temporary_output) + marker = { + "schema_version": SCHEMA_VERSION, + "status": "passed", + "content_sha256": content_sha256, + "hermes_repository": args.hermes_repository, + "hermes_ref": args.hermes_ref, + "hermes_commit": args.hermes_commit, + "relay_version": "0.7.0", + "relay_wheel_sha256": sha256_file(relay_wheel), + "relay_architecture": expected_arch, + "builder_image": BUILDER_IMAGE, + "python_version": PYTHON_VERSION, + "uv_version": UV_VERSION, + } + (temporary_output / "payload.json").write_text( + json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.chmod(temporary_output / "payload.json", 0o444) + temporary_output.rename(output) + print(json.dumps({"output": str(output), **marker}, indent=2, sort_keys=True)) + except BaseException: + shutil.rmtree(temporary_output, ignore_errors=True) + raise + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh b/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh index d37bdd521..05d40ad9c 100755 --- a/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh +++ b/examples/harbor-hermes-switchyard/scripts/build_switchyard_plugin.sh @@ -5,7 +5,7 @@ set -euo pipefail switchyard_repository="${SWITCHYARD_REPOSITORY:-https://github.com/bbednarski9/Switchyard.git}" -switchyard_commit="${SWITCHYARD_COMMIT:-5d9d3292d6154e44d50295d0d4a3fd4f144f2528}" +switchyard_commit="${SWITCHYARD_COMMIT:-8daac03edf8544144833af1fd009b3da737715bc}" target_architecture="${SWITCHYARD_TARGET_ARCHITECTURE:-x86_64}" output_dir="${1:-}" @@ -40,7 +40,7 @@ if [[ "$target_architecture" != "x86_64" && "$target_architecture" != "aarch64" exit 2 fi if [[ "$docker_architecture" == "aarch64" || "$docker_architecture" == "arm64" ]]; then - builder_image="${SWITCHYARD_BUILDER_IMAGE:-rust:1.96.1-bookworm@sha256:809725748b728a8e1f8621a3c76e49fba8780c16d99ceda20abdb44d32665c30}" + builder_image="${SWITCHYARD_BUILDER_IMAGE:-rust:1.96.1-bullseye@sha256:69e444ec65a82386d041a4a3d15e47a797967b90ae24aa342bd8a3600dd9e244}" builder_platform="linux/arm64" if [[ "$target_architecture" == "x86_64" ]]; then cargo_target="x86_64-unknown-linux-gnu" @@ -54,7 +54,7 @@ else echo "aarch64 cross-builds from an x86_64 Docker host are not supported" >&2 exit 2 fi - builder_image="${SWITCHYARD_BUILDER_IMAGE:-rust:1.96.1-bookworm@sha256:d99f7b31f49909348dc59b51f3c95d1efded1701ffb222f095aaab7de3c4abd8}" + builder_image="${SWITCHYARD_BUILDER_IMAGE:-rust:1.96.1-bullseye@sha256:65136b30fc6b10112cbae63a868da085a878679a80d562272e485ecaaad3276a}" builder_platform="linux/amd64" cargo_target="" library_path="/tmp/target/release/libswitchyard_nemo_relay_plugin.so" diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index 522d4745a..0b4b090f3 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -24,7 +24,7 @@ HERMES_REF = "feat/relay-native-plugin-init" HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" SWITCHYARD_REPOSITORY = "https://github.com/bbednarski9/Switchyard.git" -SWITCHYARD_COMMIT = "5d9d3292d6154e44d50295d0d4a3fd4f144f2528" +SWITCHYARD_COMMIT = "8daac03edf8544144833af1fd009b3da737715bc" RELAY_VERSION = "0.7.0" SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}") diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index e4421238a..5096fdebe 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -14,6 +14,7 @@ import re import shutil import subprocess +import sys import time import tomllib import urllib.error @@ -23,10 +24,16 @@ from pathlib import Path from typing import Any +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +import run_setup_admission as setup_admission # noqa: E402 + SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-cohort.v1" PLAN_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-plan.v1" TASK_STATE_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-task-state.v1" EXPECTED_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" +HERMETIC_RUNTIME_SCHEMA = "harbor-hermes-switchyard.hermetic-runtime.v1" INFRASTRUCTURE_PATTERNS = ( "apt-get update && apt-get install", "cannot connect to the docker daemon", @@ -224,6 +231,131 @@ def switchyard_library(bundle: Path) -> Path: return libraries[0] +def load_hermetic_runtime(path: Path, args: argparse.Namespace) -> dict[str, Any]: + payload = setup_admission.load_payload(path) + expected = { + "schema_version": HERMETIC_RUNTIME_SCHEMA, + "status": "passed", + "hermes_commit": EXPECTED_HERMES_COMMIT, + "relay_version": "0.7.0", + "relay_wheel_sha256": sha256_file(args.relay_wheel), + "relay_architecture": args.relay_architecture, + } + mismatches = { + key: {"expected": value, "actual": payload.get(key)} + for key, value in expected.items() + if payload.get(key) != value + } + if mismatches: + raise ValueError(f"hermetic runtime does not match cohort inputs: {mismatches}") + digest = payload.get("content_sha256") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("hermetic runtime content digest is missing or invalid") + for relative in ("bin/hermes", "bin/python", "bin/uv", "hermes-agent-src/venv"): + if not (path / relative).exists(): + raise FileNotFoundError(path / relative) + return payload + + +def ensure_hermetic_runtime(args: argparse.Namespace) -> tuple[Path, dict[str, Any]]: + wheel_digest = sha256_file(args.relay_wheel) + name = f"hermes-{EXPECTED_HERMES_COMMIT[:8]}-relay-070-{args.relay_architecture}-{wheel_digest[:12]}" + output = args.bootstrap_root / name + args.bootstrap_root.mkdir(mode=0o700, parents=True, exist_ok=True) + lock_path = args.bootstrap_root / f".{name}.lock" + with lock_path.open("a+") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if output.is_dir(): + return output, load_hermetic_runtime(output, args) + if output.exists(): + raise ValueError(f"hermetic runtime cache path is not a directory: {output}") + subprocess.run( + [ + str(args.python_bin), + str(args.hermetic_runtime_builder), + "--output", + str(output), + "--relay-wheel", + str(args.relay_wheel), + "--relay-architecture", + args.relay_architecture, + "--hermes-commit", + EXPECTED_HERMES_COMMIT, + ], + check=True, + ) + return output, load_hermetic_runtime(output, args) + + +def prepare_setup_runtime(args: argparse.Namespace) -> Path: + destination = args.run_root / "setup-runtime" + runtime = destination / "runtime" + provenance = runtime / "provenance.json" + if provenance.is_file(): + observed = read_json(provenance) + if ( + observed.get("nemo_relay", {}).get("wheel_sha256") != sha256_file(args.relay_wheel) + or observed.get("switchyard", {}).get("library_sha256") + != sha256_file(switchyard_library(args.switchyard_bundle)) + or observed.get("relay_config_sha256") != sha256_file(runtime / "plugins.toml") + ): + raise ValueError("existing setup runtime does not match cohort inputs") + return runtime + if destination.exists(): + raise ValueError(f"incomplete setup runtime already exists: {destination}") + temporary = destination.with_name(f".{destination.name}.preparing-{os.getpid()}") + if temporary.exists(): + raise ValueError(f"stale setup runtime preparation exists: {temporary}") + try: + subprocess.run( + [ + str(args.python_bin), + str(args.runtime_preparer), + "--run-root", + str(temporary), + "--switchyard-bundle", + str(args.switchyard_bundle), + "--relay-wheel", + str(args.relay_wheel), + "--relay-architecture", + args.relay_architecture, + "--plugin-config-template", + str(args.plugin_config_template), + "--openinference-endpoint", + "http://127.0.0.1:9/v1/traces", + "--phoenix-project", + args.phoenix_project, + "--eval-cohort", + args.eval_cohort, + ], + check=True, + ) + temporary.replace(destination) + except BaseException: + shutil.rmtree(temporary, ignore_errors=True) + raise + return runtime + + +def bootstrap_preflight(args: argparse.Namespace) -> dict[str, Any]: + clock = setup_admission.run_clock_preflight() + if clock["status"] != "passed": + raise RuntimeError("wall-clock preflight failed; synchronize the host and Docker clocks") + compatibility_plan = { + "inputs": { + "relay_architecture": args.relay_architecture, + "switchyard_bundle": str(args.switchyard_bundle), + "switchyard_library_sha256": sha256_file(switchyard_library(args.switchyard_bundle)), + } + } + plugin = setup_admission.run_plugin_compatibility_preflight(compatibility_plan) + if plugin["status"] != "passed": + raise RuntimeError("Switchyard plugin does not load in the oldest supported task base") + evidence = {"status": "passed", "clock": clock, "plugin_compatibility": plugin} + write_json(args.run_root / "bootstrap-preflight.json", evidence) + return evidence + + def validate_smoke_evidence( path: Path, expected_count: int, @@ -482,6 +614,9 @@ def make_plan(args: argparse.Namespace, tasks: list[Task]) -> dict[str, Any]: "sample_count": args.sample_count, "canary_task": args.canary_task, "concurrency": args.concurrency, + "setup_concurrency": args.setup_concurrency, + "setup_batch_size": args.setup_batch_size, + "setup_max_infra_attempts": args.setup_max_infra_attempts, "parallel_max_memory_gb": args.parallel_max_memory_gb, "docker_memory_reserve_gb": args.docker_memory_reserve_gb, "minimum_free_gb": args.minimum_free_gb, @@ -506,6 +641,8 @@ def make_plan(args: argparse.Namespace, tasks: list[Task]) -> dict[str, Any]: "relay_wheel_sha256": sha256_file(args.relay_wheel), "switchyard_manifest_sha256": sha256_file(manifest), "switchyard_library_sha256": sha256_file(library_candidates[0]), + "hermetic_runtime_sha256": args.hermetic_runtime_payload["content_sha256"], + "setup_runtime_provenance_sha256": sha256_file(args.setup_runtime / "provenance.json"), }, "tasks": [task.as_json() for task in tasks], } @@ -634,6 +771,64 @@ async def refresh_summary(self) -> None: write_json(self.args.run_root / "summary.json", summary) write_report(self.args.run_root, summary) + def provision_environments(self) -> bool: + output = self.args.run_root / "setup-admission" + command = [ + str(self.args.python_bin), + str(self.args.setup_admission_runner), + "--dataset", + str(self.args.dataset_root), + "--runtime-root", + str(self.args.setup_runtime), + "--hermetic-runtime", + str(self.args.hermetic_runtime), + "--output", + str(output), + "--harbor", + str(self.args.harbor_bin), + "--concurrency", + str(self.args.setup_concurrency), + "--batch-size", + str(self.args.setup_batch_size), + "--max-infra-attempts", + str(self.args.setup_max_infra_attempts), + "--backoff-seconds", + str(self.args.backoff_seconds), + "--force-build", + "--no-preserve-containers", + ] + log_path = self.args.run_root / "setup-admission-coordinator.log" + with log_path.open("a", encoding="utf-8") as log: + process = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT) + summary_path = output / "summary.json" + summary = read_json(summary_path) if summary_path.is_file() else {} + passed = ( + process.returncode == 0 + and summary.get("status") == "passed" + and summary.get("planned") == len(self.tasks) + and summary.get("passed") == len(self.tasks) + ) + write_json( + self.args.run_root / "setup-state.json", + { + "schema_version": "harbor-hermes-switchyard.phase2-setup-state.v1", + "status": "passed" if passed else "failed", + "failure_class": ( + None + if passed + else "harness_or_integration" + if process.returncode == 20 or summary.get("integration_failures") + else "infrastructure" + ), + "exit_code": process.returncode, + "summary": summary, + "hermetic_runtime_sha256": self.args.hermetic_runtime_payload["content_sha256"], + "force_build": True, + "setup_concurrency": self.args.setup_concurrency, + }, + ) + return passed + async def run_attempt(self, task: Task, attempt_number: int) -> tuple[int, Path, str]: task_root = self.args.run_root / "tasks" / task.directory_name attempts_root = task_root / "attempts" @@ -660,6 +855,13 @@ async def run_attempt(self, task: Task, attempt_number: int) -> tuple[int, Path, "AGENT_TIMEOUT_MULTIPLIER": "3", "AGENT_SETUP_TIMEOUT_MULTIPLIER": "6", "ENVIRONMENT_BUILD_TIMEOUT_MULTIPLIER": "6", + "HERMETIC_RUNTIME_DIR": str(self.args.hermetic_runtime), + "HERMETIC_RUNTIME_SHA256": self.args.hermetic_runtime_payload["content_sha256"], + # Harbor assigns trial-specific local image tags. Rebuild from the + # pinned task Dockerfile so an incompatible published prebuilt image + # cannot replace the architecture-validated setup-admission image. + # The setup lane has already populated Docker's layer cache. + "HARBOR_FORCE_BUILD": "true", } ) with log_path.open("wb") as log: @@ -782,6 +984,8 @@ async def run(self) -> bool: preflight = shared_preflight(self.args, self.tasks) write_json(self.args.run_root / "preflight.json", {"status": "passed", **preflight}) await self.refresh_summary() + if not self.provision_environments(): + return False remaining = self.tasks if self.args.canary_task: first = self.tasks[0] @@ -813,6 +1017,9 @@ def parse_args() -> argparse.Namespace: help="task to run alone before the cohort; pass an empty value to disable the canary", ) parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--setup-concurrency", type=int, default=2) + parser.add_argument("--setup-batch-size", type=int, default=89) + parser.add_argument("--setup-max-infra-attempts", type=int, default=4) parser.add_argument("--parallel-max-memory-gb", type=int, default=2) parser.add_argument("--docker-memory-reserve-gb", type=int, default=4) parser.add_argument("--max-infra-attempts", type=int, default=3) @@ -822,6 +1029,26 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--offline-evidence", type=Path, required=True) parser.add_argument("--plugin-config-template", type=Path, required=True) parser.add_argument("--task-runner", type=Path, default=example_root / "run_terminal_bench.sh") + parser.add_argument( + "--setup-admission-runner", + type=Path, + default=example_root / "scripts" / "run_setup_admission.py", + ) + parser.add_argument( + "--hermetic-runtime-builder", + type=Path, + default=example_root / "scripts" / "build_hermetic_runtime.py", + ) + parser.add_argument( + "--runtime-preparer", + type=Path, + default=example_root / "scripts" / "prepare_runtime.py", + ) + parser.add_argument( + "--bootstrap-root", + type=Path, + help="shared content-addressed bootstrap cache; generated automatically when absent", + ) parser.add_argument("--harbor-bin", type=Path, default=example_root / ".venv" / "bin" / "harbor") parser.add_argument("--python-bin", type=Path, default=example_root / ".venv" / "bin" / "python") parser.add_argument("--phoenix-url", required=True) @@ -837,6 +1064,9 @@ def parse_args() -> argparse.Namespace: for name in ( "sample_count", "concurrency", + "setup_concurrency", + "setup_batch_size", + "setup_max_infra_attempts", "parallel_max_memory_gb", "docker_memory_reserve_gb", "max_infra_attempts", @@ -846,6 +1076,9 @@ def parse_args() -> argparse.Namespace: parser.error(f"--{name.replace('_', '-')} must be positive") if not args.run_root.is_absolute(): parser.error("--run-root must be absolute") + args.bootstrap_root = ( + (args.bootstrap_root or args.run_root.parent / "harbor-hermes-switchyard-bootstrap").expanduser().resolve() + ) if args.plan_only and args.preflight_only: parser.error("--plan-only and --preflight-only are mutually exclusive") return args @@ -881,6 +1114,9 @@ def main() -> int: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: raise SystemExit(f"another Phase 2 supervisor owns {args.run_root}") from None + bootstrap_preflight(args) + args.hermetic_runtime, args.hermetic_runtime_payload = ensure_hermetic_runtime(args) + args.setup_runtime = prepare_setup_runtime(args) plan = make_plan(args, tasks) load_or_create_plan(args.run_root / "plan.json", plan) if args.plan_only: @@ -903,6 +1139,9 @@ def main() -> int: if passed: return 0 states = [read_json(path) for path in (args.run_root / "tasks").glob("*/task-state.json") if path.is_file()] + setup_state = args.run_root / "setup-state.json" + if setup_state.is_file(): + states.append(read_json(setup_state)) if any(state.get("failure_class") == "harness_or_integration" for state in states): return 20 return 75 diff --git a/examples/harbor-hermes-switchyard/scripts/run_regression_smokes.sh b/examples/harbor-hermes-switchyard/scripts/run_regression_smokes.sh deleted file mode 100755 index a904448c5..000000000 --- a/examples/harbor-hermes-switchyard/scripts/run_regression_smokes.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -regression_root="${1:-}" - -if [[ -z "$regression_root" || "$regression_root" != /* ]]; then - echo "usage: $0 /absolute/new-regression-root" >&2 - exit 2 -fi -if [[ -e "$regression_root" ]]; then - echo "regression root already exists: $regression_root" >&2 - exit 2 -fi -mkdir -m 0700 "$regression_root" - -tasks=( - adaptive-rejection-sampler - circuit-fibsqrt - gpt2-codegolf - overfull-hbox -) - -for task in "${tasks[@]}"; do - echo "Running regression smoke: $task" - run_root="$regression_root/$task" - inject=false - if [[ "$task" == "circuit-fibsqrt" ]]; then - inject=true - fi - TASK_NAME="$task" \ - PHOENIX_PROJECT="${PHOENIX_PROJECT:-harbor-hermes-switchyard-regression}-$task" \ - EVAL_COHORT="${EVAL_COHORT:-harbor-hermes-switchyard-regression}-$task" \ - EVAL_PHASE="regression" \ - INJECT_POST_RESPONSE_FAILURE="$inject" \ - "$example_root/run_terminal_bench.sh" "$run_root" -done - -python_bin="${EVAL_PYTHON:-python3}" -"$python_bin" - "$regression_root" "${tasks[@]}" <<'PY' -import json -import pathlib -import sys - -root = pathlib.Path(sys.argv[1]) -tasks = sys.argv[2:] -summaries = [] -for task in tasks: - summary_path = root / task / "summary.json" - if not summary_path.is_file(): - raise SystemExit(f"missing summary: {summary_path}") - summary = json.loads(summary_path.read_text(encoding="utf-8")) - if summary.get("status") != "passed": - raise SystemExit(f"regression did not pass: {task}") - summaries.append(summary) - -result = { - "schema_version": "harbor-hermes-switchyard.regression-smokes.v1", - "status": "passed", - "planned": len(tasks), - "completed": len(summaries), - "tasks": tasks, -} -(root / "summary.json").write_text( - json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" -) -print(json.dumps(result, indent=2)) -PY diff --git a/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py b/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py new file mode 100755 index 000000000..74e89749c --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Provision task environments through Harbor's provider-free install lifecycle. + +The Phase 2 coordinator and the standalone diagnostic share this implementation. +It stops before agent execution and the verifier, uses Harbor's real Docker and +agent setup paths, reuses successful task evidence by content hash, and never +loads provider authorization. +""" + +from __future__ import annotations + +import argparse +import email.utils +import hashlib +import json +import os +import subprocess +import time +import urllib.request +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from harbor import __version__ as harbor_version +from harbor.models.task.task import Task + +PLAN_SCHEMA = "harbor-hermes-switchyard.setup-admission-plan.v2" +RESULT_SCHEMA = "harbor-hermes-switchyard.setup-admission-task.v2" +SUMMARY_SCHEMA = "harbor-hermes-switchyard.setup-admission-summary.v2" +PAYLOAD_SCHEMA = "harbor-hermes-switchyard.hermetic-runtime.v1" +CLOCK_PREFLIGHT_SCHEMA = "harbor-hermes-switchyard.clock-preflight.v1" +CLOCK_REFERENCE_URL = "https://deb.debian.org/debian-security/dists/bookworm-security/InRelease" +CLOCK_MAX_OFFSET_SECONDS = 300 +CLOCK_PROBE_IMAGE = "python:3.11-bullseye" +PLUGIN_COMPATIBILITY_SCHEMA = "harbor-hermes-switchyard.plugin-compatibility.v1" +SETUP_AGENT_PATH = Path(__file__).resolve().parents[1] / "agents" / "harbor_hermes_agent.py" +INFRASTRUCTURE_PATTERNS = ( + "apt-get update", + "connection refused", + "connection reset", + "connection timed out", + "context deadline exceeded", + "docker build failed", + "failed to resolve source metadata", + "failed to authorize", + "i/o timeout", + "network is unreachable", + "no space left on device", + "registry-1.docker.io", + "temporary failure in name resolution", + "remote end closed connection", + "tls handshake timeout", + "too many requests", + "unexpected eof", +) + + +def classify_setup_failure(trial_root: Path, diagnostic: str) -> str: + lowered = diagnostic.lower() + if any(pattern in lowered for pattern in INFRASTRUCTURE_PATTERNS): + return "infrastructure" + for path in sorted(trial_root.rglob("*.log")): + try: + text = path.read_text(encoding="utf-8", errors="replace").lower() + except OSError: + continue + if any(pattern in text for pattern in INFRASTRUCTURE_PATTERNS): + return "infrastructure" + return "harness_or_integration" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def sha256_tree(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()): + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def hermetic_content_sha256(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(candidate for candidate in root.rglob("*") if candidate.is_file()): + if path.name == "payload.json": + continue + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(4, "big")) + digest.update(relative) + digest.update(path.stat().st_mode.to_bytes(4, "big")) + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def canonical_sha256(value: Any) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def write_json(path: Path, value: Any) -> None: + temporary = path.with_suffix(f"{path.suffix}.tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + temporary.replace(path) + + +def evaluate_clock_preflight(*, host_epoch: float, docker_epoch: float, reference_epoch: float) -> dict[str, Any]: + host_reference_offset = round(reference_epoch - host_epoch, 3) + docker_host_offset = round(docker_epoch - host_epoch, 3) + passed = ( + abs(host_reference_offset) <= CLOCK_MAX_OFFSET_SECONDS and abs(docker_host_offset) <= CLOCK_MAX_OFFSET_SECONDS + ) + return { + "schema_version": CLOCK_PREFLIGHT_SCHEMA, + "status": "passed" if passed else "failed", + "checked_at": datetime.now(UTC).isoformat(), + "reference_url": CLOCK_REFERENCE_URL, + "maximum_offset_seconds": CLOCK_MAX_OFFSET_SECONDS, + "host_reference_offset_seconds": host_reference_offset, + "docker_host_offset_seconds": docker_host_offset, + } + + +def run_clock_preflight() -> dict[str, Any]: + date_header: str | None = None + for attempt in range(1, 5): + request = urllib.request.Request( + CLOCK_REFERENCE_URL, + method="HEAD", + headers={"User-Agent": "harbor-hermes-switchyard-setup-admission/2"}, + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + date_header = response.headers.get("Date") + break + except OSError: + if attempt == 4: + raise + time.sleep(2 ** (attempt - 1)) + if not date_header: + raise RuntimeError("clock reference response omitted its Date header") + reference_datetime = email.utils.parsedate_to_datetime(date_header) + docker_output = "" + for attempt in range(1, 5): + try: + docker_output = subprocess.check_output( + [ + "docker", + "run", + "--rm", + "--pull=missing", + CLOCK_PROBE_IMAGE, + "date", + "+%s", + ], + text=True, + ).strip() + break + except subprocess.CalledProcessError: + if attempt == 4: + raise + time.sleep(2 ** (attempt - 1)) + return evaluate_clock_preflight( + host_epoch=time.time(), + docker_epoch=float(docker_output), + reference_epoch=reference_datetime.timestamp(), + ) + + +def plugin_compatibility_command(plan: dict[str, Any]) -> list[str]: + inputs = plan["inputs"] + platform = { + "aarch64": "linux/arm64", + "x86_64": "linux/amd64", + }[inputs["relay_architecture"]] + return [ + "docker", + "run", + "--rm", + "--pull=missing", + "--platform", + platform, + "--volume", + f"{inputs['switchyard_bundle']}:/bundle:ro", + CLOCK_PROBE_IMAGE, + "python", + "-c", + ( + "import ctypes; " + "library=ctypes.CDLL('/bundle/libswitchyard_nemo_relay_plugin.so'); " + "getattr(library, 'nemo_relay_register_plugin')" + ), + ] + + +def run_plugin_compatibility_preflight(plan: dict[str, Any]) -> dict[str, Any]: + for attempt in range(1, 5): + process = subprocess.run( + plugin_compatibility_command(plan), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=120, + ) + if process.returncode == 0 or attempt == 4: + break + time.sleep(2 ** (attempt - 1)) + inputs = plan["inputs"] + return { + "schema_version": PLUGIN_COMPATIBILITY_SCHEMA, + "status": "passed" if process.returncode == 0 else "failed", + "checked_at": datetime.now(UTC).isoformat(), + "probe_image": CLOCK_PROBE_IMAGE, + "relay_architecture": inputs["relay_architecture"], + "switchyard_library_sha256": inputs["switchyard_library_sha256"], + "registration_symbol": "nemo_relay_register_plugin", + "exit_code": process.returncode, + } + + +def load_payload(path: Path) -> dict[str, Any]: + marker = path / "payload.json" + payload = json.loads(marker.read_text(encoding="utf-8")) + if payload.get("schema_version") != PAYLOAD_SCHEMA or payload.get("status") != "passed": + raise ValueError(f"invalid hermetic runtime marker: {marker}") + observed = hermetic_content_sha256(path) + if payload.get("content_sha256") != observed: + raise ValueError( + "hermetic runtime content does not match its marker: " + f"expected {payload.get('content_sha256')}, observed {observed}" + ) + return payload + + +def discover_tasks(dataset: Path) -> list[Task]: + tasks = [Task(path) for path in sorted(dataset.iterdir()) if Task.is_valid_dir(path)] + names = [task.name for task in tasks] + if len(tasks) != 89 or len(set(names)) != 89: + raise ValueError(f"expected 89 unique tasks, found {len(tasks)} tasks and {len(set(names))} names") + return sorted(tasks, key=lambda task: task.name) + + +def task_record(task: Task) -> dict[str, Any]: + task_dir = task.task_dir + environment = task.paths.environment_dir + tests = task.paths.tests_dir + record = { + "name": task.name, + "task_dir": str(task_dir), + "task_sha256": sha256_tree(task_dir), + "environment_sha256": sha256_tree(environment), + "instruction_sha256": sha256_file(task.paths.instruction_path), + "verifier_sha256": sha256_tree(tests), + "docker_image": task.config.environment.docker_image, + "cpus": task.config.environment.cpus, + "memory_mb": task.config.environment.memory_mb, + "build_timeout_sec": task.config.environment.build_timeout_sec, + } + return record + + +def build_plan(args: argparse.Namespace, tasks: list[Task], payload: dict[str, Any]) -> dict[str, Any]: + runtime = args.runtime_root + provenance_path = runtime / "provenance.json" + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + relay_wheels = sorted((runtime / "wheels").glob("nemo_relay-0.7.0-*.whl")) + if len(relay_wheels) != 1: + raise ValueError(f"expected one Relay wheel below {runtime / 'wheels'}") + libraries = sorted((runtime / "switchyard-plugin").glob("*.so")) + if len(libraries) != 1: + raise ValueError("expected one Linux Switchyard library") + records = [task_record(task) for task in tasks] + inputs = { + "dataset_root": str(args.dataset), + "dataset_sha256": canonical_sha256( + [{key: value for key, value in record.items() if key != "task_dir"} for record in records] + ), + "hermetic_runtime_root": str(args.hermetic_runtime), + "hermetic_runtime_sha256": payload["content_sha256"], + "hermes_commit": payload["hermes_commit"], + "relay_architecture": payload["relay_architecture"], + "relay_wheel": str(relay_wheels[0]), + "relay_wheel_sha256": sha256_file(relay_wheels[0]), + "relay_config": str(runtime / "plugins.toml"), + "relay_config_sha256": sha256_file(runtime / "plugins.toml"), + "switchyard_bundle": str(runtime / "switchyard-plugin"), + "switchyard_library_sha256": sha256_file(libraries[0]), + "runtime_provenance_sha256": sha256_file(provenance_path), + "runtime_provenance_schema": provenance.get("schema_version"), + "setup_agent_sha256": sha256_file(SETUP_AGENT_PATH), + "harbor_version": harbor_version, + "concurrency": args.concurrency, + "batch_size": args.batch_size, + "maximum_infrastructure_attempts": args.max_infra_attempts, + "force_build": args.force_build, + "preserve_containers": args.preserve_containers, + } + if inputs["relay_wheel_sha256"] != payload["relay_wheel_sha256"]: + raise ValueError("prepared runtime Relay wheel does not match hermetic runtime") + if inputs["relay_architecture"] != provenance["nemo_relay"]["architecture"]: + raise ValueError("prepared runtime architecture does not match hermetic runtime") + return { + "schema_version": PLAN_SCHEMA, + "status": "planned", + "created_at": datetime.now(UTC).isoformat(), + "inputs": inputs, + "tasks": records, + } + + +def result_path(root: Path, task_name: str) -> Path: + return root / "task-results" / f"{task_name}.json" + + +def completed_names(root: Path, plan: dict[str, Any]) -> set[str]: + bindings = { + task["name"]: canonical_sha256( + { + "task": task, + "inputs": plan["inputs"], + } + ) + for task in plan["tasks"] + } + completed: set[str] = set() + for name, binding in bindings.items(): + path = result_path(root, name) + if not path.is_file(): + continue + result = json.loads(path.read_text(encoding="utf-8")) + if ( + result.get("schema_version") == RESULT_SCHEMA + and result.get("status") == "passed" + and result.get("binding_sha256") == binding + ): + completed.add(name) + return completed + + +def task_bindings(plan: dict[str, Any]) -> dict[str, str]: + return {task["name"]: canonical_sha256({"task": task, "inputs": plan["inputs"]}) for task in plan["tasks"]} + + +def parse_job_results(root: Path, plan: dict[str, Any]) -> None: + bindings = task_bindings(plan) + known = set(bindings) + for path in sorted((root / "jobs").glob("*/*/result.json")): + result = json.loads(path.read_text(encoding="utf-8")) + task_name = result.get("task_name") + if task_name not in known: + continue + exception = result.get("exception_info") + environment = result.get("environment_setup") + setup = result.get("agent_setup") + execution = result.get("agent_execution") + verifier = result.get("verifier") + passed = ( + exception is None + and isinstance(environment, dict) + and environment.get("finished_at") + and isinstance(setup, dict) + and setup.get("finished_at") + and execution is None + and verifier is None + ) + output = { + "schema_version": RESULT_SCHEMA, + "status": "passed" if passed else "failed", + "task_name": task_name, + "binding_sha256": bindings[task_name], + "trial_result": str(path.relative_to(root)), + "environment_setup": environment, + "agent_setup": setup, + "agent_execution_skipped": execution is None, + "verifier_skipped": verifier is None, + "exception_type": exception.get("exception_type") if isinstance(exception, dict) else None, + "exception_message": exception.get("exception_message") if isinstance(exception, dict) else None, + } + diagnostic = " ".join(str(value or "") for value in (output["exception_type"], output["exception_message"])) + output["failure_class"] = None if passed else classify_setup_failure(path.parent, diagnostic) + destination = result_path(root, task_name) + # Paths are scanned in job-name order, so always replacing the cached + # result preserves the newest attempt. A later pass overwrites an older + # failure, and a repeated failure retains its current diagnosis. + write_json(destination, output) + + +def write_summary(root: Path, plan: dict[str, Any]) -> dict[str, Any]: + passed = completed_names(root, plan) + failed: list[str] = [] + for task in plan["tasks"]: + path = result_path(root, task["name"]) + if path.is_file() and task["name"] not in passed: + failed.append(task["name"]) + failed_classes: dict[str, str] = {} + for name in failed: + result = json.loads(result_path(root, name).read_text(encoding="utf-8")) + failed_classes[name] = str(result.get("failure_class") or "harness_or_integration") + summary = { + "schema_version": SUMMARY_SCHEMA, + "status": "passed" if len(passed) == len(plan["tasks"]) else "partial", + "plan_sha256": canonical_sha256(plan), + "planned": len(plan["tasks"]), + "passed": len(passed), + "failed": len(failed), + "pending": len(plan["tasks"]) - len(passed) - len(failed), + "failed_tasks": sorted(failed), + "infrastructure_failures": sorted( + name for name, failure_class in failed_classes.items() if failure_class == "infrastructure" + ), + "integration_failures": sorted( + name for name, failure_class in failed_classes.items() if failure_class != "infrastructure" + ), + } + write_json(root / "summary.json", summary) + return summary + + +def run_harbor(args: argparse.Namespace, plan: dict[str, Any], pending: list[str]) -> int: + inputs = plan["inputs"] + job_name = f"setup-admission-{datetime.now(UTC).strftime('%Y%m%dT%H%M%S%fZ')}" + mounts = json.dumps( + [ + { + "type": "bind", + "source": str(args.hermetic_runtime), + "target": "/opt/hermes-runtime", + "read_only": True, + "bind": {"create_host_path": False}, + } + ], + separators=(",", ":"), + ) + command = [ + str(args.harbor), + "run", + "--path", + str(args.dataset), + "--n-tasks", + str(len(pending)), + "--agent", + "harbor_hermes_agent:HarborHermesAgent", + "--model", + "openai/setup-admission-stub", + "--ak", + f"commit={inputs['hermes_commit']}", + "--ak", + f"relay_config_path={inputs['relay_config']}", + "--ak", + f"switchyard_bundle_dir={inputs['switchyard_bundle']}", + "--ak", + f"relay_wheel_path={inputs['relay_wheel']}", + "--ak", + f"relay_wheel_sha256={inputs['relay_wheel_sha256']}", + "--ak", + f"relay_architecture={inputs['relay_architecture']}", + "--ak", + f"hermetic_runtime_dir={inputs['hermetic_runtime_root']}", + "--ak", + f"hermetic_runtime_sha256={inputs['hermetic_runtime_sha256']}", + "--mounts", + mounts, + "--install-only", + "--disable-verification", + "--n-concurrent", + str(args.concurrency), + "--n-attempts", + "1", + "--agent-setup-timeout-multiplier", + "6", + "--environment-build-timeout-multiplier", + "6", + "--job-name", + job_name, + "--jobs-dir", + str(args.output / "jobs"), + "--yes", + ] + if args.force_build: + command.append("--force-build") + if args.preserve_containers: + command.append("--no-delete") + for task_name in pending: + command.extend(["--include-task-name", task_name]) + env = os.environ.copy() + agent_path = Path(__file__).resolve().parents[1] / "agents" + env["PYTHONPATH"] = f"{agent_path}{os.pathsep}{env.get('PYTHONPATH', '')}".rstrip(os.pathsep) + with (args.output / "admission.log").open("a", encoding="utf-8") as log: + log.write(f"[{datetime.now(UTC).isoformat()}] starting {len(pending)} task(s)\n") + log.flush() + process = subprocess.run(command, env=env, stdout=log, stderr=subprocess.STDOUT) + log.write(f"[{datetime.now(UTC).isoformat()}] Harbor exit={process.returncode}\n") + return process.returncode + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--runtime-root", type=Path, required=True) + parser.add_argument("--hermetic-runtime", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--harbor", type=Path, required=True) + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--batch-size", type=int, default=89) + parser.add_argument("--max-infra-attempts", type=int, default=4) + parser.add_argument("--backoff-seconds", type=float, default=15) + parser.add_argument( + "--force-build", + action=argparse.BooleanOptionalAction, + default=True, + help="exercise Docker's task-image build path even when an image already exists", + ) + parser.add_argument( + "--preserve-containers", + action=argparse.BooleanOptionalAction, + default=False, + help="retain admission containers for debugging (off by default)", + ) + parser.add_argument( + "--task-name", + action="append", + default=[], + help="Run only this task in the immutable all-89 plan; repeat as needed.", + ) + parser.add_argument("--plan-only", action="store_true") + args = parser.parse_args() + if args.concurrency < 1 or args.batch_size < 1 or args.max_infra_attempts < 1: + parser.error("concurrency, batch size, and infrastructure attempts must be positive") + if args.backoff_seconds < 0: + parser.error("--backoff-seconds cannot be negative") + for name in ("dataset", "runtime_root", "hermetic_runtime", "harbor"): + value = getattr(args, name).expanduser().resolve(strict=True) + setattr(args, name, value) + args.output = args.output.expanduser().resolve() + args.output.mkdir(mode=0o700, parents=True, exist_ok=True) + (args.output / "jobs").mkdir(mode=0o700, exist_ok=True) + (args.output / "task-results").mkdir(mode=0o700, exist_ok=True) + + if harbor_version != "0.18.0": + raise RuntimeError(f"setup admission requires Harbor 0.18.0, found {harbor_version}") + payload = load_payload(args.hermetic_runtime) + tasks = discover_tasks(args.dataset) + known_names = {task.name for task in tasks} + selected_names = set(args.task_name) + unknown_names = selected_names - known_names + if unknown_names: + parser.error(f"unknown --task-name values: {', '.join(sorted(unknown_names))}") + candidate = build_plan(args, tasks, payload) + plan_path = args.output / "plan.json" + if plan_path.exists(): + plan = json.loads(plan_path.read_text(encoding="utf-8")) + comparable = dict(candidate) + comparable["created_at"] = plan.get("created_at") + if plan != comparable: + raise ValueError("existing setup-admission plan does not match current inputs") + else: + plan = candidate + write_json(plan_path, plan) + if args.plan_only: + print(json.dumps(write_summary(args.output, plan), indent=2, sort_keys=True)) + return 0 + + # Import completed/failed results before an infrastructure gate can stop a + # resumed invocation. This preserves every finished task even when the + # machine is not currently healthy enough to launch more work. + parse_job_results(args.output, plan) + clock_preflight = run_clock_preflight() + write_json(args.output / "clock-preflight.json", clock_preflight) + if clock_preflight["status"] != "passed": + write_summary(args.output, plan) + raise RuntimeError( + "wall-clock preflight failed; synchronize the host clock before building " + f"task environments (evidence: {args.output / 'clock-preflight.json'})" + ) + plugin_preflight = run_plugin_compatibility_preflight(plan) + write_json(args.output / "plugin-compatibility.json", plugin_preflight) + if plugin_preflight["status"] != "passed": + write_summary(args.output, plan) + raise RuntimeError( + "Switchyard plugin compatibility preflight failed in the oldest " + "supported task base (evidence: " + f"{args.output / 'plugin-compatibility.json'})" + ) + + passed = completed_names(args.output, plan) + pending = [ + task["name"] + for task in plan["tasks"] + if task["name"] not in passed and (not selected_names or task["name"] in selected_names) + ] + integration_blockers: set[str] = set() + for offset in range(0, len(pending), args.batch_size): + batch = pending[offset : offset + args.batch_size] + remaining = list(batch) + for attempt in range(1, args.max_infra_attempts + 1): + if not remaining: + break + run_harbor(args, plan, remaining) + # Harbor writes result files atomically enough for a completed process; + # a resumed invocation re-scans every prior job before selecting work. + parse_job_results(args.output, plan) + passed = completed_names(args.output, plan) + remaining = [name for name in remaining if name not in passed] + blockers: set[str] = set() + for name in remaining: + path = result_path(args.output, name) + if not path.is_file(): + continue + result = json.loads(path.read_text(encoding="utf-8")) + if result.get("failure_class") == "harness_or_integration": + blockers.add(name) + if blockers: + integration_blockers.update(blockers) + break + if remaining and attempt < args.max_infra_attempts: + delay = args.backoff_seconds * (2 ** (attempt - 1)) + with (args.output / "admission.log").open("a", encoding="utf-8") as log: + log.write( + f"[{datetime.now(UTC).isoformat()}] retrying {len(remaining)} " + f"infrastructure setup failure(s) after {delay:g}s\n" + ) + time.sleep(delay) + if integration_blockers: + break + summary = write_summary(args.output, plan) + print(json.dumps(summary, indent=2, sort_keys=True)) + if selected_names: + passed = completed_names(args.output, plan) + return 0 if selected_names <= passed else 1 + if integration_blockers: + return 20 + return 0 if summary["status"] == "passed" else 75 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh index a2aed79fe..795fc9545 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh +++ b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh @@ -31,10 +31,12 @@ set +a required_values=( EXAMPLE_ROOT TERMINAL_BENCH_RUN_ID TERMINAL_BENCH_RUN_ROOT TERMINAL_BENCH_ADMISSION_ROOT + TERMINAL_BENCH_BOOTSTRAP_ROOT HARBOR_BIN EVAL_PYTHON TBENCH_DATASET_PATH SWITCHYARD_BUNDLE RELAY_WHEEL RELAY_ARCHITECTURE PLUGIN_CONFIG_TEMPLATE TERMINAL_BENCH_SMOKE_EVIDENCE TERMINAL_BENCH_OFFLINE_EVIDENCE PHOENIX_BASE_URL PHOENIX_PROJECT EVAL_COHORT TBENCH_SAMPLE_COUNT TBENCH_CANARY_TASK TBENCH_CONCURRENCY + TBENCH_SETUP_CONCURRENCY TBENCH_SETUP_BATCH_SIZE TBENCH_SETUP_MAX_INFRA_ATTEMPTS TBENCH_PARALLEL_MAX_MEMORY_GB TBENCH_DOCKER_MEMORY_RESERVE_GB TBENCH_MINIMUM_FREE_GB SWITCHYARD_PROVIDER_AUTHORIZATION ) @@ -44,7 +46,8 @@ for name in "${required_values[@]}"; do exit 2 fi done -for name in EXAMPLE_ROOT TERMINAL_BENCH_RUN_ROOT TERMINAL_BENCH_ADMISSION_ROOT HARBOR_BIN EVAL_PYTHON \ +for name in EXAMPLE_ROOT TERMINAL_BENCH_RUN_ROOT TERMINAL_BENCH_ADMISSION_ROOT \ + TERMINAL_BENCH_BOOTSTRAP_ROOT HARBOR_BIN EVAL_PYTHON \ TBENCH_DATASET_PATH SWITCHYARD_BUNDLE RELAY_WHEEL PLUGIN_CONFIG_TEMPLATE \ TERMINAL_BENCH_SMOKE_EVIDENCE TERMINAL_BENCH_OFFLINE_EVIDENCE; do if [[ "${!name}" != /* ]]; then @@ -52,7 +55,8 @@ for name in EXAMPLE_ROOT TERMINAL_BENCH_RUN_ROOT TERMINAL_BENCH_ADMISSION_ROOT H exit 2 fi done -for name in TBENCH_SAMPLE_COUNT TBENCH_CONCURRENCY TBENCH_PARALLEL_MAX_MEMORY_GB \ +for name in TBENCH_SAMPLE_COUNT TBENCH_CONCURRENCY TBENCH_SETUP_CONCURRENCY \ + TBENCH_SETUP_BATCH_SIZE TBENCH_SETUP_MAX_INFRA_ATTEMPTS TBENCH_PARALLEL_MAX_MEMORY_GB \ TBENCH_DOCKER_MEMORY_RESERVE_GB TBENCH_MINIMUM_FREE_GB; do if [[ ! "${!name}" =~ ^[1-9][0-9]*$ ]]; then echo "Phase 2 capacity value must be a positive integer: $name" >&2 diff --git a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py index 39dcbd18d..73c4e61df 100644 --- a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py @@ -34,7 +34,7 @@ def make_args(tmp_path: Path, *, error_type: str = "") -> argparse.Namespace: relay_wheel_sha256="a" * 64, hermes_repository="https://github.com/bbednarski9/hermes-agent.git", hermes_commit="efb63e714abc436af88af9b0d6734751c199aa6d", - switchyard_commit="5d9d3292d6154e44d50295d0d4a3fd4f144f2528", + switchyard_commit="8daac03edf8544144833af1fd009b3da737715bc", session_handle="phase1-session", started_at=1.0, error_type=error_type, diff --git a/examples/harbor-hermes-switchyard/tests/test_config_contract.py b/examples/harbor-hermes-switchyard/tests/test_config_contract.py deleted file mode 100644 index 463700429..000000000 --- a/examples/harbor-hermes-switchyard/tests/test_config_contract.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from __future__ import annotations - -import asyncio -import importlib.util -import re -import sys -import tomllib -from pathlib import Path -from unittest.mock import AsyncMock, patch - -from harbor.agents.installed.hermes import Hermes - -EXAMPLE_ROOT = Path(__file__).resolve().parents[1] - - -def render_template(**values: str) -> dict: - text = (EXAMPLE_ROOT / "config" / "plugins.toml.in").read_text(encoding="utf-8") - defaults = { - "HERMES_COMMIT": "efb63e714abc436af88af9b0d6734751c199aa6d", - "OPENINFERENCE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", - "PHOENIX_PROJECT": "phase1-test", - "EVAL_COHORT": "phase1-test", - } - defaults.update(values) - for key, value in defaults.items(): - text = text.replace(f"@{key}@", value) - assert not re.search(r"@[A-Z0-9_]+@", text) - return tomllib.loads(text) - - -def test_config_uses_static_schema_v3_and_one_standard_dynamic_plugin() -> None: - config = render_template() - assert config["version"] == 1 - components = {item["kind"]: item for item in config["components"]} - assert components["pricing"]["enabled"] is True - assert components["observability"]["config"]["version"] == 3 - assert "dynamic_plugins" not in config - assert len(config["plugins"]["dynamic"]) == 1 - plugin = config["plugins"]["dynamic"][0] - assert plugin["manifest"].endswith("/nvidia.switchyard/relay-plugin.toml") - algorithm = plugin["config"]["algorithm"] - assert algorithm == { - "kind": "llm_classifier", - "classifier_target": "weak", - "weak_target": "weak", - "strong_target": "strong", - "base_threshold": 0.5, - "recent_turn_window": 0, - "session_affinity": True, - "message_hash_fallback": True, - } - assert plugin["config"]["default_targets"] == {"openai_chat": "strong"} - assert set(plugin["config"]["targets"]) == {"strong", "weak"} - for target in plugin["config"]["targets"].values(): - assert target["header_env"] == {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"} - assert target["drop_caller_extra_body"] is True - - -def test_config_contains_no_literal_provider_headers_or_credentials() -> None: - config = render_template() - - def walk(value: object) -> None: - if isinstance(value, dict): - assert "headers" not in value - for nested in value.values(): - walk(nested) - elif isinstance(value, list): - for nested in value: - walk(nested) - - walk(config) - - -def test_pricing_does_not_duplicate_relay_generated_aliases() -> None: - config = render_template() - entries = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] - assert [entry["model_id"] for entry in entries] == [ - "aws/anthropic/bedrock-claude-opus-4-6", - "aws/anthropic/bedrock-claude-sonnet-4-6", - ] - assert all("aliases" not in entry for entry in entries) - - -def test_pricing_uses_nonzero_claude_46_list_rates() -> None: - config = render_template() - entries = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] - rates = {entry["model_id"]: entry["rates"] for entry in entries} - assert rates["aws/anthropic/bedrock-claude-opus-4-6"] == { - "input_per_million": 5.0, - "output_per_million": 25.0, - "cache_read_per_million": 0.5, - "cache_write_per_million": 6.25, - } - assert rates["aws/anthropic/bedrock-claude-sonnet-4-6"] == { - "input_per_million": 3.0, - "output_per_million": 15.0, - "cache_read_per_million": 0.3, - "cache_write_per_million": 3.75, - } - - -def test_switchyard_models_are_distinct_from_fail_closed_hermes_caller() -> None: - config = render_template() - plugin = config["plugins"]["dynamic"][0] - provider_models = {target["model"] for target in plugin["config"]["targets"].values()} - assert provider_models == { - "aws/anthropic/bedrock-claude-opus-4-6", - "aws/anthropic/bedrock-claude-sonnet-4-6", - } - observability = next(item for item in config["components"] if item["kind"] == "observability") - assert observability["config"]["atif"]["model_name"] == "ollama-route-stub" - assert "ollama-route-stub" not in provider_models - - -def test_task_runner_defaults_to_production_x86_64_architecture() -> None: - runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") - assert 'relay_architecture="${RELAY_ARCHITECTURE:-x86_64}"' in runner - assert '--relay-architecture "$relay_architecture"' in runner - assert 'SWITCHYARD_TARGET_ARCHITECTURE="$relay_architecture"' in runner - assert '--ak "relay_architecture=$relay_architecture"' in runner - - -def test_task_runner_defaults_to_inference_hub_tiers_and_fail_closed_caller() -> None: - runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") - assert "STRONG_MODEL" not in runner - assert "WEAK_MODEL" not in runner - assert 'plugin_config_template="${PLUGIN_CONFIG_TEMPLATE:-$example_root/config/plugins.toml.in}"' in runner - assert '--model "openai/$hermes_caller_model"' in runner - assert 'fail_closed_openai_base_url="http://127.0.0.1:9/v1"' in runner - assert '--ae "OPENAI_BASE_URL=$fail_closed_openai_base_url"' in runner - - -def test_task_runner_projects_provider_authorization_by_read_only_mount() -> None: - runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") - assert '--ae "$upstream_auth_env=' not in runner - assert 'host_temporary_root="$(cd "${TMPDIR:-/tmp}" && pwd -P)"' in runner - assert '"$(dirname "$run_root")/.phase2-secret.' not in runner - assert '"$run_root/"*)' in runner - assert 'provider_authorization_target="/run/secrets/switchyard-provider-authorization"' in runner - assert '"read_only": True' in runner - assert '"bind": {"create_host_path": False}' in runner - assert '--mounts "$mounts_json"' in runner - assert "'OPENAI_API_KEY=${OPENAI_API_KEY}'" in runner - - -def test_agent_reads_provider_authorization_inside_container_only() -> None: - path = EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py" - spec = importlib.util.spec_from_file_location("phase2_secret_agent", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - agent = object.__new__(module.HarborHermesAgent) - agent._load_provider_authorization = True - with patch.object(Hermes, "exec_as_agent", new_callable=AsyncMock) as parent: - asyncio.run(agent.exec_as_agent(object(), "hermes --yolo chat", env={"SAFE": "value"})) - command = parent.await_args.args[-1] - assert "cat -- /run/secrets/switchyard-provider-authorization" in command - assert 'export SWITCHYARD_PROVIDER_AUTHORIZATION="$(cat --' in command - assert parent.await_args.kwargs["env"] == {"SAFE": "value"} - - -def test_agent_install_retries_transient_apt_failures() -> None: - agent = (EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py").read_text(encoding="utf-8") - assert "for attempt in 1 2 3; do " in agent - assert "apt-get update && apt-get install -y --no-install-recommends " in agent - assert "sleep $((attempt * 5))" in agent - - -def test_phase2_environment_template_consolidates_secret_without_legacy_file() -> None: - template = (EXAMPLE_ROOT / ".env.example").read_text(encoding="utf-8") - assert "SWITCHYARD_PROVIDER_AUTHORIZATION='Bearer replace-with-provider-token'" in template - assert "INFERENCE_SECRETS_FILE" not in template - assert "NV_INFERENCEHUB_KEY" not in template - assert "NV_INFERENCEHUB_ENDPOINT" not in template - assert "STRONG_MODEL" not in template - assert "WEAK_MODEL" not in template - assert "UPSTREAM_BASE_URL" not in template - assert ".env" in (EXAMPLE_ROOT / ".gitignore").read_text(encoding="utf-8") - - -def test_readme_uses_admissions_instead_of_regression_smokes() -> None: - readme = (EXAMPLE_ROOT / "README.md").read_text(encoding="utf-8") - assert "run_regression_smokes.sh" not in readme - assert "PHASE1_EVIDENCE_ROOT" not in readme - assert "INFERENCE_SECRETS_FILE" not in readme - assert "all-89 no-token admission" in readme.lower() - assert "Docker offline runtime admission" in readme - - -def test_phase2_runner_requires_and_uses_the_local_dataset_export() -> None: - runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") - assert 'tbench_dataset_path="${TBENCH_DATASET_PATH:-}"' in runner - assert 'dataset_args=(--path "$tbench_dataset_path")' in runner - assert '"${dataset_args[@]}"' in runner - assert "Phase 2 requires TBENCH_DATASET_PATH" in runner diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 774afdd1c..97494112e 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -290,6 +290,60 @@ def test_phase2_launcher_is_local_dataset_only() -> None: assert "INFERENCE_SECRETS_FILE" not in launcher +def test_coordinator_owns_bounded_force_build_setup_lane( + tmp_path: Path, +) -> None: + module = load_coordinator() + args = argparse.Namespace( + run_root=tmp_path / "run", + python_bin=tmp_path / "python", + setup_admission_runner=tmp_path / "run_setup_admission.py", + dataset_root=tmp_path / "dataset", + setup_runtime=tmp_path / "setup-runtime", + hermetic_runtime=tmp_path / "hermetic-runtime", + harbor_bin=tmp_path / "harbor", + setup_concurrency=2, + setup_batch_size=89, + setup_max_infra_attempts=4, + backoff_seconds=0, + hermetic_runtime_payload={"content_sha256": "a" * 64}, + ) + args.run_root.mkdir() + captured: list[str] = [] + + def fake_run(command: list[str], **_: object) -> SimpleNamespace: + captured.extend(command) + output = Path(command[command.index("--output") + 1]) + output.mkdir(parents=True) + (output / "summary.json").write_text( + json.dumps({"status": "passed", "planned": 2, "passed": 2}), + encoding="utf-8", + ) + return SimpleNamespace(returncode=0) + + tasks = [module.Task(1, "one", 2), module.Task(2, "two", 2)] + with patch.object(module.subprocess, "run", side_effect=fake_run): + assert module.CohortRunner(args, tasks).provision_environments() + assert captured[captured.index("--concurrency") + 1] == "2" + assert captured[captured.index("--batch-size") + 1] == "89" + assert "--force-build" in captured + assert "--no-preserve-containers" in captured + rendered = " ".join(captured) + assert "SWITCHYARD_PROVIDER_AUTHORIZATION" not in rendered + assert "Bearer " not in rendered + + +def test_provider_attempt_rebuilds_from_cached_layers_and_uses_hermetic_runtime() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + coordinator = (EXAMPLE_ROOT / "scripts" / "run_phase2_cohort.py").read_text(encoding="utf-8") + assert 'HARBOR_FORCE_BUILD": "true"' in coordinator + assert 'HERMETIC_RUNTIME_DIR": str(self.args.hermetic_runtime)' in coordinator + assert 'harbor_force_build="${HARBOR_FORCE_BUILD:-true}"' in runner + assert "harbor_build_args+=(--force-build)" in runner + assert '--ak "hermetic_runtime_sha256=$hermetic_runtime_sha256"' in runner + assert '"target": "/opt/hermes-runtime"' in runner + + def test_plugin_contract_owns_routes_and_authorization_name() -> None: module = load_coordinator() contract = module.plugin_contract(EXAMPLE_ROOT / "config" / "plugins.toml.in") diff --git a/examples/harbor-hermes-switchyard/tests/test_setup_admission.py b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py new file mode 100644 index 000000000..609932869 --- /dev/null +++ b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py @@ -0,0 +1,338 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import subprocess +from pathlib import Path +from types import ModuleType + +import pytest + +EXAMPLE_ROOT = Path(__file__).resolve().parents[1] + + +def load_module(name: str, path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +agent_module = load_module( + "setup_admission_agent", + EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py", +) +admission_module = load_module( + "setup_admission_runner", + EXAMPLE_ROOT / "scripts" / "run_setup_admission.py", +) +builder_module = load_module( + "hermetic_runtime_builder", + EXAMPLE_ROOT / "scripts" / "build_hermetic_runtime.py", +) + + +def make_payload(root: Path, *, digest: str = "a" * 64) -> dict[str, object]: + for relative in ( + "bin/hermes", + "bin/python", + "bin/uv", + ): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("stub", encoding="utf-8") + (root / "hermes-agent-src" / "venv").mkdir(parents=True) + ca_bundle = root / agent_module._HERMETIC_CA_BUNDLE_RELATIVE + ca_bundle.parent.mkdir(parents=True, exist_ok=True) + ca_bundle.write_text("test CA bundle", encoding="utf-8") + marker = { + "schema_version": agent_module._HERMETIC_RUNTIME_SCHEMA, + "status": "passed", + "content_sha256": digest, + "hermes_commit": "b" * 40, + "relay_wheel_sha256": "c" * 64, + "relay_architecture": "aarch64", + } + (root / "payload.json").write_text(json.dumps(marker), encoding="utf-8") + return marker + + +def test_hermetic_runtime_contract_accepts_bound_payload(tmp_path: Path) -> None: + marker = make_payload(tmp_path) + actual = agent_module._load_hermetic_runtime( + tmp_path, + expected_digest=str(marker["content_sha256"]), + hermes_commit=str(marker["hermes_commit"]), + relay_wheel_sha256=str(marker["relay_wheel_sha256"]), + relay_architecture=str(marker["relay_architecture"]), + ) + assert actual == marker + + +def test_hermetic_runtime_contract_rejects_changed_architecture(tmp_path: Path) -> None: + marker = make_payload(tmp_path) + with pytest.raises(ValueError, match="metadata mismatch"): + agent_module._load_hermetic_runtime( + tmp_path, + expected_digest=str(marker["content_sha256"]), + hermes_commit=str(marker["hermes_commit"]), + relay_wheel_sha256=str(marker["relay_wheel_sha256"]), + relay_architecture="x86_64", + ) + + +def test_hermetic_runtime_readiness_retries_nested_entrypoints(tmp_path: Path) -> None: + runtime = tmp_path / "runtime" + bin_dir = runtime / "bin" + bin_dir.mkdir(parents=True) + counter = tmp_path / "attempts" + (bin_dir / "python").write_text( + "#!/bin/sh\n" + f'counter="{counter}"\n' + 'attempts="$(cat "$counter" 2>/dev/null || printf 0)"\n' + 'attempts="$((attempts + 1))"\n' + 'printf "%s" "$attempts" > "$counter"\n' + '[ "$attempts" -ge 3 ]\n', + encoding="utf-8", + ) + (bin_dir / "hermes").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + os.chmod(bin_dir / "python", 0o755) + os.chmod(bin_dir / "hermes", 0o755) + command = agent_module._hermetic_runtime_readiness_command(str(runtime), attempts=4, delay_seconds=0) + completed = subprocess.run(["bash", "-c", f"set -euo pipefail; {command}"], check=False) + assert completed.returncode == 0 + assert counter.read_text(encoding="utf-8") == "3" + + +def test_setup_admission_binds_agent_source() -> None: + expected = EXAMPLE_ROOT / "agents" / "harbor_hermes_agent.py" + assert admission_module.SETUP_AGENT_PATH == expected + assert admission_module.sha256_file(expected) == agent_module._sha256(expected) + + +def test_hermetic_runtime_requires_portable_ca_bundle(tmp_path: Path) -> None: + marker = make_payload(tmp_path) + (tmp_path / agent_module._HERMETIC_CA_BUNDLE_RELATIVE).unlink() + with pytest.raises(FileNotFoundError, match="hermetic runtime is incomplete"): + agent_module._load_hermetic_runtime( + tmp_path, + expected_digest=str(marker["content_sha256"]), + hermes_commit=str(marker["hermes_commit"]), + relay_wheel_sha256=str(marker["relay_wheel_sha256"]), + relay_architecture=str(marker["relay_architecture"]), + ) + + +def test_payload_tree_digest_ignores_its_marker(tmp_path: Path) -> None: + content = tmp_path / "bin" / "python" + content.parent.mkdir(parents=True) + content.write_text("payload", encoding="utf-8") + first = builder_module.sha256_tree(tmp_path) + (tmp_path / "payload.json").write_text("first", encoding="utf-8") + assert builder_module.sha256_tree(tmp_path) == first + (tmp_path / "payload.json").write_text("second", encoding="utf-8") + assert builder_module.sha256_tree(tmp_path) == first + content.write_text("changed", encoding="utf-8") + assert builder_module.sha256_tree(tmp_path) != first + + +def test_admission_rejects_tampered_hermetic_runtime(tmp_path: Path) -> None: + content = tmp_path / "bin" / "python" + content.parent.mkdir(parents=True) + content.write_text("payload", encoding="utf-8") + marker = { + "schema_version": admission_module.PAYLOAD_SCHEMA, + "status": "passed", + "content_sha256": admission_module.hermetic_content_sha256(tmp_path), + } + (tmp_path / "payload.json").write_text(json.dumps(marker), encoding="utf-8") + assert admission_module.load_payload(tmp_path) == marker + content.write_text("tampered", encoding="utf-8") + with pytest.raises(ValueError, match="does not match"): + admission_module.load_payload(tmp_path) + + +def test_payload_builder_forwards_non_secret_version_pins() -> None: + source = (EXAMPLE_ROOT / "scripts" / "build_hermetic_runtime.py").read_text(encoding="utf-8") + assert 'f"UV_VERSION={UV_VERSION}"' in source + assert 'f"PYTHON_VERSION={PYTHON_VERSION}"' in source + assert 'f"RELAY_WHEEL_NAME={relay_wheel.name}"' in source + + +def test_completed_result_is_invalidated_by_plan_input_change(tmp_path: Path) -> None: + plan = { + "inputs": {"concurrency": 4, "hermetic_runtime_sha256": "a" * 64}, + "tasks": [{"name": "task-one", "task_sha256": "b" * 64}], + } + binding = admission_module.task_bindings(plan)["task-one"] + results = tmp_path / "task-results" + results.mkdir() + (results / "task-one.json").write_text( + json.dumps( + { + "schema_version": admission_module.RESULT_SCHEMA, + "status": "passed", + "binding_sha256": binding, + } + ), + encoding="utf-8", + ) + assert admission_module.completed_names(tmp_path, plan) == {"task-one"} + plan["inputs"]["concurrency"] = 5 + assert admission_module.completed_names(tmp_path, plan) == set() + + +def test_job_result_import_keeps_newest_attempt(tmp_path: Path) -> None: + plan = { + "inputs": {"concurrency": 4}, + "tasks": [{"name": "task-one", "task_sha256": "b" * 64}], + } + for job_name, message in (("job-001", "old failure"), ("job-002", "new failure")): + trial = tmp_path / "jobs" / job_name / "trial-one" + trial.mkdir(parents=True) + (trial / "result.json").write_text( + json.dumps( + { + "task_name": "task-one", + "exception_info": { + "exception_type": "RuntimeError", + "exception_message": message, + }, + "environment_setup": None, + "agent_setup": None, + "agent_execution": None, + "verifier": None, + } + ), + encoding="utf-8", + ) + (tmp_path / "task-results").mkdir() + admission_module.parse_job_results(tmp_path, plan) + result = json.loads((tmp_path / "task-results" / "task-one.json").read_text()) + assert result["exception_message"] == "new failure" + + +def test_clock_preflight_rejects_remote_time_drift() -> None: + evidence = admission_module.evaluate_clock_preflight( + host_epoch=1_000.0, + docker_epoch=1_001.0, + reference_epoch=4_611.0, + ) + assert evidence["status"] == "failed" + assert evidence["host_reference_offset_seconds"] == 3_611.0 + assert evidence["docker_host_offset_seconds"] == 1.0 + + +def test_clock_preflight_accepts_small_offsets() -> None: + evidence = admission_module.evaluate_clock_preflight( + host_epoch=1_000.0, + docker_epoch=1_001.0, + reference_epoch=1_002.0, + ) + assert evidence["status"] == "passed" + + +def test_plugin_compatibility_uses_oldest_supported_base_without_secrets( + tmp_path: Path, +) -> None: + plan = { + "inputs": { + "relay_architecture": "aarch64", + "switchyard_bundle": str(tmp_path / "switchyard"), + } + } + command = admission_module.plugin_compatibility_command(plan) + assert "python:3.11-bullseye" in command + assert "linux/arm64" in command + assert "nemo_relay_register_plugin" in command[-1] + rendered = " ".join(command) + assert "SWITCHYARD_PROVIDER_AUTHORIZATION" not in rendered + assert "Bearer " not in rendered + + +def test_harbor_command_uses_install_only_without_provider_secret( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + output = tmp_path / "output" + output.mkdir() + payload = tmp_path / "payload" + payload.mkdir() + harbor = tmp_path / "harbor" + harbor.write_text("stub", encoding="utf-8") + args = argparse.Namespace( + harbor=harbor, + dataset=tmp_path / "dataset", + hermetic_runtime=payload, + output=output, + concurrency=6, + force_build=True, + preserve_containers=False, + ) + plan = { + "inputs": { + "hermes_commit": "b" * 40, + "relay_config": str(tmp_path / "plugins.toml"), + "switchyard_bundle": str(tmp_path / "switchyard"), + "relay_wheel": str(tmp_path / "relay.whl"), + "relay_wheel_sha256": "c" * 64, + "relay_architecture": "aarch64", + "hermetic_runtime_root": str(payload), + "hermetic_runtime_sha256": "d" * 64, + } + } + captured: list[str] = [] + + class Completed: + returncode = 0 + + def fake_run(command: list[str], **_: object) -> Completed: + captured.extend(command) + return Completed() + + monkeypatch.setattr(admission_module.subprocess, "run", fake_run) + assert admission_module.run_harbor(args, plan, ["one", "two"]) == 0 + assert "--install-only" in captured + assert "--disable-verification" in captured + assert "--force-build" in captured + assert "--no-delete" not in captured + assert captured.count("--include-task-name") == 2 + rendered = " ".join(captured) + assert "SWITCHYARD_PROVIDER_AUTHORIZATION" not in rendered + assert "provider-authorization" not in rendered + + +def test_setup_failure_classifies_transient_downloads_for_retry(tmp_path: Path) -> None: + root = tmp_path / "admission" + (root / "task-results").mkdir(parents=True) + result = root / "jobs" / "job" / "trial" / "result.json" + result.parent.mkdir(parents=True) + result.write_text( + json.dumps( + { + "task_name": "one", + "exception_info": { + "exception_type": "DockerBuildError", + "exception_message": "TLS handshake timeout contacting registry-1.docker.io", + }, + "environment_setup": None, + "agent_setup": None, + "agent_execution": None, + "verifier": None, + } + ), + encoding="utf-8", + ) + plan = { + "inputs": {"concurrency": 2}, + "tasks": [{"name": "one", "task_sha256": "a" * 64}], + } + admission_module.parse_job_results(root, plan) + parsed = json.loads(admission_module.result_path(root, "one").read_text()) + assert parsed["status"] == "failed" + assert parsed["failure_class"] == "infrastructure" From 658eaf43a1266a9f5d93b79c9ec053ef34226f7a Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 00:10:56 -0600 Subject: [PATCH 13/34] fix(example): handle empty validation expectations Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/run_terminal_bench.sh | 6 +++--- .../harbor-hermes-switchyard/tests/test_phase2_cohort.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index 279c2c60d..dc73995d6 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -235,10 +235,8 @@ values = json.load(open(sys.argv[1]))["routing"] print("\n".join(sorted({urlsplit(value).hostname for key, value in values.items() if key.endswith("_base_url")}))) ' "$run_root/runtime/provenance.json") agent_kwargs=() -validation_expectations=() if [[ "$inject_post_response_failure" == "true" ]]; then agent_kwargs+=(--ak inject_post_response_failure=true) - validation_expectations+=(--expect-late-failure) elif [[ "$inject_post_response_failure" != "false" ]]; then echo "INJECT_POST_RESPONSE_FAILURE must be true or false" >&2 exit 2 @@ -320,8 +318,10 @@ validation_args=( --scan-root "$run_root/jobs/$job_name" --secret-env "$upstream_auth_env" --output "$artifact_root/validation.json" - "${validation_expectations[@]}" ) +if [[ "$inject_post_response_failure" == "true" ]]; then + validation_args+=(--expect-late-failure) +fi "$python_bin" "${validation_args[@]}" >"$run_root/validation.log" "$python_bin" "$example_root/scripts/upload_openinference.py" \ diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 97494112e..194595245 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -344,6 +344,13 @@ def test_provider_attempt_rebuilds_from_cached_layers_and_uses_hermetic_runtime( assert '"target": "/opt/hermes-runtime"' in runner +def test_runner_does_not_expand_an_empty_validation_expectations_array() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + assert "validation_expectations=()" not in runner + assert '"${validation_expectations[@]}"' not in runner + assert "validation_args+=(--expect-late-failure)" in runner + + def test_plugin_contract_owns_routes_and_authorization_name() -> None: module = load_coordinator() contract = module.plugin_contract(EXAMPLE_ROOT / "config" / "plugins.toml.in") From 456fb2173ce7075b253539f06520ee40eaaf57af Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 02:31:58 -0600 Subject: [PATCH 14/34] fix(example): retry unresponsive providers Signed-off-by: Bryan Bednarski --- .../harbor-hermes-switchyard/scripts/run_phase2_cohort.py | 1 + .../harbor-hermes-switchyard/tests/test_phase2_cohort.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index 5096fdebe..fc8c0289e 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -50,6 +50,7 @@ "network is unreachable", "no space left on device", "phoenix upload", + "provider has been unresponsive", "registry-1.docker.io", "temporary failure in name resolution", "tls handshake timeout", diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 194595245..8add5a040 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -146,6 +146,12 @@ def test_failure_classifier_retries_only_known_infrastructure_failures() -> None assert module.classify_failure("TLS handshake timeout contacting registry-1.docker.io") == "infrastructure" assert module.classify_failure("ConnectError: Error getting dataset terminal-bench@2.0") == "infrastructure" assert module.classify_failure("Command failed (exit 100): apt-get update && apt-get install") == "infrastructure" + assert ( + module.classify_failure( + "Provider has been unresponsive (no response received) for 11 consecutive stale attempts" + ) + == "infrastructure" + ) assert module.classify_failure("receipt did not prove plugin close") == "harness_or_integration" From 20801f46d492ca841caaab681b161194bc0db59a Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 03:13:41 -0600 Subject: [PATCH 15/34] fix(examples): retry provider request timeouts Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py | 1 + examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index fc8c0289e..45eab945e 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -51,6 +51,7 @@ "no space left on device", "phoenix upload", "provider has been unresponsive", + "provider returned http 408", "registry-1.docker.io", "temporary failure in name resolution", "tls handshake timeout", diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 8add5a040..40c5a16e4 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -152,6 +152,8 @@ def test_failure_classifier_retries_only_known_infrastructure_failures() -> None ) == "infrastructure" ) + assert module.classify_failure("trusted fallback: provider returned HTTP 408") == "infrastructure" + assert module.classify_failure("provider returned HTTP 400") == "harness_or_integration" assert module.classify_failure("receipt did not prove plugin close") == "harness_or_integration" From c2dafe1642f43ec076528cf1b7083364a2d237cc Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 04:45:21 -0600 Subject: [PATCH 16/34] fix(examples): classify nested Harbor failures Signed-off-by: Bryan Bednarski --- .../scripts/run_phase2_cohort.py | 6 ++++- .../tests/test_phase2_cohort.py | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index 45eab945e..99eb201ce 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -181,7 +181,11 @@ def classify_attempt_failure(log_text: str, attempt: Path) -> str: if classify_failure(log_text) == "infrastructure": return "infrastructure" - for path in sorted(attempt.rglob("*.log")): + diagnostic_paths = [ + *attempt.rglob("*.log"), + *attempt.rglob("result.json"), + ] + for path in sorted(diagnostic_paths): try: with path.open(encoding="utf-8", errors="replace") as stream: while chunk := stream.read(1024 * 1024): diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 40c5a16e4..f595d356f 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -171,6 +171,28 @@ def test_failure_classifier_reads_nested_harbor_logs(tmp_path: Path) -> None: assert module.classify_attempt_failure("expected one direct Hermes result, found 0", attempt) == "infrastructure" +def test_failure_classifier_reads_nested_harbor_result_json(tmp_path: Path) -> None: + module = load_coordinator() + attempt = tmp_path / "attempt" + result = attempt / "jobs" / "task" / "result.json" + result.parent.mkdir(parents=True) + result.write_text( + json.dumps( + { + "exception_info": { + "exception_message": ( + "Provider has been unresponsive (no response received) " + "for 10 consecutive stale attempts" + ) + } + } + ), + encoding="utf-8", + ) + + assert module.classify_attempt_failure("invalid direct result status", attempt) == "infrastructure" + + def test_smoke_evidence_is_bound_to_exact_local_dataset(tmp_path: Path) -> None: module = load_coordinator() dataset = tmp_path / "dataset" From 276464b18d69fd6e6bd9fefd8b8c0b5350ca3064 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 05:18:37 -0600 Subject: [PATCH 17/34] fix(examples): fail close delegated Hermes workers Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/run_terminal_bench.sh | 2 ++ .../scripts/smoke_phase2_dataset.py | 2 ++ .../harbor-hermes-switchyard/tests/test_phase2_cohort.py | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index dc73995d6..b8ed7b4d0 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -269,6 +269,8 @@ fi "${agent_kwargs[@]}" \ --ae 'OPENAI_API_KEY=${OPENAI_API_KEY}' \ --ae "OPENAI_BASE_URL=$fail_closed_openai_base_url" \ + --ae 'OPENROUTER_API_KEY=relay-intercepted' \ + --ae "OPENROUTER_BASE_URL=$fail_closed_openai_base_url" \ --mounts "$mounts_json" \ "${agent_hosts[@]}" \ --artifact /logs/agent/direct-hermes \ diff --git a/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py index 6d9252e1e..ee33026c6 100755 --- a/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py +++ b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py @@ -137,6 +137,8 @@ def deny_network(*_args: Any, **_kwargs: Any) -> None: env={ "OPENAI_API_KEY": "${OPENAI_API_KEY}", "OPENAI_BASE_URL": "http://127.0.0.1:9/v1", + "OPENROUTER_API_KEY": "relay-intercepted", + "OPENROUTER_BASE_URL": "http://127.0.0.1:9/v1", }, ) ], diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index f595d356f..119f733c6 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -372,6 +372,15 @@ def test_provider_attempt_rebuilds_from_cached_layers_and_uses_hermetic_runtime( assert "harbor_build_args+=(--force-build)" in runner assert '--ak "hermetic_runtime_sha256=$hermetic_runtime_sha256"' in runner assert '"target": "/opt/hermes-runtime"' in runner + assert "--ae 'OPENROUTER_API_KEY=relay-intercepted'" in runner + assert '--ae "OPENROUTER_BASE_URL=$fail_closed_openai_base_url"' in runner + + +def test_all_task_smoke_fail_closes_parent_and_delegated_provider_urls() -> None: + smoke = (EXAMPLE_ROOT / "scripts" / "smoke_phase2_dataset.py").read_text(encoding="utf-8") + assert '"OPENAI_BASE_URL": "http://127.0.0.1:9/v1"' in smoke + assert '"OPENROUTER_API_KEY": "relay-intercepted"' in smoke + assert '"OPENROUTER_BASE_URL": "http://127.0.0.1:9/v1"' in smoke def test_runner_does_not_expand_an_empty_validation_expectations_array() -> None: From 4ece06272e2f9d248d480d2b64de4ac39f2a9915 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 07:00:41 -0600 Subject: [PATCH 18/34] fix(example): accept verified agent timeout nonpasses Signed-off-by: Bryan Bednarski --- .../scripts/validate_run.py | 38 +++++++++++++++++++ .../tests/test_validation_contract.py | 20 ++++++++++ 2 files changed, 58 insertions(+) diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py index b2a877405..56c5f17d6 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_run.py +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -43,6 +43,31 @@ def read_benchmark_passed(value: dict[str, Any]) -> bool | None: return None +def is_verifier_backed_agent_timeout_nonpass( + direct_result: dict[str, Any], harbor_result: dict[str, Any] +) -> bool: + """Recognize a completed benchmark non-pass caused by Harbor's agent deadline. + + Harbor cancels the agent subprocess at its configured deadline, so the + direct adapter records ``CancelledError`` and cannot emit a final response + or terminal ATIF/AGENT span. The outcome is complete only when Harbor + independently records ``AgentTimeoutError`` and the verifier produces a + non-passing reward. Other cancellation and missing-artifact cases remain + integration failures. + """ + error = direct_result.get("error") + exception = harbor_result.get("exception_info") + return ( + direct_result.get("status") == "failed" + and isinstance(error, dict) + and error.get("phase") == "agent" + and error.get("type") == "CancelledError" + and isinstance(exception, dict) + and exception.get("exception_type") == "AgentTimeoutError" + and read_benchmark_passed(harbor_result) is False + ) + + def validate_harbor_job_config(job_dir: Path) -> tuple[dict[str, float], list[str]]: """Validate the timeout multipliers serialized by Harbor for this job.""" errors: list[str] = [] @@ -432,6 +457,7 @@ def main() -> int: harbor_results: list[Path] = [] harbor_timeout_multipliers: dict[str, float] = {} benchmark_passed: bool | None = None + harbor_result: dict[str, Any] = {} if args.harbor_job_dir: harbor_timeout_multipliers, timeout_errors = validate_harbor_job_config(args.harbor_job_dir) errors.extend(timeout_errors) @@ -444,6 +470,17 @@ def main() -> int: harbor_result = read_json(harbor_results[0]) benchmark_passed = read_benchmark_passed(harbor_result) + terminal_timeout_nonpass = is_verifier_backed_agent_timeout_nonpass(result, harbor_result) + if terminal_timeout_nonpass: + tolerated_errors = { + "missing ATIF trajectory", + "invalid direct result status: 'failed'", + "direct result has no normalized final response", + } + if "LLM" in openinference_evidence["span_kinds"]: + tolerated_errors.add("OpenInference artifact does not contain both AGENT and LLM span kinds") + errors = [error for error in errors if error not in tolerated_errors] + validation = { "schema_version": SCHEMA_VERSION, "status": "passed" if not errors else "failed", @@ -452,6 +489,7 @@ def main() -> int: "harbor_trial_count": len(harbor_results) if args.harbor_job_dir else None, "harbor_timeout_multipliers": harbor_timeout_multipliers, "benchmark_task_passed": benchmark_passed, + "terminal_agent_timeout_nonpass": terminal_timeout_nonpass, "atof_event_count": event_count, "atif_trajectory_count": len(atif_files), "openinference": openinference_evidence, diff --git a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py index 248630b10..564e6691c 100644 --- a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py @@ -31,6 +31,26 @@ def test_harbor_018_numeric_reward_is_normalized() -> None: assert module.read_benchmark_passed({"verifier_result": {"rewards": {"reward": 1.0}}}) is True +def test_verifier_backed_agent_timeout_is_a_completed_nonpass() -> None: + module = load_validator() + direct_result = { + "status": "failed", + "error": {"phase": "agent", "type": "CancelledError"}, + } + harbor_result = { + "exception_info": {"exception_type": "AgentTimeoutError"}, + "verifier_result": {"rewards": {"reward": 0.0}}, + } + assert module.is_verifier_backed_agent_timeout_nonpass(direct_result, harbor_result) is True + + harbor_result["exception_info"]["exception_type"] = "RuntimeError" + assert module.is_verifier_backed_agent_timeout_nonpass(direct_result, harbor_result) is False + + harbor_result["exception_info"]["exception_type"] = "AgentTimeoutError" + harbor_result["verifier_result"]["rewards"]["reward"] = 1.0 + assert module.is_verifier_backed_agent_timeout_nonpass(direct_result, harbor_result) is False + + def test_harbor_timeout_multipliers_are_validated(tmp_path: Path) -> None: module = load_validator() config = { From 4401b0fc8a1ef86deafc4e85d2b30d1055db1784 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 08:53:06 -0600 Subject: [PATCH 19/34] fix(example): validate routing mark lineage Signed-off-by: Bryan Bednarski --- .../scripts/validate_run.py | 10 +++++++- .../tests/test_validation_contract.py | 25 ++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py index 56c5f17d6..0e82aa369 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_run.py +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -223,7 +223,15 @@ def inspect_openinference(path: Path) -> dict[str, Any]: kind = attributes.get("openinference.span.kind") if isinstance(kind, str) and kind: span_kinds.add(kind) - if attributes.get("nemo_relay.uuid") and attributes.get("nemo_relay.scope_type"): + scope_lineage = attributes.get("nemo_relay.uuid") and attributes.get( + "nemo_relay.scope_type" + ) + mark_lineage = ( + attributes.get("nemo_relay.mark.uuid") + and attributes.get("nemo_relay.mark.parent_uuid") + and span.get("parentSpanId") + ) + if scope_lineage or mark_lineage: lineage_spans += 1 return { "documents": documents, diff --git a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py index 564e6691c..ffad0e7ad 100644 --- a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py @@ -193,7 +193,24 @@ def test_openinference_inspection_extracts_semantic_and_lineage_evidence(tmp_pat {"key": "nemo_relay.uuid", "value": {"stringValue": "uuid"}}, {"key": "nemo_relay.scope_type", "value": {"stringValue": "llm"}}, ] - } + }, + { + "parentSpanId": "parent-span", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "CHAIN"}}, + {"key": "nemo_relay.mark.uuid", "value": {"stringValue": "mark-uuid"}}, + { + "key": "nemo_relay.mark.parent_uuid", + "value": {"stringValue": "parent-uuid"}, + }, + ], + }, + { + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "CHAIN"}}, + {"key": "nemo_relay.mark.uuid", "value": {"stringValue": "orphan-mark"}}, + ], + }, ], } ], @@ -203,8 +220,8 @@ def test_openinference_inspection_extracts_semantic_and_lineage_evidence(tmp_pat artifact.write_text(json.dumps(payload) + "\n", encoding="utf-8") evidence = module.inspect_openinference(artifact) assert evidence["documents"] == 1 - assert evidence["spans"] == 1 - assert evidence["span_kinds"] == ["LLM"] + assert evidence["spans"] == 3 + assert evidence["span_kinds"] == ["CHAIN", "LLM"] assert evidence["scope_names"] == ["harbor-hermes-switchyard"] - assert evidence["lineage_spans"] == 1 + assert evidence["lineage_spans"] == 2 assert evidence["resource_attributes"]["openinference.project.name"] == ["project"] From 54ad451f625a7f891d078a318c0d5d74ad97216a Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 09:53:44 -0600 Subject: [PATCH 20/34] fix(example): separate benchmark completion from telemetry gates Signed-off-by: Bryan Bednarski --- .../run_terminal_bench.sh | 12 ++- .../scripts/run_phase2_cohort.py | 43 ++++++--- .../scripts/validate_run.py | 88 ++++++++++++------- .../tests/test_phase2_cohort.py | 42 +++++++++ 4 files changed, 139 insertions(+), 46 deletions(-) diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index b8ed7b4d0..8888ae8d6 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -342,15 +342,21 @@ artifacts = pathlib.Path(sys.argv[1]) run_root = pathlib.Path(sys.argv[2]) summary = { "schema_version": "harbor-hermes-switchyard.task-summary.v1", - "status": "passed", "job_name": sys.argv[3], "task_name": sys.argv[4], "artifacts": str(artifacts), "validation": json.loads((artifacts / "validation.json").read_text()), "phoenix_upload": json.loads((artifacts / "phoenix-upload.json").read_text()), } -if summary["validation"].get("status") != "passed" or summary["phoenix_upload"].get("status") != "passed": - raise SystemExit("task evidence gates did not pass") +summary["benchmark_completion"] = summary["validation"].get("benchmark", {}) +summary["integration_validation"] = summary["validation"].get("integration", {}) +benchmark_complete = summary["benchmark_completion"].get( + "status", summary["validation"].get("status") +) == "passed" +uploaded = summary["phoenix_upload"].get("status") == "passed" +summary["status"] = "passed" if benchmark_complete and uploaded else "failed" +if summary["status"] != "passed": + raise SystemExit("benchmark completion or Phoenix upload did not pass") (run_root / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") print(json.dumps(summary, indent=2)) PY diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index 99eb201ce..0b344af30 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -148,12 +148,11 @@ def task_summary_passed(path: Path) -> bool: summary = read_json(path) except (OSError, ValueError, json.JSONDecodeError): return False - return ( - summary.get("status") == "passed" - and isinstance(summary.get("validation"), dict) - and summary["validation"].get("status") == "passed" - and isinstance(summary.get("phoenix_upload"), dict) - and summary["phoenix_upload"].get("status") == "passed" + validation = summary.get("validation") + benchmark = validation.get("benchmark", {}) if isinstance(validation, dict) else {} + benchmark_status = benchmark.get("status", validation.get("status") if isinstance(validation, dict) else None) + return summary.get("status") == "passed" and benchmark_status == "passed" and ( + isinstance(summary.get("phoenix_upload"), dict) and summary["phoenix_upload"].get("status") == "passed" ) @@ -675,12 +674,15 @@ def task_record(task: Task, attempt: Path | None) -> dict[str, Any]: summary = read_json(attempt / "summary.json") validation = summary["validation"] upload = summary["phoenix_upload"] + integration = validation.get("integration", {"status": validation.get("status"), "errors": []}) record.update( { - "status": "passed", + "status": "completed", "successful_attempt": attempt.name, "attempt_root": str(attempt), "benchmark_task_passed": validation.get("benchmark_task_passed"), + "benchmark_completion": validation.get("benchmark", {"status": validation.get("status")}), + "integration_validation": integration, "direct_result_status": validation.get("direct_result_status"), "switchyard_decision_count": validation.get("switchyard_decision_count", 0), "routed_models": validation.get("routed_models", []), @@ -700,15 +702,29 @@ def aggregate_summary(args: argparse.Namespace, tasks: list[Task]) -> dict[str, for task in tasks: task_root = args.run_root / "tasks" / task.directory_name records.append(task_record(task, successful_attempt(task_root))) - complete = all(record["status"] == "passed" for record in records) + complete = all(record["status"] == "completed" for record in records) cache_read_tokens = sum(int(record.get("cache_read_tokens") or 0) for record in records) observed_models = sorted( {model for record in records for model in record.get("routed_models", []) if isinstance(model, str)} ) missing_models = sorted(set(args.required_model).difference(observed_models)) - secrets_clean = all(not record.get("secret_findings") for record in records if record["status"] == "passed") + completed_records = [record for record in records if record["status"] == "completed"] + secrets_clean = all(not record.get("secret_findings") for record in completed_records) + integration_failures = [ + { + "task": record["name"], + "errors": record.get("integration_validation", {}).get("errors", []), + } + for record in completed_records + if record.get("integration_validation", {}).get("status") != "passed" + ] gates = { - "task_outputs": {"passed": complete, "completed": sum(r["status"] == "passed" for r in records)}, + "task_outputs": {"passed": complete, "completed": sum(r["status"] == "completed" for r in records)}, + "integration_validation": { + "passed": not integration_failures, + "failed_task_count": len(integration_failures), + "failures": integration_failures, + }, "cache_hit": { "required": args.require_cache_hit, "passed": not args.require_cache_hit or cache_read_tokens > 0, @@ -730,7 +746,7 @@ def aggregate_summary(args: argparse.Namespace, tasks: list[Task]) -> dict[str, "phoenix_project": args.phoenix_project, "evaluation_cohort": args.eval_cohort, "planned_tasks": len(records), - "completed_tasks": sum(record["status"] == "passed" for record in records), + "completed_tasks": sum(record["status"] == "completed" for record in records), "benchmark_pass_count": sum(record.get("benchmark_task_passed") is True for record in records), "benchmark_nonpass_count": sum(record.get("benchmark_task_passed") is False for record in records), "uploaded_spans": sum(int(record.get("uploaded_spans") or 0) for record in records), @@ -1144,6 +1160,11 @@ def main() -> int: print(json.dumps(summary, indent=2)) if passed: return 0 + if ( + summary["completed_tasks"] == summary["planned_tasks"] + and not summary["cohort_gates"]["integration_validation"]["passed"] + ): + return 20 states = [read_json(path) for path in (args.run_root / "tasks").glob("*/task-state.json") if path.is_file()] setup_state = args.run_root / "setup-state.json" if setup_state.is_file(): diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py index 0e82aa369..7f5d51c4e 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_run.py +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -323,13 +323,20 @@ def main() -> int: parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() - errors: list[str] = [] + # Keep benchmark completion distinct from the optional-but-audited + # Relay/Switchyard evidence contract. A verifier-backed task result is + # still a completed Terminal-Bench evaluation if observability evidence is + # incomplete; the cohort summary decides whether that evidence is strong + # enough for this integration example to pass as a whole. + integration_errors: list[str] = [] + benchmark_errors: list[str] = [] + integration_warnings: list[str] = [] root = args.artifacts.resolve() try: files = contained_files(root) except Exception as error: files = [] - errors.append(str(error)) + integration_errors.append(str(error)) required = { "result": root / "direct-hermes-result.json", @@ -339,12 +346,13 @@ def main() -> int: } for name, path in required.items(): if not path.is_file(): - errors.append(f"missing {name}: {path}") + target = benchmark_errors if name == "result" else integration_errors + target.append(f"missing {name}: {path}") atif_files = sorted((root / "relay" / "atif").glob("trajectory-*.atif.json")) if not atif_files: - errors.append("missing ATIF trajectory") + integration_errors.append("missing ATIF trajectory") if not args.openinference.is_file() or args.openinference.stat().st_size == 0: - errors.append("missing OpenInference OTLP artifact") + integration_errors.append("missing OpenInference OTLP artifact") result: dict[str, Any] = {} receipt: dict[str, Any] = {} @@ -352,23 +360,23 @@ def main() -> int: if required["result"].is_file(): result = read_json(required["result"]) if result.get("status") not in {"completed", "preserved_completed_response"}: - errors.append(f"invalid direct result status: {result.get('status')!r}") + benchmark_errors.append(f"invalid direct result status: {result.get('status')!r}") if not isinstance(result.get("final_response"), str) or not result["final_response"]: - errors.append("direct result has no normalized final response") + benchmark_errors.append("direct result has no normalized final response") if args.expect_late_failure: if result.get("status") != "preserved_completed_response": - errors.append("expected a preserved completed response after late failure") + benchmark_errors.append("expected a preserved completed response after late failure") if result.get("error", {}).get("type") != "InjectedPostResponseFailure": - errors.append("deterministic post-response failure was not recorded") + benchmark_errors.append("deterministic post-response failure was not recorded") if required["receipt"].is_file(): receipt = read_json(required["receipt"]) if args.provenance.is_file(): provenance = read_json(args.provenance) else: - errors.append("missing runtime provenance") + integration_errors.append("missing runtime provenance") if receipt: - errors.extend(validate_receipt_provenance(receipt, provenance)) + integration_errors.extend(validate_receipt_provenance(receipt, provenance)) openinference_evidence = { "documents": 0, @@ -382,15 +390,15 @@ def main() -> int: try: openinference_evidence = inspect_openinference(args.openinference) except Exception as error: - errors.append(f"invalid OpenInference OTLP artifact: {error}") + integration_errors.append(f"invalid OpenInference OTLP artifact: {error}") if openinference_evidence["documents"] == 0 or openinference_evidence["spans"] == 0: - errors.append("OpenInference artifact contains no spans") + integration_errors.append("OpenInference artifact contains no spans") if not {"AGENT", "LLM"}.issubset(openinference_evidence["span_kinds"]): - errors.append("OpenInference artifact does not contain both AGENT and LLM span kinds") + integration_errors.append("OpenInference artifact does not contain both AGENT and LLM span kinds") if openinference_evidence["scope_names"] != ["harbor-hermes-switchyard"]: - errors.append("OpenInference instrumentation scope does not match the example") + integration_errors.append("OpenInference instrumentation scope does not match the example") if openinference_evidence["lineage_spans"] != openinference_evidence["spans"]: - errors.append("OpenInference spans are missing Relay UUID or scope-type lineage") + integration_errors.append("OpenInference spans are missing Relay UUID or scope-type lineage") expected_resources = { "openinference.project.name": provenance.get("phoenix_project"), "evaluation.cohort": provenance.get("eval_cohort"), @@ -399,7 +407,7 @@ def main() -> int: } for key, expected in expected_resources.items(): if openinference_evidence["resource_attributes"].get(key) != [expected]: - errors.append(f"OpenInference resource attribute {key!r} does not match runtime provenance") + integration_errors.append(f"OpenInference resource attribute {key!r} does not match runtime provenance") event_count = 0 routing_marks: list[str] = [] @@ -418,14 +426,14 @@ def main() -> int: cache_read_tokens = atof_evidence["cache_read_tokens"] cache_write_tokens = atof_evidence["cache_write_tokens"] if event_count == 0: - errors.append("ATOF artifact is empty") + integration_errors.append("ATOF artifact is empty") if not routing_marks: - errors.append("ATOF artifact has no Switchyard routing evidence") + integration_errors.append("ATOF artifact has no Switchyard routing evidence") if not routed_targets: - errors.append("ATOF artifact has no selected Switchyard target") + integration_errors.append("ATOF artifact has no selected Switchyard target") unexpected_targets = sorted(set(routed_targets) - {"strong", "weak"}) if unexpected_targets: - errors.append(f"ATOF artifact selected unexpected targets: {unexpected_targets}") + integration_errors.append(f"ATOF artifact selected unexpected targets: {unexpected_targets}") caller_model = provenance.get("routing", {}).get("hermes_caller_model") target_models = { @@ -439,7 +447,7 @@ def main() -> int: } ) if caller_model and caller_model in routed_models: - errors.append("Hermes caller stub appeared as a routed provider model") + integration_errors.append("Hermes caller stub appeared as a routed provider model") secret_values: list[bytes] = [] for name in args.secret_env: @@ -460,7 +468,7 @@ def main() -> int: files_to_scan.extend(scan_files(scan_root.resolve())) findings = scan_secrets(sorted(set(files_to_scan)), secret_values) if findings: - errors.append(f"secret scan found {len(findings)} persisted value(s)") + integration_errors.append(f"secret scan found {len(findings)} persisted value(s)") harbor_results: list[Path] = [] harbor_timeout_multipliers: dict[str, float] = {} @@ -468,31 +476,47 @@ def main() -> int: harbor_result: dict[str, Any] = {} if args.harbor_job_dir: harbor_timeout_multipliers, timeout_errors = validate_harbor_job_config(args.harbor_job_dir) - errors.extend(timeout_errors) + integration_errors.extend(timeout_errors) harbor_results = [ path for path in sorted(args.harbor_job_dir.glob("**/result.json")) if is_trial_result(read_json(path)) ] if len(harbor_results) != 1: - errors.append(f"expected one Harbor trial result, found {len(harbor_results)}") + benchmark_errors.append(f"expected one Harbor trial result, found {len(harbor_results)}") elif harbor_results: harbor_result = read_json(harbor_results[0]) benchmark_passed = read_benchmark_passed(harbor_result) + if benchmark_passed is None: + benchmark_errors.append("Harbor trial did not contain a normalized benchmark reward") + else: + benchmark_errors.append("missing Harbor job directory for benchmark completion") terminal_timeout_nonpass = is_verifier_backed_agent_timeout_nonpass(result, harbor_result) if terminal_timeout_nonpass: - tolerated_errors = { - "missing ATIF trajectory", + tolerated_benchmark_errors = { "invalid direct result status: 'failed'", "direct result has no normalized final response", } + benchmark_errors = [error for error in benchmark_errors if error not in tolerated_benchmark_errors] + tolerated_integration_errors = { + "missing ATIF trajectory", + } if "LLM" in openinference_evidence["span_kinds"]: - tolerated_errors.add("OpenInference artifact does not contain both AGENT and LLM span kinds") - errors = [error for error in errors if error not in tolerated_errors] + tolerated_integration_errors.add("OpenInference artifact does not contain both AGENT and LLM span kinds") + for error in integration_errors: + if error in tolerated_integration_errors: + integration_warnings.append(error) + integration_errors = [error for error in integration_errors if error not in tolerated_integration_errors] validation = { "schema_version": SCHEMA_VERSION, - "status": "passed" if not errors else "failed", - "errors": errors, + "status": "passed" if not benchmark_errors else "failed", + "errors": benchmark_errors, + "benchmark": {"status": "passed" if not benchmark_errors else "failed", "errors": benchmark_errors}, + "integration": { + "status": "passed" if not integration_errors else "failed", + "errors": integration_errors, + "warnings": integration_warnings, + }, "direct_result_status": result.get("status"), "harbor_trial_count": len(harbor_results) if args.harbor_job_dir else None, "harbor_timeout_multipliers": harbor_timeout_multipliers, @@ -513,7 +537,7 @@ def main() -> int: args.output.parent.mkdir(mode=0o700, parents=True, exist_ok=True) args.output.write_text(json.dumps(validation, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(json.dumps(validation, indent=2)) - return 0 if not errors else 1 + return 0 if not benchmark_errors else 1 if __name__ == "__main__": diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 119f733c6..a7879ef13 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -141,6 +141,41 @@ def test_cohort_summary_blocks_missing_route_even_when_tasks_pass(tmp_path: Path assert summary["cohort_gates"]["route_diversity"]["missing_models"] == ["sonnet"] +def test_integration_failure_does_not_erase_completed_benchmark_output(tmp_path: Path) -> None: + module = load_coordinator() + task = module.Task(1, "one", 2) + attempt = tmp_path / "tasks" / task.directory_name / "attempts" / "001" + attempt.mkdir(parents=True) + attempt_summary = { + "status": "passed", + "validation": { + "status": "passed", + "benchmark": {"status": "passed", "errors": []}, + "integration": {"status": "failed", "errors": ["missing route mark"], "warnings": []}, + "benchmark_task_passed": True, + "routed_models": ["sonnet"], + "routed_targets": ["weak"], + "cache_read_tokens": 12, + "secret_findings": [], + }, + "phoenix_upload": {"status": "passed", "uploaded_spans": 10}, + } + (attempt / "summary.json").write_text(json.dumps(attempt_summary), encoding="utf-8") + + assert module.task_summary_passed(attempt / "summary.json") is True + args = cohort_args(tmp_path) + args.required_model = ["sonnet"] + summary = module.aggregate_summary(args, [task]) + assert summary["completed_tasks"] == 1 + assert summary["tasks"][0]["status"] == "completed" + assert summary["cohort_gates"]["integration_validation"] == { + "passed": False, + "failed_task_count": 1, + "failures": [{"task": "one", "errors": ["missing route mark"]}], + } + assert summary["status"] == "partial" + + def test_failure_classifier_retries_only_known_infrastructure_failures() -> None: module = load_coordinator() assert module.classify_failure("TLS handshake timeout contacting registry-1.docker.io") == "infrastructure" @@ -390,6 +425,13 @@ def test_runner_does_not_expand_an_empty_validation_expectations_array() -> None assert "validation_args+=(--expect-late-failure)" in runner +def test_runner_keeps_benchmark_completion_separate_from_integration_validation() -> None: + runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") + assert 'summary["benchmark_completion"] = summary["validation"].get("benchmark", {})' in runner + assert 'summary["integration_validation"] = summary["validation"].get("integration", {})' in runner + assert "benchmark completion or Phoenix upload did not pass" in runner + + def test_plugin_contract_owns_routes_and_authorization_name() -> None: module = load_coordinator() contract = module.plugin_contract(EXAMPLE_ROOT / "config" / "plugins.toml.in") From cadad7b91492d0b7b1c585dcb8fbb3c83cd82167 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 09:55:06 -0600 Subject: [PATCH 21/34] docs(example): distinguish benchmark and integration gates Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 48d81359b..445b27a51 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -286,13 +286,21 @@ subject to the cohort's bounded retry limit. ## 11. Completion gates -A task is complete only when both its `validation.json` and -`phoenix-upload.json` have `status=passed`. A benchmark -`reward.task_passed=false` is a valid completed result and is never retried. +A task is complete when `validation.json` records +`benchmark.status=passed` and `phoenix-upload.json` has `status=passed`. +The top-level validation status mirrors benchmark completion for compatibility. +A benchmark `reward.task_passed=false` is a valid completed result and is never +retried. + +Relay/Switchyard artifact checks are recorded separately in +`validation.integration`. An integration finding is preserved in the task and +cohort report; it does not discard a completed benchmark result or cause the +agent to be run again. It remains a cohort-level acceptance gate. The cohort passes only when: - all 89 tasks are independently validated and uploaded; +- `cohort_gates.integration_validation` passes; - direct artifacts and logs pass secret scans; - cache-read evidence is nonzero; - both models derived from `plugins.toml.in` appear in committed routes; and From 498266ab5d98681b8f1a3ecc2ee2f36fb03e3324 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 10:20:15 -0600 Subject: [PATCH 22/34] fix(example): classify Phoenix export as integration evidence Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 15 +++++----- .../run_terminal_bench.sh | 28 +++++++++++++++---- .../scripts/run_phase2_cohort.py | 4 +-- .../tests/test_phase2_cohort.py | 19 +++++++++---- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 445b27a51..2622cb467 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -287,19 +287,20 @@ subject to the cohort's bounded retry limit. ## 11. Completion gates A task is complete when `validation.json` records -`benchmark.status=passed` and `phoenix-upload.json` has `status=passed`. -The top-level validation status mirrors benchmark completion for compatibility. +`benchmark.status=passed`. The top-level validation status mirrors benchmark +completion for compatibility. A benchmark `reward.task_passed=false` is a valid completed result and is never retried. -Relay/Switchyard artifact checks are recorded separately in -`validation.integration`. An integration finding is preserved in the task and -cohort report; it does not discard a completed benchmark result or cause the -agent to be run again. It remains a cohort-level acceptance gate. +Relay/Switchyard artifact checks, including the Phoenix upload result, are +recorded separately in `validation.integration`. An integration finding is +preserved in the task and cohort report; it does not discard a completed +benchmark result or cause the agent to be run again. It remains a cohort-level +acceptance gate. The cohort passes only when: -- all 89 tasks are independently validated and uploaded; +- all 89 tasks have independently completed benchmark results; - `cohort_gates.integration_validation` passes; - direct artifacts and logs pass secret scans; - cache-read evidence is nonzero; diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index 8888ae8d6..3f47dfdf7 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -326,12 +326,21 @@ if [[ "$inject_post_response_failure" == "true" ]]; then fi "$python_bin" "${validation_args[@]}" >"$run_root/validation.log" -"$python_bin" "$example_root/scripts/upload_openinference.py" \ +if ! "$python_bin" "$example_root/scripts/upload_openinference.py" \ --openinference "$openinference" \ --phoenix-url "$phoenix_base" \ --project "$phoenix_project" \ --output "$artifact_root/phoenix-upload.json" \ - >"$run_root/phoenix-upload.log" + >"$run_root/phoenix-upload.log"; then + "$python_bin" - "$artifact_root/phoenix-upload.json" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +path.write_text(json.dumps({"status": "failed", "error": "Phoenix upload command failed"}, indent=2) + "\n") +PY +fi "$python_bin" - "$artifact_root" "$run_root" "$job_name" "$task_name" <<'PY' import json @@ -348,15 +357,22 @@ summary = { "validation": json.loads((artifacts / "validation.json").read_text()), "phoenix_upload": json.loads((artifacts / "phoenix-upload.json").read_text()), } +integration = summary["validation"].setdefault("integration", {"status": "passed", "errors": [], "warnings": []}) +integration_errors = list(integration.get("errors", [])) +integration["phoenix_upload"] = summary["phoenix_upload"] +if summary["phoenix_upload"].get("status") != "passed": + integration_errors.append("Phoenix upload did not pass") +integration["errors"] = sorted(set(integration_errors)) +integration["status"] = "passed" if not integration["errors"] else "failed" +(artifacts / "validation.json").write_text(json.dumps(summary["validation"], indent=2, sort_keys=True) + "\n") summary["benchmark_completion"] = summary["validation"].get("benchmark", {}) -summary["integration_validation"] = summary["validation"].get("integration", {}) +summary["integration_validation"] = integration benchmark_complete = summary["benchmark_completion"].get( "status", summary["validation"].get("status") ) == "passed" -uploaded = summary["phoenix_upload"].get("status") == "passed" -summary["status"] = "passed" if benchmark_complete and uploaded else "failed" +summary["status"] = "passed" if benchmark_complete else "failed" if summary["status"] != "passed": - raise SystemExit("benchmark completion or Phoenix upload did not pass") + raise SystemExit("benchmark completion did not pass") (run_root / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") print(json.dumps(summary, indent=2)) PY diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index 0b344af30..71d514181 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -151,9 +151,7 @@ def task_summary_passed(path: Path) -> bool: validation = summary.get("validation") benchmark = validation.get("benchmark", {}) if isinstance(validation, dict) else {} benchmark_status = benchmark.get("status", validation.get("status") if isinstance(validation, dict) else None) - return summary.get("status") == "passed" and benchmark_status == "passed" and ( - isinstance(summary.get("phoenix_upload"), dict) and summary["phoenix_upload"].get("status") == "passed" - ) + return summary.get("status") == "passed" and benchmark_status == "passed" def successful_attempt(task_root: Path) -> Path | None: diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index a7879ef13..8d9fb244a 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -151,14 +151,19 @@ def test_integration_failure_does_not_erase_completed_benchmark_output(tmp_path: "validation": { "status": "passed", "benchmark": {"status": "passed", "errors": []}, - "integration": {"status": "failed", "errors": ["missing route mark"], "warnings": []}, + "integration": { + "status": "failed", + "errors": ["missing route mark", "Phoenix upload did not pass"], + "warnings": [], + "phoenix_upload": {"status": "failed"}, + }, "benchmark_task_passed": True, "routed_models": ["sonnet"], "routed_targets": ["weak"], "cache_read_tokens": 12, "secret_findings": [], }, - "phoenix_upload": {"status": "passed", "uploaded_spans": 10}, + "phoenix_upload": {"status": "failed", "error": "Phoenix upload command failed"}, } (attempt / "summary.json").write_text(json.dumps(attempt_summary), encoding="utf-8") @@ -171,7 +176,9 @@ def test_integration_failure_does_not_erase_completed_benchmark_output(tmp_path: assert summary["cohort_gates"]["integration_validation"] == { "passed": False, "failed_task_count": 1, - "failures": [{"task": "one", "errors": ["missing route mark"]}], + "failures": [ + {"task": "one", "errors": ["missing route mark", "Phoenix upload did not pass"]} + ], } assert summary["status"] == "partial" @@ -428,8 +435,10 @@ def test_runner_does_not_expand_an_empty_validation_expectations_array() -> None def test_runner_keeps_benchmark_completion_separate_from_integration_validation() -> None: runner = (EXAMPLE_ROOT / "run_terminal_bench.sh").read_text(encoding="utf-8") assert 'summary["benchmark_completion"] = summary["validation"].get("benchmark", {})' in runner - assert 'summary["integration_validation"] = summary["validation"].get("integration", {})' in runner - assert "benchmark completion or Phoenix upload did not pass" in runner + assert 'integration["phoenix_upload"] = summary["phoenix_upload"]' in runner + assert 'summary["integration_validation"] = integration' in runner + assert "benchmark completion did not pass" in runner + assert "benchmark completion or Phoenix upload did not pass" not in runner def test_plugin_contract_owns_routes_and_authorization_name() -> None: From 6d2efce09ecd6b5bb6e626c790e34beb9b18fe48 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 11:40:44 -0600 Subject: [PATCH 23/34] docs(example): clarify canary completion gate Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 2622cb467..2f87e1084 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -197,10 +197,12 @@ immutable inputs is refused. Choose concurrency before this point. **Optional canary-first scheduling.** The default `TBENCH_CANARY_TASK=adaptive-rejection-sampler` runs that one real -task first. A passed validation and upload result opens the parallel lane even -when its benchmark reward is a non-pass. This is a conservative production -check, not a separate command: launching the full cohort on a fresh run root -automatically starts the canary and then continues with the remaining tasks. +task first. A completed benchmark result (`validation.benchmark.status=passed`) +opens the parallel lane even when its benchmark reward is a non-pass. Phoenix +upload and other integration evidence remain cohort-level acceptance gates. +This is a conservative production check, not a separate command: launching the +full cohort on a fresh run root automatically starts the canary and then +continues with the remaining tasks. To skip that one-task checkpoint, set an explicitly blank value in `.env`: From 78b7546468e9be1098758a206d40118b6fa48ad4 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 11:54:01 -0600 Subject: [PATCH 24/34] docs(example): use local phase two environment file Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 16 +++++++--------- .../scripts/launch_phase2_tmux.sh | 17 +++++++++++++---- .../scripts/run_phase2_from_env.sh | 5 +++-- .../scripts/validate_phase2_environment.sh | 5 +++-- .../tests/test_phase2_cohort.py | 2 ++ 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 2f87e1084..6aed42684 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -94,14 +94,14 @@ python3 -m venv .venv .venv/bin/python -m pip install -r requirements.txt ``` -Copy and protect the environment file outside the checkout. Replace every +Copy and protect the environment file at the example root. Replace every placeholder, including the complete provider Authorization header. Do not source this file into the interactive shell used to start `tmux`. ```bash -cp .env.example /absolute/private/.env -chmod 0600 /absolute/private/.env -./scripts/validate_phase2_environment.sh /absolute/private/.env +cp .env.example .env +chmod 0600 .env +./scripts/validate_phase2_environment.sh .env ``` The validator reports names and paths only. It rejects legacy secret-file @@ -119,7 +119,7 @@ For the commands below, enter a short-lived shell with tracing disabled: ```bash set +x set -a -source /absolute/private/.env +source .env set +a set +x ``` @@ -248,9 +248,7 @@ active cohort. No environment file or secret is copied into the snapshot. ```bash exit # only when returning from the short-lived admission shell above -./scripts/launch_phase2_tmux.sh \ - /absolute/private/.env \ - harbor-hermes-switchyard-phase2-run-1 +./scripts/launch_phase2_tmux.sh harbor-hermes-switchyard-phase2-run-1 ``` Operational commands: @@ -273,7 +271,7 @@ tmux send-keys -t harbor-hermes-switchyard-phase2-run-1 C-c # After the old session exits, resume from the run-bound snapshot. This also # works after a checkout update or host reboot. /absolute/path/to/phase2-run-root/runtime-harness/scripts/launch_phase2_tmux.sh \ - /absolute/private/.env \ + /absolute/path/to/examples/harbor-hermes-switchyard/.env \ harbor-hermes-switchyard-phase2-run-1 ``` diff --git a/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh index d9f22dcc4..19a27aa4c 100755 --- a/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh +++ b/examples/harbor-hermes-switchyard/scripts/launch_phase2_tmux.sh @@ -5,12 +5,21 @@ set -euo pipefail example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -env_file="${1:-}" -session="${2:-}" -if [[ -z "$env_file" || "$env_file" != /* || -z "$session" ]]; then - echo "usage: $0 /absolute/.env tmux-session-name" >&2 +if [[ "$#" -eq 1 ]]; then + env_file="$example_root/.env" + session="$1" +elif [[ "$#" -eq 2 ]]; then + env_file="$1" + session="$2" +else + echo "usage: $0 [env-file] tmux-session-name" >&2 exit 2 fi +if [[ ! -f "$env_file" ]]; then + echo "Phase 2 environment file does not exist: $env_file" >&2 + exit 2 +fi +env_file="$(cd "$(dirname "$env_file")" && pwd)/$(basename "$env_file")" if [[ ! "$session" =~ ^[A-Za-z0-9_.-]+$ ]]; then echo "tmux session name may contain only letters, digits, dot, underscore, and dash" >&2 exit 2 diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh index 5e50e7bb9..04b8be508 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_from_env.sh @@ -7,10 +7,11 @@ set +x example_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" env_file="${TERMINAL_BENCH_ENV_FILE:-${1:-}}" -if [[ -z "$env_file" || "$env_file" != /* ]]; then - echo "TERMINAL_BENCH_ENV_FILE must be an absolute path" >&2 +if [[ -z "$env_file" || ! -f "$env_file" ]]; then + echo "TERMINAL_BENCH_ENV_FILE must reference an existing environment file" >&2 exit 2 fi +env_file="$(cd "$(dirname "$env_file")" && pwd)/$(basename "$env_file")" "$example_root/scripts/validate_phase2_environment.sh" "$env_file" set -a diff --git a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh index 795fc9545..f8be22c79 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh +++ b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh @@ -6,10 +6,11 @@ set -euo pipefail set +x env_file="${1:-}" -if [[ -z "$env_file" || "$env_file" != /* || ! -f "$env_file" ]]; then - echo "usage: $0 /absolute/.env" >&2 +if [[ -z "$env_file" || ! -f "$env_file" ]]; then + echo "usage: $0 .env" >&2 exit 2 fi +env_file="$(cd "$(dirname "$env_file")" && pwd)/$(basename "$env_file")" if mode="$(stat -f '%Lp' "$env_file" 2>/dev/null)"; then : else diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 8d9fb244a..d8772d8da 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -314,6 +314,8 @@ def test_durable_supervisor_owns_the_coordinator_process_group() -> None: def test_tmux_launcher_projects_only_the_protected_file_path() -> None: launcher = (EXAMPLE_ROOT / "scripts" / "launch_phase2_tmux.sh").read_text(encoding="utf-8") child = (EXAMPLE_ROOT / "scripts" / "run_phase2_from_env.sh").read_text(encoding="utf-8") + assert 'env_file="$example_root/.env"' in launcher + assert 'usage: $0 [env-file] tmux-session-name' in launcher assert '-e "TERMINAL_BENCH_ENV_FILE=$env_file"' in launcher assert 'source "$env_file"' not in launcher assert "tmux has-session" in launcher From 2f940a35454d06e87bfb8fbb026fe0d91b086e8c Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 12:25:49 -0600 Subject: [PATCH 25/34] fix(example): accept current Relay releases Signed-off-by: Bryan Bednarski --- .../harbor-hermes-switchyard/.env.example | 4 +- examples/harbor-hermes-switchyard/README.md | 8 ++-- .../agents/harbor_hermes_agent.py | 8 ++-- .../scripts/build_hermetic_runtime.py | 10 ++++- .../scripts/finalize_artifacts.py | 12 +++++- .../scripts/prepare_runtime.py | 27 +++++-------- .../scripts/relay_version.py | 40 +++++++++++++++++++ .../run_offline_compatibility_smoke.sh | 4 +- .../scripts/run_phase2_cohort.py | 3 +- .../scripts/run_setup_admission.py | 2 +- .../scripts/smoke_phase2_dataset.py | 9 ++++- .../scripts/validate_run.py | 15 ++++++- .../tests/test_agent_result_contract.py | 8 ++-- .../tests/test_plugin_lifecycle.py | 2 +- .../tests/test_validation_contract.py | 4 +- 15 files changed, 110 insertions(+), 46 deletions(-) create mode 100644 examples/harbor-hermes-switchyard/scripts/relay_version.py diff --git a/examples/harbor-hermes-switchyard/.env.example b/examples/harbor-hermes-switchyard/.env.example index 657142803..723c15f41 100644 --- a/examples/harbor-hermes-switchyard/.env.example +++ b/examples/harbor-hermes-switchyard/.env.example @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Copy this file outside the checkout, chmod 0600, and replace every /absolute +# Copy this file to .env, chmod 0600, and replace every /absolute # placeholder. Never commit the populated file. EXAMPLE_ROOT=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard TERMINAL_BENCH_RUN_ID=harbor-hermes-switchyard-phase2-run-1 @@ -15,7 +15,7 @@ HARBOR_BIN=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/ EVAL_PYTHON=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/.venv/bin/python TBENCH_DATASET_PATH=/absolute/path/to/exported/terminal-bench SWITCHYARD_BUNDLE=/absolute/path/to/pinned-switchyard-bundle -RELAY_WHEEL=/absolute/path/to/nemo_relay-0.7.0-platform-wheel.whl +RELAY_WHEEL=/absolute/path/to/nemo_relay-0.7.1-platform-wheel.whl RELAY_ARCHITECTURE=x86_64 PLUGIN_CONFIG_TEMPLATE=/absolute/path/to/NeMo-Relay/examples/harbor-hermes-switchyard/config/plugins.toml.in diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 6aed42684..450b716bc 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -1,7 +1,7 @@ # Harbor + Hermes + Switchyard evaluation This example runs one complete Terminal-Bench 2.0 cohort through Harbor and -Hermes. Hermes owns an in-process NeMo Relay 0.7.0 runtime; Relay loads the +Hermes. Hermes owns an in-process NeMo Relay runtime satisfying `nemo-relay>=0.7.0`; Relay loads the Switchyard native plugin, and Switchyard selects and calls the configured provider route. It runs one resumable 89-task cohort. Multi-cohort execution and result aggregation are intentionally out of scope for this example. @@ -10,13 +10,13 @@ and result aggregation are intentionally out of scope for this example. | Dependency | Input used by this example | |---|---| -| NeMo Relay | Released `nemo-relay==0.7.0` platform wheel, installed by digest rather than from this source checkout. | +| NeMo Relay | Latest released `nemo-relay>=0.7.0` platform wheel, installed by digest rather than from this source checkout. | | Hermes | `bbednarski9/hermes-agent`, detached commit `efb63e714abc436af88af9b0d6734751c199aa6d` from PR #77915. | | Switchyard | `bbednarski9/Switchyard`, detached commit `8daac03edf8544144833af1fd009b3da737715bc` from PR #270. | | Harbor | `harbor==0.18.0`, local export of dataset `terminal-bench@2.0`. | Every source checkout is detached and verified. The Hermes installer is -followed by `uv sync --frozen`, then the selected Relay 0.7.0 wheel is +followed by `uv sync --frozen`, then the selected released Relay wheel is force-installed without dependencies and verified by digest. ## 2. Request and lifecycle ownership @@ -76,7 +76,7 @@ reaching a provider. - Linux or macOS, Bash, Python 3.11+, Docker, and `tmux`; - a local, immutable Terminal-Bench 2.0 dataset export containing 89 tasks; -- a Switchyard plugin bundle and Relay 0.7.0 wheel matching Docker's +- a Switchyard plugin bundle and released Relay wheel satisfying `nemo-relay>=0.7.0`, matching Docker's architecture (`x86_64` or `aarch64`); - a Phoenix endpoint accepting OTLP/HTTP OpenInference traces; and - provider and registry access for the full cohort. The all-89 admission uses diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py index 9a83ac80f..452ff82fd 100644 --- a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -121,7 +121,7 @@ def _hermetic_runtime_readiness_command( "runtime_ready=1; " f"for attempt in {attempt_numbers}; do " f'if {runtime}/bin/python -c "import importlib.metadata as m; ' - "assert m.version('nemo-relay') == '0.7.0'\" " + "assert tuple(map(int, m.version('nemo-relay').split('.'))) >= (0, 7, 0)\" " f"&& {runtime}/bin/hermes version; then " "runtime_ready=0; break; " "else runtime_ready=$?; fi; " @@ -482,7 +482,7 @@ async def install(self, environment: BaseEnvironment) -> None: 'export PATH="$HOME/.local/bin:$PATH"; ' "hermes version; " f'{install_dir}/venv/bin/python -c "import importlib.metadata as m; ' - "assert m.version('nemo-relay') == '0.7.0'\"" + "assert tuple(map(int, m.version('nemo-relay').split('.'))) >= (0, 7, 0)\"" ), ) @@ -514,7 +514,7 @@ async def setup(self, environment: BaseEnvironment) -> None: f"test \"$(sha256sum {shlex.quote(relay_wheel)} | cut -d' ' -f1)\" = " f"{shlex.quote(self.relay_wheel_sha256)}; " f'{relay_install} -c "import importlib.metadata as m; ' - "assert m.version('nemo-relay') == '0.7.0'\"" + "assert tuple(map(int, m.version('nemo-relay').split('.'))) >= (0, 7, 0)\"" ), timeout_sec=120, ) @@ -532,7 +532,7 @@ async def setup(self, environment: BaseEnvironment) -> None: f"{probe_python} -c " + shlex.quote( "import ctypes, importlib.metadata as m; " - "assert m.version('nemo-relay') == '0.7.0'; " + "assert tuple(map(int, m.version('nemo-relay').split('.'))) >= (0, 7, 0); " f"library = ctypes.CDLL({switchyard_library!r}); " "assert getattr(library, 'nemo_relay_register_plugin')" ) diff --git a/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py b/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py index 6236ef890..ac8f206ef 100755 --- a/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py @@ -18,10 +18,16 @@ import os import shutil import subprocess +import sys import tempfile import time from pathlib import Path +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +from relay_version import wheel_version + SCHEMA_VERSION = "harbor-hermes-switchyard.hermetic-runtime.v1" DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" @@ -149,7 +155,7 @@ def build_payload( /opt/hermes-runtime/bin/hermes version /opt/hermes-runtime/bin/python -c \ - 'import importlib.metadata as m; assert m.version("nemo-relay") == "0.7.0"' + 'import importlib.metadata as m; assert tuple(map(int, m.version("nemo-relay").split("."))) >= (0, 7, 0)' ''' env = os.environ.copy() env.update({"UV_VERSION": UV_VERSION, "PYTHON_VERSION": PYTHON_VERSION}) @@ -240,7 +246,7 @@ def main() -> int: "hermes_repository": args.hermes_repository, "hermes_ref": args.hermes_ref, "hermes_commit": args.hermes_commit, - "relay_version": "0.7.0", + "relay_version": wheel_version(relay_wheel), "relay_wheel_sha256": sha256_file(relay_wheel), "relay_architecture": expected_arch, "builder_image": BUILDER_IMAGE, diff --git a/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py index 449ca71c8..4f13efaa4 100755 --- a/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py +++ b/examples/harbor-hermes-switchyard/scripts/finalize_artifacts.py @@ -11,9 +11,15 @@ import json import os import re +import sys import time import tomllib from pathlib import Path + +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +from relay_version import require_supported_version from typing import Any SCHEMA_VERSION = "harbor-hermes-switchyard.phase1.v1" @@ -117,8 +123,10 @@ def initialize(args: argparse.Namespace, root: Path) -> None: "completion_marker_written": False, }, } - if receipt["dependencies"]["nemo_relay"]["version"] != "0.7.0": - raise RuntimeError("Hermes environment did not install nemo-relay==0.7.0") + try: + require_supported_version(receipt["dependencies"]["nemo_relay"]["version"]) + except ValueError as error: + raise RuntimeError("Hermes environment did not install a supported nemo-relay release") from error (root / "relay" / "atif").mkdir(mode=0o700, parents=True, exist_ok=True) (root / "diagnostics").mkdir(mode=0o700, parents=True, exist_ok=True) atomic_json(root / "direct-hermes-receipt.json", receipt) diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index 0b4b090f3..877ff5192 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -16,16 +16,15 @@ import tomllib from pathlib import Path from urllib.parse import urlsplit -from zipfile import ZipFile import tomli_w +from relay_version import RELAY_REQUIREMENT, require_supported_version, wheel_version HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" HERMES_REF = "feat/relay-native-plugin-init" HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" SWITCHYARD_REPOSITORY = "https://github.com/bbednarski9/Switchyard.git" SWITCHYARD_COMMIT = "8daac03edf8544144833af1fd009b3da737715bc" -RELAY_VERSION = "0.7.0" SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}") @@ -76,28 +75,22 @@ def download_relay_wheel(destination: Path, architecture: str) -> Path: "abi3", "--dest", str(destination), - f"nemo-relay=={RELAY_VERSION}", + RELAY_REQUIREMENT, ], check=True, ) - wheels = sorted(destination.glob("nemo_relay-0.7.0-*.whl")) + wheels = sorted(destination.glob("nemo_relay-*.whl")) if len(wheels) != 1: raise RuntimeError(f"expected one Relay wheel, found {len(wheels)}") return wheels[0] -def verify_relay_wheel(path: Path, architecture: str) -> None: - if not path.is_file() or not path.name.startswith("nemo_relay-0.7.0-"): - raise ValueError("Relay wheel must be a nemo_relay-0.7.0 wheel") +def verify_relay_wheel(path: Path, architecture: str) -> str: + if not path.is_file() or not path.name.startswith("nemo_relay-"): + raise ValueError("Relay wheel must be a nemo_relay wheel") if "manylinux" not in path.name or architecture not in path.name: raise ValueError(f"Relay wheel must target Linux {architecture}") - with ZipFile(path) as wheel: - metadata_names = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")] - if len(metadata_names) != 1: - raise ValueError("Relay wheel has an ambiguous METADATA payload") - metadata = wheel.read(metadata_names[0]).decode("utf-8", errors="strict") - if "Name: nemo-relay\n" not in metadata or "Version: 0.7.0\n" not in metadata: - raise ValueError("Relay wheel metadata does not identify nemo-relay==0.7.0") + return wheel_version(path) def verify_native_library(path: Path, architecture: str) -> None: @@ -227,14 +220,14 @@ def main() -> int: if args.relay_wheel: source_wheel = args.relay_wheel.expanduser().resolve() - verify_relay_wheel(source_wheel, args.relay_architecture) + relay_version = verify_relay_wheel(source_wheel, args.relay_architecture) wheel_dir = runtime / "wheels" wheel_dir.mkdir(mode=0o700) relay_wheel = wheel_dir / source_wheel.name shutil.copy2(source_wheel, relay_wheel) else: relay_wheel = download_relay_wheel(runtime / "wheels", args.relay_architecture) - verify_relay_wheel(relay_wheel, args.relay_architecture) + relay_version = verify_relay_wheel(relay_wheel, args.relay_architecture) openinference_endpoint = checked_url(args.openinference_endpoint, "openinference_endpoint") phoenix_project = checked_label(args.phoenix_project, "phoenix_project") @@ -272,7 +265,7 @@ def main() -> int: provenance = { "schema_version": "harbor-hermes-switchyard.phase1.v1", "nemo_relay": { - "version": RELAY_VERSION, + "version": relay_version, "architecture": args.relay_architecture, "wheel": relay_wheel.name, "wheel_sha256": sha256(relay_wheel), diff --git a/examples/harbor-hermes-switchyard/scripts/relay_version.py b/examples/harbor-hermes-switchyard/scripts/relay_version.py new file mode 100644 index 000000000..01e856627 --- /dev/null +++ b/examples/harbor-hermes-switchyard/scripts/relay_version.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Version helpers for the released NeMo Relay wheel used by Phase 2.""" + +from __future__ import annotations + +import re +from pathlib import Path +from zipfile import ZipFile + +RELAY_REQUIREMENT = "nemo-relay>=0.7.0" +RELAY_MIN_VERSION = "0.7.0" + + +def version_tuple(value: str) -> tuple[int, int, int]: + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", value) + if match is None: + raise ValueError(f"nemo-relay version must be a stable semantic version: {value!r}") + return tuple(int(component) for component in match.groups()) + + +def require_supported_version(value: str) -> str: + if version_tuple(value) < version_tuple(RELAY_MIN_VERSION): + raise ValueError(f"nemo-relay must satisfy {RELAY_REQUIREMENT}; found {value}") + return value + + +def wheel_version(path: Path) -> str: + with ZipFile(path) as wheel: + metadata_names = [name for name in wheel.namelist() if name.endswith(".dist-info/METADATA")] + if len(metadata_names) != 1: + raise ValueError("Relay wheel has an ambiguous METADATA payload") + metadata = wheel.read(metadata_names[0]).decode("utf-8", errors="strict") + name = re.search(r"^Name: (.+)$", metadata, re.MULTILINE) + version = re.search(r"^Version: (.+)$", metadata, re.MULTILINE) + if name is None or name.group(1) != "nemo-relay" or version is None: + raise ValueError("Relay wheel metadata does not identify nemo-relay") + return require_supported_version(version.group(1)) diff --git a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh index 4c5a27073..24ef6f34b 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh +++ b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh @@ -83,8 +83,8 @@ docker run --rm \ /tmp/hermes/bin/uv sync --frozen --extra all cd / /tmp/hermes-agent-src/venv/bin/python -c \ - "import importlib.metadata as m; assert m.version(\"nemo-relay\") == \"0.7.0\"" - relay_wheel="$(find /runtime/wheels -maxdepth 1 -type f -name "nemo_relay-0.7.0-*.whl" -print)" + "import importlib.metadata as m; assert tuple(map(int, m.version(\"nemo-relay\").split(\".\"))) >= (0, 7, 0)" + relay_wheel="$(find /runtime/wheels -maxdepth 1 -type f -name "nemo_relay-*.whl" -print)" test -n "$relay_wheel" expected_wheel_sha="$(python3 -c "import json; print(json.load(open(\"/runtime/provenance.json\"))[\"nemo_relay\"][\"wheel_sha256\"])")" test "$(sha256sum "$relay_wheel" | cut -d" " -f1)" = "$expected_wheel_sha" diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index 71d514181..e8aa64e1e 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -28,6 +28,7 @@ if str(_SCRIPT_ROOT) not in sys.path: sys.path.insert(0, str(_SCRIPT_ROOT)) import run_setup_admission as setup_admission # noqa: E402 +from relay_version import wheel_version # noqa: E402 SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-cohort.v1" PLAN_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-plan.v1" @@ -240,7 +241,7 @@ def load_hermetic_runtime(path: Path, args: argparse.Namespace) -> dict[str, Any "schema_version": HERMETIC_RUNTIME_SCHEMA, "status": "passed", "hermes_commit": EXPECTED_HERMES_COMMIT, - "relay_version": "0.7.0", + "relay_version": wheel_version(args.relay_wheel), "relay_wheel_sha256": sha256_file(args.relay_wheel), "relay_architecture": args.relay_architecture, } diff --git a/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py b/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py index 74e89749c..b87255c81 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py +++ b/examples/harbor-hermes-switchyard/scripts/run_setup_admission.py @@ -276,7 +276,7 @@ def build_plan(args: argparse.Namespace, tasks: list[Task], payload: dict[str, A runtime = args.runtime_root provenance_path = runtime / "provenance.json" provenance = json.loads(provenance_path.read_text(encoding="utf-8")) - relay_wheels = sorted((runtime / "wheels").glob("nemo_relay-0.7.0-*.whl")) + relay_wheels = sorted((runtime / "wheels").glob("nemo_relay-*.whl")) if len(relay_wheels) != 1: raise ValueError(f"expected one Relay wheel below {runtime / 'wheels'}") libraries = sorted((runtime / "switchyard-plugin").glob("*.so")) diff --git a/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py index ee33026c6..98922df8b 100755 --- a/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py +++ b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py @@ -20,6 +20,11 @@ from typing import Any from unittest.mock import patch +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +from relay_version import wheel_version + from harbor.job import Job from harbor.models.job.config import DatasetConfig, JobConfig from harbor.models.task.task import Task as HarborTask @@ -273,7 +278,7 @@ def validate_relay_runtime( components = {component["kind"]: component for component in relay_config.get("components", [])} plugin_id = switchyard_manifest.get("plugin", {}).get("id") if ( - provenance.get("nemo_relay", {}).get("version") != "0.7.0" + provenance.get("nemo_relay", {}).get("version") != wheel_version(relay_wheel) or provenance.get("nemo_relay", {}).get("wheel_sha256") != sha256_file(relay_wheel) or provenance.get("switchyard", {}).get("library_sha256") is None or compatibility.get("status") != "passed" @@ -285,7 +290,7 @@ def validate_relay_runtime( raise ValueError("Relay/Hermes/Switchyard smoke wiring did not pass") return { "status": "passed", - "relay_version": "0.7.0", + "relay_version": provenance["nemo_relay"]["version"], "relay_wheel_sha256": provenance["nemo_relay"]["wheel_sha256"], "switchyard_library_sha256": provenance["switchyard"]["library_sha256"], "plugin_config_template_sha256": provenance["plugin_config_template_sha256"], diff --git a/examples/harbor-hermes-switchyard/scripts/validate_run.py b/examples/harbor-hermes-switchyard/scripts/validate_run.py index 7f5d51c4e..1be9bcf10 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_run.py +++ b/examples/harbor-hermes-switchyard/scripts/validate_run.py @@ -8,7 +8,13 @@ import argparse import json import os +import sys from pathlib import Path + +_SCRIPT_ROOT = Path(__file__).resolve().parent +if str(_SCRIPT_ROOT) not in sys.path: + sys.path.insert(0, str(_SCRIPT_ROOT)) +from relay_version import require_supported_version from typing import Any, Iterable SCHEMA_VERSION = "harbor-hermes-switchyard.validation.v1" @@ -147,8 +153,13 @@ def validate_receipt_provenance(receipt: dict[str, Any], provenance: dict[str, A provenance_relay = provenance.get("nemo_relay", {}) provenance_hermes = provenance.get("hermes", {}) provenance_switchyard = provenance.get("switchyard", {}) - if relay.get("version") != "0.7.0": - errors.append("receipt did not record nemo-relay==0.7.0") + if relay.get("version") != provenance_relay.get("version"): + errors.append("receipt Relay version does not match runtime provenance") + else: + try: + require_supported_version(relay.get("version", "")) + except ValueError: + errors.append("receipt did not record nemo-relay>=0.7.0") if relay.get("wheel_sha256") != provenance_relay.get("wheel_sha256"): errors.append("Relay wheel digest does not match runtime provenance") if receipt.get("relay_config_sha256") != provenance.get("relay_config_sha256"): diff --git a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py index 73c4e61df..e24fc5ce2 100644 --- a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py @@ -63,7 +63,7 @@ def test_completed_response_is_preserved_after_post_response_failure(tmp_path: P log.write_text("normal shutdown\n", encoding="utf-8") monkeypatch.setattr(module, "HERMES_SESSION", session) monkeypatch.setattr(module, "HERMES_LOG", log) - monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.0") + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.1") args = make_args(tmp_path, error_type="InjectedPostResponseFailure") module.initialize(args, root) @@ -89,7 +89,7 @@ def test_no_response_never_creates_a_passed_completion(tmp_path: Path, monkeypat log.write_text("agent stopped\n", encoding="utf-8") monkeypatch.setattr(module, "HERMES_SESSION", session) monkeypatch.setattr(module, "HERMES_LOG", log) - monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.0") + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.1") args = make_args(tmp_path) module.initialize(args, root) @@ -109,7 +109,7 @@ def test_empty_session_uses_bounded_quiet_cli_output(tmp_path: Path, monkeypatch log.write_text("startup warning\n\nsession_id: cli-session\ncompleted\nanswer\n", encoding="utf-8") monkeypatch.setattr(module, "HERMES_SESSION", session) monkeypatch.setattr(module, "HERMES_LOG", log) - monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.0") + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.1") args = make_args(tmp_path) module.initialize(args, root) @@ -134,7 +134,7 @@ def test_failed_agent_output_is_not_promoted_to_a_completed_response(tmp_path: P ) monkeypatch.setattr(module, "HERMES_SESSION", session) monkeypatch.setattr(module, "HERMES_LOG", log) - monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.0") + monkeypatch.setattr(module.importlib.metadata, "version", lambda _: "0.7.1") args = make_args(tmp_path, error_type="NonZeroAgentExitCodeError") module.initialize(args, root) diff --git a/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py index 6321243f5..79696773c 100644 --- a/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py +++ b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py @@ -57,4 +57,4 @@ def test_install_verifies_detached_commit_and_relay_release() -> None: assert "rev-parse HEAD" in source assert "/tmp/hermes-install-path/ffmpeg" in source assert "uv sync --frozen --extra all" in source - assert "m.version('nemo-relay') == '0.7.0'" in source + assert "m.version('nemo-relay').split('.')" in source diff --git a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py index ffad0e7ad..40a0fa884 100644 --- a/examples/harbor-hermes-switchyard/tests/test_validation_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_validation_contract.py @@ -134,7 +134,7 @@ def test_secret_scan_finds_raw_key_within_artifact(tmp_path: Path) -> None: def test_receipt_provenance_validation_covers_every_staged_digest() -> None: module = load_validator() provenance = { - "nemo_relay": {"wheel_sha256": "relay"}, + "nemo_relay": {"version": "0.7.1", "wheel_sha256": "relay"}, "hermes": {"commit": "hermes"}, "switchyard": { "commit": "switchyard", @@ -148,7 +148,7 @@ def test_receipt_provenance_validation_covers_every_staged_digest() -> None: "dynamic_plugin_ids": ["nvidia.switchyard"], "relay_config_sha256": "config", "dependencies": { - "nemo_relay": {"version": "0.7.0", "wheel_sha256": "relay"}, + "nemo_relay": {"version": "0.7.1", "wheel_sha256": "relay"}, "hermes": {"commit": "hermes"}, "switchyard": { "commit": "switchyard", From c6aba5ffc0a919dd9d747895f602b01555cb1159 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 13:05:36 -0600 Subject: [PATCH 26/34] chore(example): refresh Hermes commit pin Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 2 +- examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py | 2 +- examples/harbor-hermes-switchyard/run_terminal_bench.sh | 2 +- .../harbor-hermes-switchyard/scripts/build_hermetic_runtime.py | 2 +- examples/harbor-hermes-switchyard/scripts/prepare_runtime.py | 2 +- .../scripts/run_offline_compatibility_smoke.sh | 2 +- examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py | 2 +- .../harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py | 2 +- .../tests/test_agent_result_contract.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 450b716bc..8cdf70459 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -11,7 +11,7 @@ and result aggregation are intentionally out of scope for this example. | Dependency | Input used by this example | |---|---| | NeMo Relay | Latest released `nemo-relay>=0.7.0` platform wheel, installed by digest rather than from this source checkout. | -| Hermes | `bbednarski9/hermes-agent`, detached commit `efb63e714abc436af88af9b0d6734751c199aa6d` from PR #77915. | +| Hermes | `bbednarski9/hermes-agent`, detached commit `a3d472f0e6bdc376df87b1436a461c4796db6747` from PR #77915. | | Switchyard | `bbednarski9/Switchyard`, detached commit `8daac03edf8544144833af1fd009b3da737715bc` from PR #270. | | Harbor | `harbor==0.18.0`, local export of dataset `terminal-bench@2.0`. | diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py index 452ff82fd..c9332398b 100644 --- a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -30,7 +30,7 @@ _SHA256 = re.compile(r"[0-9a-f]{64}") _DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" _DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" -_DEFAULT_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" +_DEFAULT_HERMES_COMMIT = "a3d472f0e6bdc376df87b1436a461c4796db6747" _DEFAULT_SWITCHYARD_COMMIT = "8daac03edf8544144833af1fd009b3da737715bc" _ENV_NAME = re.compile(r"[A-Z_][A-Z0-9_]*") _PROVIDER_AUTHORIZATION_FILE = "/run/secrets/switchyard-provider-authorization" diff --git a/examples/harbor-hermes-switchyard/run_terminal_bench.sh b/examples/harbor-hermes-switchyard/run_terminal_bench.sh index 3f47dfdf7..6cf56c1f5 100755 --- a/examples/harbor-hermes-switchyard/run_terminal_bench.sh +++ b/examples/harbor-hermes-switchyard/run_terminal_bench.sh @@ -260,7 +260,7 @@ fi --model "openai/$hermes_caller_model" \ --ak "repository_url=https://github.com/bbednarski9/hermes-agent.git" \ --ak "repository_ref=feat/relay-native-plugin-init" \ - --ak "commit=efb63e714abc436af88af9b0d6734751c199aa6d" \ + --ak "commit=a3d472f0e6bdc376df87b1436a461c4796db6747" \ --ak "relay_config_path=$run_root/runtime/plugins.toml" \ --ak "switchyard_bundle_dir=$run_root/runtime/switchyard-plugin" \ --ak "relay_wheel_path=$relay_wheel_path" \ diff --git a/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py b/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py index ac8f206ef..edf06ae3d 100755 --- a/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/build_hermetic_runtime.py @@ -31,7 +31,7 @@ SCHEMA_VERSION = "harbor-hermes-switchyard.hermetic-runtime.v1" DEFAULT_HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" DEFAULT_HERMES_REF = "feat/relay-native-plugin-init" -DEFAULT_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" +DEFAULT_HERMES_COMMIT = "a3d472f0e6bdc376df87b1436a461c4796db6747" UV_VERSION = "0.11.16" PYTHON_VERSION = "3.11.13" BUILDER_IMAGE = "python:3.11-bullseye" diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index 877ff5192..a3f8eba89 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -22,7 +22,7 @@ HERMES_REPOSITORY = "https://github.com/bbednarski9/hermes-agent.git" HERMES_REF = "feat/relay-native-plugin-init" -HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" +HERMES_COMMIT = "a3d472f0e6bdc376df87b1436a461c4796db6747" SWITCHYARD_REPOSITORY = "https://github.com/bbednarski9/Switchyard.git" SWITCHYARD_COMMIT = "8daac03edf8544144833af1fd009b3da737715bc" SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]{0,127}") diff --git a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh index 24ef6f34b..a1f156628 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh +++ b/examples/harbor-hermes-switchyard/scripts/run_offline_compatibility_smoke.sh @@ -11,7 +11,7 @@ image="${OFFLINE_COMPAT_IMAGE:-python:3.11-bookworm}" platform="${OFFLINE_COMPAT_PLATFORM:-linux/amd64}" hermes_repository="${HERMES_REPOSITORY:-https://github.com/bbednarski9/hermes-agent.git}" hermes_ref="${HERMES_REF:-feat/relay-native-plugin-init}" -hermes_commit="${HERMES_COMMIT:-efb63e714abc436af88af9b0d6734751c199aa6d}" +hermes_commit="${HERMES_COMMIT:-a3d472f0e6bdc376df87b1436a461c4796db6747}" if [[ -z "$run_root" || "$run_root" != /* ]]; then echo "usage: $0 /absolute/prepared-run-root" >&2 diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index e8aa64e1e..bb5c9891a 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -33,7 +33,7 @@ SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-cohort.v1" PLAN_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-plan.v1" TASK_STATE_SCHEMA_VERSION = "harbor-hermes-switchyard.phase2-task-state.v1" -EXPECTED_HERMES_COMMIT = "efb63e714abc436af88af9b0d6734751c199aa6d" +EXPECTED_HERMES_COMMIT = "a3d472f0e6bdc376df87b1436a461c4796db6747" HERMETIC_RUNTIME_SCHEMA = "harbor-hermes-switchyard.hermetic-runtime.v1" INFRASTRUCTURE_PATTERNS = ( "apt-get update && apt-get install", diff --git a/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py index 98922df8b..56b829c52 100755 --- a/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py +++ b/examples/harbor-hermes-switchyard/scripts/smoke_phase2_dataset.py @@ -133,7 +133,7 @@ def deny_network(*_args: Any, **_kwargs: Any) -> None: kwargs={ "repository_url": "https://github.com/bbednarski9/hermes-agent.git", "repository_ref": "feat/relay-native-plugin-init", - "commit": "efb63e714abc436af88af9b0d6734751c199aa6d", + "commit": "a3d472f0e6bdc376df87b1436a461c4796db6747", "relay_config_path": "/smoke/runtime/plugins.toml", "switchyard_bundle_dir": "/smoke/runtime/switchyard-plugin", "relay_wheel_path": "/smoke/runtime/nemo-relay.whl", diff --git a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py index e24fc5ce2..fe41de1bc 100644 --- a/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py +++ b/examples/harbor-hermes-switchyard/tests/test_agent_result_contract.py @@ -33,7 +33,7 @@ def make_args(tmp_path: Path, *, error_type: str = "") -> argparse.Namespace: switchyard_library=library, relay_wheel_sha256="a" * 64, hermes_repository="https://github.com/bbednarski9/hermes-agent.git", - hermes_commit="efb63e714abc436af88af9b0d6734751c199aa6d", + hermes_commit="a3d472f0e6bdc376df87b1436a461c4796db6747", switchyard_commit="8daac03edf8544144833af1fd009b3da737715bc", session_handle="phase1-session", started_at=1.0, From 29d64d63836760856724b284781b657c9daac29a Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 13:36:03 -0600 Subject: [PATCH 27/34] feat(example): configure GLM classifier routing Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 10 +++--- .../config/plugins.toml.in | 35 ++++++++++++++++--- .../scripts/prepare_runtime.py | 22 +++++++----- .../scripts/run_phase2_cohort.py | 13 ++++--- .../tests/test_phase2_cohort.py | 5 +-- 5 files changed, 62 insertions(+), 23 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 8cdf70459..7f7f6ab15 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -48,13 +48,15 @@ The two configuration files have deliberately different responsibilities: destination, manually selected capacity, and the real `SWITCHYARD_PROVIDER_AUTHORIZATION` header. - `config/plugins.toml.in` is checked in and non-secret. It is the only source - of provider URLs, protocols, strong and weak models, routing/classifier + of provider URLs, protocols, strong, weak, and judge models, routing/classifier policy, native plugin manifest, authorization variable **name**, Relay components, and OpenInference export behavior. -The template configures Opus 4.6 as the strong route and Sonnet 4.6 as the -classifier and weak route. The coordinator derives its required route-diversity -gates from this TOML; environment variables cannot override these settings. +The template configures Opus 4.8 as the strong route, GLM 5.2 as the efficient +route, and Sonnet 4.6 as the classifier judge, with a `0.5` threshold and +session affinity. The coordinator derives its required route-diversity gates +from the strong and efficient targets, while preflight also verifies the judge. +Environment variables cannot override these settings. Runtime rendering is limited to the Hermes revision, collector endpoint, Phoenix project/cohort attributes, and task-owned artifact locations. diff --git a/examples/harbor-hermes-switchyard/config/plugins.toml.in b/examples/harbor-hermes-switchyard/config/plugins.toml.in index 06a34f157..a5a696d80 100644 --- a/examples/harbor-hermes-switchyard/config/plugins.toml.in +++ b/examples/harbor-hermes-switchyard/config/plugins.toml.in @@ -15,7 +15,7 @@ version = 1 [[components.config.sources.catalog.entries]] provider = "openai" -model_id = "aws/anthropic/bedrock-claude-opus-4-6" +model_id = "azure/anthropic/claude-opus-4-8" currency = "USD" unit = "per_token" pricing_as_of = "2026-05-27" @@ -47,6 +47,22 @@ cache_write_per_million = 3.75 [components.config.sources.catalog.entries.prompt_cache] read_accounting = "included_in_prompt_tokens" +[[components.config.sources.catalog.entries]] +provider = "openai" +model_id = "nvidia/zai-org/glm-5.2" +currency = "USD" +unit = "per_token" +pricing_as_of = "2026-08-07" +pricing_source = "Z.ai public API list pricing" + +[components.config.sources.catalog.entries.rates] +input_per_million = 1.4 +output_per_million = 4.4 +cache_read_per_million = 0.26 + +[components.config.sources.catalog.entries.prompt_cache] +read_accounting = "included_in_prompt_tokens" + [[components]] kind = "observability" enabled = true @@ -97,7 +113,7 @@ max_retries = 1 [plugins.dynamic.config.algorithm] kind = "llm_classifier" -classifier_target = "weak" +classifier_target = "judge" weak_target = "weak" strong_target = "strong" base_threshold = 0.5 @@ -109,7 +125,7 @@ message_hash_fallback = true openai_chat = "strong" [plugins.dynamic.config.targets.strong] -model = "aws/anthropic/bedrock-claude-opus-4-6" +model = "azure/anthropic/claude-opus-4-8" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "https://inference-api.nvidia.com/v1" @@ -120,7 +136,7 @@ drop_caller_extra_body = true authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" [plugins.dynamic.config.targets.weak] -model = "aws/anthropic/bedrock-claude-sonnet-4-6" +model = "nvidia/zai-org/glm-5.2" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "https://inference-api.nvidia.com/v1" @@ -129,3 +145,14 @@ drop_caller_extra_body = true [plugins.dynamic.config.targets.weak.header_env] authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" + +[plugins.dynamic.config.targets.judge] +model = "azure/anthropic/claude-sonnet-4-6" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://inference-api.nvidia.com/v1" +weight = 1 +drop_caller_extra_body = true + +[plugins.dynamic.config.targets.judge.header_env] +authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index a3f8eba89..6df6f1eb5 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -115,10 +115,10 @@ def plugin_settings(config: dict[str, object]) -> dict[str, str]: if not isinstance(plugin_config, dict) or not isinstance(plugin_config.get("targets"), dict): raise ValueError("Switchyard dynamic plugin targets are missing") targets = plugin_config["targets"] - if set(targets) != {"strong", "weak"}: - raise ValueError("Switchyard must define strong and weak targets") + if set(targets) != {"strong", "weak", "judge"}: + raise ValueError("Switchyard must define strong, weak, and judge targets") settings: dict[str, str] = {} - for name in ("strong", "weak"): + for name in ("strong", "weak", "judge"): target = targets[name] if not isinstance(target, dict): raise ValueError(f"Switchyard target is invalid: {name}") @@ -150,7 +150,9 @@ def plugin_settings(config: dict[str, object]) -> dict[str, str]: settings["hermes_caller_model"] = checked_label( str(observation_config["atif"].get("model_name", "")), "hermes_caller_model" ) - if settings["hermes_caller_model"] in {settings["strong_model"], settings["weak_model"]}: + if settings["hermes_caller_model"] in { + settings["strong_model"], settings["weak_model"], settings["judge_model"] + }: raise ValueError("Hermes caller model must be distinct from Switchyard targets") return settings @@ -172,12 +174,16 @@ def render_config( config = tomllib.loads(rendered) if test_overrides: plugin = config["plugins"]["dynamic"][0]["config"] - old_models = {name: plugin["targets"][name]["model"] for name in ("strong", "weak")} - for name in ("strong", "weak"): - plugin["targets"][name]["model"] = test_overrides[f"{name}_model"] + old_models = {name: plugin["targets"][name]["model"] for name in ("strong", "weak", "judge")} + for name in ("strong", "weak", "judge"): + override = "weak_model" if name == "judge" else f"{name}_model" + plugin["targets"][name]["model"] = test_overrides[override] plugin["targets"][name]["base_url"] = test_overrides["provider_base_url"] pricing = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] - replacement_models = {old_models[name]: test_overrides[f"{name}_model"] for name in old_models} + replacement_models = { + old_models[name]: test_overrides["weak_model" if name == "judge" else f"{name}_model"] + for name in old_models + } for entry in pricing: entry["model_id"] = replacement_models.get(entry["model_id"], entry["model_id"]) settings = plugin_settings(config) diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index bb5c9891a..32e1aa054 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -201,14 +201,15 @@ def plugin_contract(path: Path) -> dict[str, Any]: if len(plugins) != 1: raise ValueError("plugin configuration must define exactly one dynamic plugin") targets = plugins[0].get("config", {}).get("targets", {}) - if set(targets) != {"strong", "weak"}: - raise ValueError("plugin configuration must define strong and weak targets") + if set(targets) != {"strong", "weak", "judge"}: + raise ValueError("plugin configuration must define strong, weak, and judge targets") for target in targets.values(): if target.get("header_env") != {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"}: raise ValueError("plugin authorization must reference SWITCHYARD_PROVIDER_AUTHORIZATION") models = [targets[name].get("model") for name in ("weak", "strong")] - base_urls = sorted({targets[name].get("base_url") for name in ("weak", "strong")}) - if any(not isinstance(value, str) or not value for value in models + base_urls): + catalog_models = [*models, targets["judge"].get("model")] + base_urls = sorted({targets[name].get("base_url") for name in ("weak", "strong", "judge")}) + if any(not isinstance(value, str) or not value for value in catalog_models + base_urls): raise ValueError("plugin target models and base URLs must be non-empty") for base_url in base_urls: parsed = urllib.parse.urlsplit(base_url) @@ -220,8 +221,10 @@ def plugin_contract(path: Path) -> dict[str, Any]: raise ValueError("Hermes caller model must be distinct from plugin target models") return { "required_models": sorted(models), + "catalog_models": sorted(catalog_models), "strong_model": targets["strong"]["model"], "weak_model": targets["weak"]["model"], + "judge_model": targets["judge"]["model"], "hermes_caller_model": caller, "provider_base_urls": base_urls, "sha256": sha256_file(path), @@ -564,7 +567,7 @@ def shared_preflight(args: argparse.Namespace, tasks: list[Task]) -> dict[str, A verify_provider_catalog( provider_url, provider_authorization, - args.plugin_contract["required_models"], + args.plugin_contract["catalog_models"], ) ) if not args.switchyard_bundle.is_dir(): diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index d8772d8da..3f548f896 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -446,8 +446,9 @@ def test_runner_keeps_benchmark_completion_separate_from_integration_validation( def test_plugin_contract_owns_routes_and_authorization_name() -> None: module = load_coordinator() contract = module.plugin_contract(EXAMPLE_ROOT / "config" / "plugins.toml.in") - assert contract["strong_model"] == "aws/anthropic/bedrock-claude-opus-4-6" - assert contract["weak_model"] == "aws/anthropic/bedrock-claude-sonnet-4-6" + assert contract["strong_model"] == "azure/anthropic/claude-opus-4-8" + assert contract["weak_model"] == "nvidia/zai-org/glm-5.2" + assert contract["judge_model"] == "azure/anthropic/claude-sonnet-4-6" assert contract["hermes_caller_model"] == "ollama-route-stub" assert contract["provider_base_urls"] == ["https://inference-api.nvidia.com/v1"] From ae46312dfe936f9ff50853695e2d36def9f542f6 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 13:52:08 -0600 Subject: [PATCH 28/34] fix(example): install finalizer support module Signed-off-by: Bryan Bednarski --- .../agents/harbor_hermes_agent.py | 4 ++++ .../tests/test_plugin_lifecycle.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py index c9332398b..0155df6d8 100644 --- a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -375,6 +375,9 @@ def __init__( self._finalizer_path = self._example_root / "scripts" / "finalize_artifacts.py" if not self._finalizer_path.is_file(): raise FileNotFoundError(self._finalizer_path) + self._relay_version_path = self._example_root / "scripts" / "relay_version.py" + if not self._relay_version_path.is_file(): + raise FileNotFoundError(self._relay_version_path) extra_env = dict(kwargs.pop("extra_env", None) or {}) extra_env["HERMES_NEMO_RELAY_PLUGINS_TOML"] = "/tmp/hermes/relay/plugins.toml" @@ -520,6 +523,7 @@ async def setup(self, environment: BaseEnvironment) -> None: ) await environment.upload_dir(self.switchyard_bundle_dir, "/opt/relay-plugins/nvidia.switchyard") await environment.upload_file(self._finalizer_path, "/installed-agent/finalize_artifacts.py") + await environment.upload_file(self._relay_version_path, "/installed-agent/relay_version.py") probe_python = ( f"{_HERMETIC_RUNTIME_ROOT}/bin/python" if self.hermetic_runtime_dir is not None diff --git a/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py index 79696773c..a433eb699 100644 --- a/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py +++ b/examples/harbor-hermes-switchyard/tests/test_plugin_lifecycle.py @@ -58,3 +58,17 @@ def test_install_verifies_detached_commit_and_relay_release() -> None: assert "/tmp/hermes-install-path/ffmpeg" in source assert "uv sync --frozen --extra all" in source assert "m.version('nemo-relay').split('.')" in source + + +def test_setup_uploads_finalizer_with_its_relay_version_dependency() -> None: + uploads = { + (ast.unparse(call.args[0]), ast.literal_eval(call.args[1])) + for call in ast.walk(method("setup")) + if isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "upload_file" + and len(call.args) == 2 + and isinstance(call.args[1], ast.Constant) + } + assert ("self._finalizer_path", "/installed-agent/finalize_artifacts.py") in uploads + assert ("self._relay_version_path", "/installed-agent/relay_version.py") in uploads From 5a65bf920995eb41d0abd2db8de8ae40d5087959 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 13:54:42 -0600 Subject: [PATCH 29/34] fix(example): validate classifier judge target Signed-off-by: Bryan Bednarski --- .../scripts/validate_phase2_environment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh index f8be22c79..9ff693512 100755 --- a/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh +++ b/examples/harbor-hermes-switchyard/scripts/validate_phase2_environment.sh @@ -85,8 +85,8 @@ plugins = config.get("plugins", {}).get("dynamic", []) if len(plugins) != 1: raise SystemExit("plugin config must contain one dynamic plugin") targets = plugins[0].get("config", {}).get("targets", {}) -if set(targets) != {"strong", "weak"}: - raise SystemExit("plugin config must contain strong and weak targets") +if set(targets) != {"strong", "weak", "judge"}: + raise SystemExit("plugin config must contain strong, weak, and judge targets") for target in targets.values(): if target.get("header_env") != {"authorization": "SWITCHYARD_PROVIDER_AUTHORIZATION"}: raise SystemExit("plugin config must reference SWITCHYARD_PROVIDER_AUTHORIZATION") From 0a961ecc4dbb2f337ce5ad623392060eef9010bd Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 13:56:44 -0600 Subject: [PATCH 30/34] fix(example): align classifier routing contract Signed-off-by: Bryan Bednarski --- .../agents/harbor_hermes_agent.py | 10 +++++----- .../harbor-hermes-switchyard/config/plugins.toml.in | 2 +- .../tests/test_setup_admission.py | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py index 0155df6d8..ff33f7a0b 100644 --- a/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py +++ b/examples/harbor-hermes-switchyard/agents/harbor_hermes_agent.py @@ -192,7 +192,7 @@ def _validate_relay_config(path: Path) -> None: algorithm = plugin_config.get("algorithm") expected_algorithm = { "kind": "llm_classifier", - "classifier_target": "weak", + "classifier_target": "judge", "weak_target": "weak", "strong_target": "strong", "base_threshold": 0.5, @@ -206,8 +206,8 @@ def _validate_relay_config(path: Path) -> None: raise ValueError("the Switchyard OpenAI default target must be strong") targets = plugin_config.get("targets") - if not isinstance(targets, dict) or set(targets) != {"strong", "weak"}: - raise ValueError("Switchyard must define exactly the strong and weak targets") + if not isinstance(targets, dict) or set(targets) != {"strong", "weak", "judge"}: + raise ValueError("Switchyard must define exactly the strong, weak, and judge targets") provider_models: set[str] = set() for name, target in targets.items(): if not isinstance(target, dict): @@ -234,8 +234,8 @@ def _validate_relay_config(path: Path) -> None: authorization_env = header_env.get("authorization") if isinstance(header_env, dict) else None if not isinstance(authorization_env, str) or not _ENV_NAME.fullmatch(authorization_env): raise ValueError(f"Switchyard target {name!r} must source authorization from an environment variable") - if len(provider_models) != 2: - raise ValueError("Switchyard strong and weak targets must use distinct models") + if len(provider_models) != 3: + raise ValueError("Switchyard strong, weak, and judge targets must use distinct models") pricing = _find_named_component(config, "pricing") pricing_config = pricing.get("config") diff --git a/examples/harbor-hermes-switchyard/config/plugins.toml.in b/examples/harbor-hermes-switchyard/config/plugins.toml.in index a5a696d80..7c0ea9b05 100644 --- a/examples/harbor-hermes-switchyard/config/plugins.toml.in +++ b/examples/harbor-hermes-switchyard/config/plugins.toml.in @@ -32,7 +32,7 @@ read_accounting = "included_in_prompt_tokens" [[components.config.sources.catalog.entries]] provider = "openai" -model_id = "aws/anthropic/bedrock-claude-sonnet-4-6" +model_id = "azure/anthropic/claude-sonnet-4-6" currency = "USD" unit = "per_token" pricing_as_of = "2026-05-27" diff --git a/examples/harbor-hermes-switchyard/tests/test_setup_admission.py b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py index 609932869..f0aa41e7c 100644 --- a/examples/harbor-hermes-switchyard/tests/test_setup_admission.py +++ b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py @@ -116,6 +116,10 @@ def test_setup_admission_binds_agent_source() -> None: assert admission_module.sha256_file(expected) == agent_module._sha256(expected) +def test_bridge_accepts_current_classifier_routing_contract() -> None: + agent_module._validate_relay_config(EXAMPLE_ROOT / "config" / "plugins.toml.in") + + def test_hermetic_runtime_requires_portable_ca_bundle(tmp_path: Path) -> None: marker = make_payload(tmp_path) (tmp_path / agent_module._HERMETIC_CA_BUNDLE_RELATIVE).unlink() From 55f02b8d9eaa9805758f57bdbc4b95b4e634890e Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 13:58:12 -0600 Subject: [PATCH 31/34] fix(example): derive classifier provenance Signed-off-by: Bryan Bednarski --- .../scripts/prepare_runtime.py | 6 +++++- .../tests/test_setup_admission.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index 6df6f1eb5..66cc5af7d 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -118,6 +118,11 @@ def plugin_settings(config: dict[str, object]) -> dict[str, str]: if set(targets) != {"strong", "weak", "judge"}: raise ValueError("Switchyard must define strong, weak, and judge targets") settings: dict[str, str] = {} + algorithm = plugin_config.get("algorithm") + classifier_target = algorithm.get("classifier_target") if isinstance(algorithm, dict) else None + if classifier_target not in targets: + raise ValueError("Switchyard classifier_target must reference a configured target") + settings["classifier_target"] = classifier_target for name in ("strong", "weak", "judge"): target = targets[name] if not isinstance(target, dict): @@ -292,7 +297,6 @@ def main() -> int: "plugin_config_template_sha256": sha256(plugin_template), "routing": { "algorithm": "llm_classifier", - "classifier_target": "weak", **routing, }, "phoenix_project": phoenix_project, diff --git a/examples/harbor-hermes-switchyard/tests/test_setup_admission.py b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py index f0aa41e7c..97b9d7fc1 100644 --- a/examples/harbor-hermes-switchyard/tests/test_setup_admission.py +++ b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py @@ -8,6 +8,7 @@ import json import os import subprocess +import tomllib from pathlib import Path from types import ModuleType @@ -36,6 +37,10 @@ def load_module(name: str, path: Path) -> ModuleType: "hermetic_runtime_builder", EXAMPLE_ROOT / "scripts" / "build_hermetic_runtime.py", ) +runtime_preparer_module = load_module( + "phase2_runtime_preparer", + EXAMPLE_ROOT / "scripts" / "prepare_runtime.py", +) def make_payload(root: Path, *, digest: str = "a" * 64) -> dict[str, object]: @@ -120,6 +125,12 @@ def test_bridge_accepts_current_classifier_routing_contract() -> None: agent_module._validate_relay_config(EXAMPLE_ROOT / "config" / "plugins.toml.in") +def test_runtime_provenance_derives_classifier_target_from_plugin_config() -> None: + with (EXAMPLE_ROOT / "config" / "plugins.toml.in").open("rb") as stream: + config = tomllib.load(stream) + assert runtime_preparer_module.plugin_settings(config)["classifier_target"] == "judge" + + def test_hermetic_runtime_requires_portable_ca_bundle(tmp_path: Path) -> None: marker = make_payload(tmp_path) (tmp_path / agent_module._HERMETIC_CA_BUNDLE_RELATIVE).unlink() From c2dd11ce8aea497c9f00575493ad966345511582 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 14:01:19 -0600 Subject: [PATCH 32/34] Configure AWS Claude 5 routing targets Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/README.md | 6 +++-- .../config/plugins.toml.in | 25 ++++++++++--------- .../tests/test_phase2_cohort.py | 6 ++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/examples/harbor-hermes-switchyard/README.md b/examples/harbor-hermes-switchyard/README.md index 7f7f6ab15..bd1c35bb7 100644 --- a/examples/harbor-hermes-switchyard/README.md +++ b/examples/harbor-hermes-switchyard/README.md @@ -52,8 +52,9 @@ The two configuration files have deliberately different responsibilities: policy, native plugin manifest, authorization variable **name**, Relay components, and OpenInference export behavior. -The template configures Opus 4.8 as the strong route, GLM 5.2 as the efficient -route, and Sonnet 4.6 as the classifier judge, with a `0.5` threshold and +The template configures AWS-hosted Opus 5 as the strong route, AWS-hosted +Sonnet 5 as the efficient route, and AWS-hosted Sonnet 4.6 as the classifier +judge, with a `0.5` threshold and session affinity. The coordinator derives its required route-diversity gates from the strong and efficient targets, while preflight also verifies the judge. Environment variables cannot override these settings. @@ -168,6 +169,7 @@ OFFLINE_ROOT="$TERMINAL_BENCH_ADMISSION_ROOT/offline-runtime" --test-provider-base-url http://127.0.0.1:8000/v1 \ --test-strong-model phase2/fake-strong \ --test-weak-model phase2/fake-weak \ + --test-judge-model phase2/fake-judge \ --openinference-endpoint http://127.0.0.1:4318/v1/traces \ --phoenix-project phase2-offline \ --eval-cohort phase2-offline diff --git a/examples/harbor-hermes-switchyard/config/plugins.toml.in b/examples/harbor-hermes-switchyard/config/plugins.toml.in index 7c0ea9b05..e391f86bc 100644 --- a/examples/harbor-hermes-switchyard/config/plugins.toml.in +++ b/examples/harbor-hermes-switchyard/config/plugins.toml.in @@ -15,10 +15,10 @@ version = 1 [[components.config.sources.catalog.entries]] provider = "openai" -model_id = "azure/anthropic/claude-opus-4-8" +model_id = "aws/anthropic/bedrock-claude-opus-5" currency = "USD" unit = "per_token" -pricing_as_of = "2026-05-27" +pricing_as_of = "2026-08-07" pricing_source = "Anthropic public list pricing" [components.config.sources.catalog.entries.rates] @@ -32,10 +32,10 @@ read_accounting = "included_in_prompt_tokens" [[components.config.sources.catalog.entries]] provider = "openai" -model_id = "azure/anthropic/claude-sonnet-4-6" +model_id = "aws/anthropic/bedrock-claude-sonnet-4-6" currency = "USD" unit = "per_token" -pricing_as_of = "2026-05-27" +pricing_as_of = "2026-08-07" pricing_source = "Anthropic public list pricing" [components.config.sources.catalog.entries.rates] @@ -49,16 +49,17 @@ read_accounting = "included_in_prompt_tokens" [[components.config.sources.catalog.entries]] provider = "openai" -model_id = "nvidia/zai-org/glm-5.2" +model_id = "aws/anthropic/bedrock-claude-sonnet-5" currency = "USD" unit = "per_token" pricing_as_of = "2026-08-07" -pricing_source = "Z.ai public API list pricing" +pricing_source = "Anthropic public list pricing" [components.config.sources.catalog.entries.rates] -input_per_million = 1.4 -output_per_million = 4.4 -cache_read_per_million = 0.26 +input_per_million = 3.0 +output_per_million = 15.0 +cache_read_per_million = 0.3 +cache_write_per_million = 3.75 [components.config.sources.catalog.entries.prompt_cache] read_accounting = "included_in_prompt_tokens" @@ -125,7 +126,7 @@ message_hash_fallback = true openai_chat = "strong" [plugins.dynamic.config.targets.strong] -model = "azure/anthropic/claude-opus-4-8" +model = "aws/anthropic/bedrock-claude-opus-5" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "https://inference-api.nvidia.com/v1" @@ -136,7 +137,7 @@ drop_caller_extra_body = true authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" [plugins.dynamic.config.targets.weak] -model = "nvidia/zai-org/glm-5.2" +model = "aws/anthropic/bedrock-claude-sonnet-5" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "https://inference-api.nvidia.com/v1" @@ -147,7 +148,7 @@ drop_caller_extra_body = true authorization = "SWITCHYARD_PROVIDER_AUTHORIZATION" [plugins.dynamic.config.targets.judge] -model = "azure/anthropic/claude-sonnet-4-6" +model = "aws/anthropic/bedrock-claude-sonnet-4-6" protocol = "openai_chat" endpoint = "/v1/chat/completions" base_url = "https://inference-api.nvidia.com/v1" diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 3f548f896..6e4debd23 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -446,9 +446,9 @@ def test_runner_keeps_benchmark_completion_separate_from_integration_validation( def test_plugin_contract_owns_routes_and_authorization_name() -> None: module = load_coordinator() contract = module.plugin_contract(EXAMPLE_ROOT / "config" / "plugins.toml.in") - assert contract["strong_model"] == "azure/anthropic/claude-opus-4-8" - assert contract["weak_model"] == "nvidia/zai-org/glm-5.2" - assert contract["judge_model"] == "azure/anthropic/claude-sonnet-4-6" + assert contract["strong_model"] == "aws/anthropic/bedrock-claude-opus-5" + assert contract["weak_model"] == "aws/anthropic/bedrock-claude-sonnet-5" + assert contract["judge_model"] == "aws/anthropic/bedrock-claude-sonnet-4-6" assert contract["hermes_caller_model"] == "ollama-route-stub" assert contract["provider_base_urls"] == ["https://inference-api.nvidia.com/v1"] From 9f4b3ae958dd0076db13829ab7267f5545802066 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 14:01:41 -0600 Subject: [PATCH 33/34] fix(example): isolate offline judge override Signed-off-by: Bryan Bednarski --- .../scripts/prepare_runtime.py | 8 +++-- .../tests/test_setup_admission.py | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py index 66cc5af7d..5e8c54b7b 100755 --- a/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py +++ b/examples/harbor-hermes-switchyard/scripts/prepare_runtime.py @@ -181,12 +181,12 @@ def render_config( plugin = config["plugins"]["dynamic"][0]["config"] old_models = {name: plugin["targets"][name]["model"] for name in ("strong", "weak", "judge")} for name in ("strong", "weak", "judge"): - override = "weak_model" if name == "judge" else f"{name}_model" + override = f"{name}_model" plugin["targets"][name]["model"] = test_overrides[override] plugin["targets"][name]["base_url"] = test_overrides["provider_base_url"] pricing = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] replacement_models = { - old_models[name]: test_overrides["weak_model" if name == "judge" else f"{name}_model"] + old_models[name]: test_overrides[f"{name}_model"] for name in old_models } for entry in pricing: @@ -207,6 +207,7 @@ def main() -> int: parser.add_argument("--test-provider-base-url") parser.add_argument("--test-strong-model") parser.add_argument("--test-weak-model") + parser.add_argument("--test-judge-model") parser.add_argument("--openinference-endpoint", required=True) parser.add_argument("--phoenix-project", required=True) parser.add_argument("--eval-cohort", required=True) @@ -244,7 +245,7 @@ def main() -> int: phoenix_project = checked_label(args.phoenix_project, "phoenix_project") eval_cohort = checked_label(args.eval_cohort, "eval_cohort") plugin_template = (args.plugin_config_template or example_root / "config" / "plugins.toml.in").resolve(strict=True) - test_values = (args.test_provider_base_url, args.test_strong_model, args.test_weak_model) + test_values = (args.test_provider_base_url, args.test_strong_model, args.test_weak_model, args.test_judge_model) if any(test_values) and not all(test_values): raise ValueError("all test provider overrides must be supplied together") test_overrides = None @@ -253,6 +254,7 @@ def main() -> int: "provider_base_url": checked_url(args.test_provider_base_url, "test_provider_base_url"), "strong_model": checked_label(args.test_strong_model, "test_strong_model"), "weak_model": checked_label(args.test_weak_model, "test_weak_model"), + "judge_model": checked_label(args.test_judge_model, "test_judge_model"), } config_path = runtime / "plugins.toml" diff --git a/examples/harbor-hermes-switchyard/tests/test_setup_admission.py b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py index 97b9d7fc1..e0cc5f5d2 100644 --- a/examples/harbor-hermes-switchyard/tests/test_setup_admission.py +++ b/examples/harbor-hermes-switchyard/tests/test_setup_admission.py @@ -131,6 +131,35 @@ def test_runtime_provenance_derives_classifier_target_from_plugin_config() -> No assert runtime_preparer_module.plugin_settings(config)["classifier_target"] == "judge" +def test_offline_overrides_keep_classifier_pricing_aliases_distinct(tmp_path: Path) -> None: + output = tmp_path / "plugins.toml" + settings = runtime_preparer_module.render_config( + EXAMPLE_ROOT / "config" / "plugins.toml.in", + output, + { + "HERMES_COMMIT": "a" * 40, + "OPENINFERENCE_ENDPOINT": "http://127.0.0.1:4318/v1/traces", + "PHOENIX_PROJECT": "offline", + "EVAL_COHORT": "offline", + }, + { + "provider_base_url": "http://127.0.0.1:8000/v1", + "strong_model": "phase2/fake-strong", + "weak_model": "phase2/fake-weak", + "judge_model": "phase2/fake-judge", + }, + ) + with output.open("rb") as stream: + config = tomllib.load(stream) + entries = config["components"][0]["config"]["sources"][0]["catalog"]["entries"] + assert settings["judge_model"] == "phase2/fake-judge" + assert {entry["model_id"] for entry in entries} == { + "phase2/fake-strong", + "phase2/fake-weak", + "phase2/fake-judge", + } + + def test_hermetic_runtime_requires_portable_ca_bundle(tmp_path: Path) -> None: marker = make_payload(tmp_path) (tmp_path / agent_module._HERMETIC_CA_BUNDLE_RELATIVE).unlink() From 351f3bf51a773833676baf399f85b53a3b30b140 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Fri, 7 Aug 2026 14:37:28 -0600 Subject: [PATCH 34/34] fix(example): retry terminated agent attempts Signed-off-by: Bryan Bednarski --- examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py | 1 + examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py index 32e1aa054..f01251867 100755 --- a/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/scripts/run_phase2_cohort.py @@ -41,6 +41,7 @@ "connection refused", "connection reset", "connection timed out", + "command failed (exit 137)", "connecterror", "context deadline exceeded", "docker build failed", diff --git a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py index 6e4debd23..f8613855c 100644 --- a/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py +++ b/examples/harbor-hermes-switchyard/tests/test_phase2_cohort.py @@ -196,6 +196,7 @@ def test_failure_classifier_retries_only_known_infrastructure_failures() -> None ) assert module.classify_failure("trusted fallback: provider returned HTTP 408") == "infrastructure" assert module.classify_failure("provider returned HTTP 400") == "harness_or_integration" + assert module.classify_failure("Command failed (exit 137): agent process") == "infrastructure" assert module.classify_failure("receipt did not prove plugin close") == "harness_or_integration"